From 0649270112bb6fb33fd3267ab0ee5518b7d52cff Mon Sep 17 00:00:00 2001 From: Kristoffer Moberg Christensen Date: Mon, 5 Aug 2024 15:29:12 +0200 Subject: [PATCH] Implement set-security-context feature for affinity assistant containers Ensures that when using Affinity Assistant, one can adhere to restricted pod security standards. Enables users to apply a container level securityContext for Affinity Assistants. --- docs/additional-configs.md | 3 +- .../affinityassistant_types.go | 6 + pkg/pod/pod.go | 20 +-- pkg/pod/pod_test.go | 12 +- pkg/pod/script.go | 4 +- pkg/pod/script_test.go | 14 +- pkg/pod/workingdir_init.go | 4 +- pkg/pod/workingdir_init_test.go | 4 +- .../pipelinerun/affinity_assistant.go | 24 ++- .../pipelinerun/affinity_assistant_test.go | 138 ++++++++++++++++-- 10 files changed, 185 insertions(+), 44 deletions(-) diff --git a/docs/additional-configs.md b/docs/additional-configs.md index 159fa5506b4..e71f09123e4 100644 --- a/docs/additional-configs.md +++ b/docs/additional-configs.md @@ -461,7 +461,8 @@ Out-of-the-box, Tekton Pipelines Controller is configured for relatively small-s To allow TaskRuns and PipelineRuns to run in namespaces with [restricted pod security standards](https://kubernetes.io/docs/concepts/security/pod-security-standards/), set the "set-security-context" feature flag to "true" in the [feature-flags configMap](#customizing-the-pipelines-controller-behavior). This configuration option applies a [SecurityContext](https://kubernetes.io/docs/tasks/configure-pod-container/security-context/) -to any containers injected into TaskRuns by the Pipelines controller. This SecurityContext may not be supported in all Kubernetes implementations (for example, OpenShift). +to any containers injected into TaskRuns by the Pipelines controller. If the [Affinity Assistants](affinityassistants.md) feature is enabled, the SecurityContext is also applied to those containers. +This SecurityContext may not be supported in all Kubernetes implementations (for example, OpenShift). **Note**: running TaskRuns and PipelineRuns in the "tekton-pipelines" namespace is discouraged. diff --git a/pkg/internal/affinityassistant/affinityassistant_types.go b/pkg/internal/affinityassistant/affinityassistant_types.go index e4fddb4e23d..15afe705b98 100644 --- a/pkg/internal/affinityassistant/affinityassistant_types.go +++ b/pkg/internal/affinityassistant/affinityassistant_types.go @@ -54,3 +54,9 @@ func GetAffinityAssistantBehavior(ctx context.Context) (AffinityAssistantBehavio return "", fmt.Errorf("unknown combination of disable-affinity-assistant: %v and coschedule: %v", disableAA, coschedule) } + +// ContainerConfig defines AffinityAssistant container configuration +type ContainerConfig struct { + Image string + SetSecurityContext bool +} diff --git a/pkg/pod/pod.go b/pkg/pod/pod.go index 1a8f444b597..ad58b473910 100644 --- a/pkg/pod/pod.go +++ b/pkg/pod/pod.go @@ -60,8 +60,8 @@ const ( // SpiffeCsiDriver is the CSI storage plugin needed for injection of SPIFFE workload api. SpiffeCsiDriver = "csi.spiffe.io" - // osSelectorLabel is the label Kubernetes uses for OS-specific workloads (https://kubernetes.io/docs/reference/labels-annotations-taints/#kubernetes-io-os) - osSelectorLabel = "kubernetes.io/os" + // OsSelectorLabel is the label Kubernetes uses for OS-specific workloads (https://kubernetes.io/docs/reference/labels-annotations-taints/#kubernetes-io-os) + OsSelectorLabel = "kubernetes.io/os" // TerminationReasonTimeoutExceeded indicates a step execution timed out. TerminationReasonTimeoutExceeded = "TimeoutExceeded" @@ -132,10 +132,10 @@ var ( allowPrivilegeEscalation = false runAsNonRoot = true - // The following security contexts allow init containers to run in namespaces + // LinuxSecurityContext allow init containers to run in namespaces // with "restricted" pod security admission // See https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted - linuxSecurityContext = &corev1.SecurityContext{ + LinuxSecurityContext = &corev1.SecurityContext{ AllowPrivilegeEscalation: &allowPrivilegeEscalation, Capabilities: &corev1.Capabilities{ Drop: []corev1.Capability{"ALL"}, @@ -145,7 +145,7 @@ var ( Type: corev1.SeccompProfileTypeRuntimeDefault, }, } - windowsSecurityContext = &corev1.SecurityContext{ + WindowsSecurityContext = &corev1.SecurityContext{ RunAsNonRoot: &runAsNonRoot, } ) @@ -607,9 +607,9 @@ func entrypointInitContainer(image string, steps []v1.Step, setSecurityContext, command = append(command, StepName(s.Name, i)) } volumeMounts := []corev1.VolumeMount{binMount, internalStepsMount} - securityContext := linuxSecurityContext + securityContext := LinuxSecurityContext if windows { - securityContext = windowsSecurityContext + securityContext = WindowsSecurityContext } // Rewrite steps with entrypoint binary. Append the entrypoint init @@ -679,9 +679,9 @@ func createResultsSidecar(taskSpec v1.TaskSpec, image string, setSecurityContext Image: image, Command: command, } - securityContext := linuxSecurityContext + securityContext := LinuxSecurityContext if windows { - securityContext = windowsSecurityContext + securityContext = WindowsSecurityContext } if setSecurityContext { sidecar.SecurityContext = securityContext @@ -696,7 +696,7 @@ func usesWindows(tr *v1.TaskRun) bool { if tr.Spec.PodTemplate == nil || tr.Spec.PodTemplate.NodeSelector == nil { return false } - osSelector := tr.Spec.PodTemplate.NodeSelector[osSelectorLabel] + osSelector := tr.Spec.PodTemplate.NodeSelector[OsSelectorLabel] return osSelector == "windows" } diff --git a/pkg/pod/pod_test.go b/pkg/pod/pod_test.go index c73c0242ee5..4b5fc667cc0 100644 --- a/pkg/pod/pod_test.go +++ b/pkg/pod/pod_test.go @@ -2162,7 +2162,7 @@ _EOF_ {Name: "tekton-internal-bin", ReadOnly: true, MountPath: "/tekton/bin"}, {Name: "tekton-internal-run-0", ReadOnly: true, MountPath: "/tekton/run/0"}, }, implicitVolumeMounts...), - SecurityContext: linuxSecurityContext, + SecurityContext: LinuxSecurityContext, }}, Volumes: append(implicitVolumes, binVolume, runVolume(0), downwardVolume, corev1.Volume{ Name: "tekton-creds-init-home-0", @@ -2403,7 +2403,7 @@ _EOF_ {Name: "tekton-internal-bin", ReadOnly: true, MountPath: "/tekton/bin"}, {Name: "tekton-internal-run-0", ReadOnly: true, MountPath: "/tekton/run/0"}, }, implicitVolumeMounts...), - SecurityContext: linuxSecurityContext, + SecurityContext: LinuxSecurityContext, }}, Volumes: append(implicitVolumes, binVolume, runVolume(0), downwardVolume, corev1.Volume{ Name: "tekton-creds-init-home-0", @@ -3424,7 +3424,7 @@ func TestPrepareInitContainers(t *testing.T) { WorkingDir: "/", Command: []string{"/ko-app/entrypoint", "init", "/ko-app/entrypoint", entrypointBinary, "step-foo", "step-bar"}, VolumeMounts: []corev1.VolumeMount{binMount, internalStepsMount}, - SecurityContext: linuxSecurityContext, + SecurityContext: LinuxSecurityContext, }, }, { name: "nothing-special-two-steps-windows", @@ -3456,7 +3456,7 @@ func TestPrepareInitContainers(t *testing.T) { WorkingDir: "/", Command: []string{"/ko-app/entrypoint", "init", "/ko-app/entrypoint", entrypointBinary, "step-foo", "step-bar"}, VolumeMounts: []corev1.VolumeMount{binMount, internalStepsMount}, - SecurityContext: windowsSecurityContext, + SecurityContext: WindowsSecurityContext, }, }} for _, tc := range tcs { @@ -3485,13 +3485,13 @@ func TestUsesWindows(t *testing.T) { }, { name: "uses linux", taskRun: &v1.TaskRun{Spec: v1.TaskRunSpec{PodTemplate: &pod.Template{NodeSelector: map[string]string{ - osSelectorLabel: "linux", + OsSelectorLabel: "linux", }}}}, want: false, }, { name: "uses windows", taskRun: &v1.TaskRun{Spec: v1.TaskRunSpec{PodTemplate: &pod.Template{NodeSelector: map[string]string{ - osSelectorLabel: "windows", + OsSelectorLabel: "windows", }}}}, want: true, }} diff --git a/pkg/pod/script.go b/pkg/pod/script.go index 70b64ba3472..611c109c6ee 100644 --- a/pkg/pod/script.go +++ b/pkg/pod/script.go @@ -87,13 +87,13 @@ func convertScripts(shellImageLinux string, shellImageWin string, steps []v1.Ste shellImage := shellImageLinux shellCommand := "sh" shellArg := "-c" - securityContext := linuxSecurityContext + securityContext := LinuxSecurityContext // Set windows variants for Image, Command and Args if requiresWindows { shellImage = shellImageWin shellCommand = "pwsh" shellArg = "-Command" - securityContext = windowsSecurityContext + securityContext = WindowsSecurityContext } placeScriptsInit := corev1.Container{ diff --git a/pkg/pod/script_test.go b/pkg/pod/script_test.go index 19d0c7523e7..b18856534f8 100644 --- a/pkg/pod/script_test.go +++ b/pkg/pod/script_test.go @@ -159,7 +159,7 @@ _EOF_ /tekton/bin/entrypoint decode-script "${scriptfile}" `}, VolumeMounts: []corev1.VolumeMount{writeScriptsVolumeMount, binMount}, - SecurityContext: linuxSecurityContext, + SecurityContext: LinuxSecurityContext, } want := []corev1.Container{{ Image: "step-1", @@ -465,7 +465,7 @@ fi debug-fail-continue-heredoc-randomly-generated-6nl7g `}, VolumeMounts: []corev1.VolumeMount{writeScriptsVolumeMount, binMount, debugScriptsVolumeMount}, - SecurityContext: linuxSecurityContext, + SecurityContext: LinuxSecurityContext, }, wantSteps: []corev1.Container{{ Image: "step-1", @@ -608,7 +608,7 @@ fi debug-beforestep-fail-continue-heredoc-randomly-generated-6nl7g `}, VolumeMounts: []corev1.VolumeMount{writeScriptsVolumeMount, binMount, debugScriptsVolumeMount}, - SecurityContext: linuxSecurityContext}, + SecurityContext: LinuxSecurityContext}, wantSteps: []corev1.Container{{ Name: "step-1", Image: "step-1", @@ -690,7 +690,7 @@ _EOF_ /tekton/bin/entrypoint decode-script "${scriptfile}" `}, VolumeMounts: []corev1.VolumeMount{writeScriptsVolumeMount, binMount}, - SecurityContext: linuxSecurityContext, + SecurityContext: LinuxSecurityContext, } want := []corev1.Container{{ Image: "step-1", @@ -777,7 +777,7 @@ no-shebang "@ | Out-File -FilePath /tekton/scripts/script-3-mssqb.cmd `}, VolumeMounts: []corev1.VolumeMount{writeScriptsVolumeMount, binMount}, - SecurityContext: windowsSecurityContext, + SecurityContext: WindowsSecurityContext, } want := []corev1.Container{{ Image: "step-1", @@ -860,7 +860,7 @@ sidecar-1 "@ | Out-File -FilePath /tekton/scripts/sidecar-script-0-mssqb `}, VolumeMounts: []corev1.VolumeMount{writeScriptsVolumeMount, binMount}, - SecurityContext: windowsSecurityContext, + SecurityContext: WindowsSecurityContext, } want := []corev1.Container{{ Image: "step-1", @@ -922,7 +922,7 @@ sidecar-1 "@ | Out-File -FilePath /tekton/scripts/sidecar-script-0-9l9zj `}, VolumeMounts: []corev1.VolumeMount{writeScriptsVolumeMount, binMount}, - SecurityContext: windowsSecurityContext, + SecurityContext: WindowsSecurityContext, } want := []corev1.Container{{ Image: "step-1", diff --git a/pkg/pod/workingdir_init.go b/pkg/pod/workingdir_init.go index af001dc104d..65da50b01e3 100644 --- a/pkg/pod/workingdir_init.go +++ b/pkg/pod/workingdir_init.go @@ -60,9 +60,9 @@ func workingDirInit(workingdirinitImage string, stepContainers []corev1.Containe // There are no workingDirs to initialize. return nil } - securityContext := linuxSecurityContext + securityContext := LinuxSecurityContext if windows { - securityContext = windowsSecurityContext + securityContext = WindowsSecurityContext } c := &corev1.Container{ diff --git a/pkg/pod/workingdir_init_test.go b/pkg/pod/workingdir_init_test.go index 05f1f65f133..c4a7b3b723d 100644 --- a/pkg/pod/workingdir_init_test.go +++ b/pkg/pod/workingdir_init_test.go @@ -90,7 +90,7 @@ func TestWorkingDirInit(t *testing.T) { Args: []string{"/workspace/bbb", "aaa", "zzz"}, WorkingDir: pipeline.WorkspaceDir, VolumeMounts: implicitVolumeMounts, - SecurityContext: linuxSecurityContext, + SecurityContext: LinuxSecurityContext, }, }, { desc: "workingDirs are unique and sorted, absolute dirs are ignored, uses windows", @@ -144,7 +144,7 @@ func TestWorkingDirInit(t *testing.T) { Args: []string{"/workspace/bbb", "aaa", "zzz"}, WorkingDir: pipeline.WorkspaceDir, VolumeMounts: implicitVolumeMounts, - SecurityContext: windowsSecurityContext, + SecurityContext: WindowsSecurityContext, }, }} { t.Run(c.desc, func(t *testing.T) { diff --git a/pkg/reconciler/pipelinerun/affinity_assistant.go b/pkg/reconciler/pipelinerun/affinity_assistant.go index 8c102da4905..27b9874effd 100644 --- a/pkg/reconciler/pipelinerun/affinity_assistant.go +++ b/pkg/reconciler/pipelinerun/affinity_assistant.go @@ -29,6 +29,7 @@ import ( v1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" "github.com/tektoncd/pipeline/pkg/internal/affinityassistant" aa "github.com/tektoncd/pipeline/pkg/internal/affinityassistant" + pipelinePod "github.com/tektoncd/pipeline/pkg/pod" "github.com/tektoncd/pipeline/pkg/reconciler/volumeclaim" "github.com/tektoncd/pipeline/pkg/workspace" appsv1 "k8s.io/api/apps/v1" @@ -139,7 +140,12 @@ func (c *Reconciler) createOrUpdateAffinityAssistant(ctx context.Context, affini if err != nil { return []error{err} } - affinityAssistantStatefulSet := affinityAssistantStatefulSet(aaBehavior, affinityAssistantName, pr, claimTemplates, claimNames, c.Images.NopImage, cfg.Defaults.DefaultAAPodTemplate) + affinityAssistantContainerConfig := aa.ContainerConfig{ + Image: c.Images.NopImage, + SetSecurityContext: cfg.FeatureFlags.SetSecurityContext, + } + + affinityAssistantStatefulSet := affinityAssistantStatefulSet(aaBehavior, affinityAssistantName, pr, claimTemplates, claimNames, affinityAssistantContainerConfig, cfg.Defaults.DefaultAAPodTemplate) _, err = c.KubeClientSet.AppsV1().StatefulSets(pr.Namespace).Create(ctx, affinityAssistantStatefulSet, metav1.CreateOptions{}) if err != nil { errs = append(errs, fmt.Errorf("failed to create StatefulSet %s: %w", affinityAssistantName, err)) @@ -281,7 +287,7 @@ func getStatefulSetLabels(pr *v1.PipelineRun, affinityAssistantName string) map[ // with the given AffinityAssistantTemplate applied to the StatefulSet PodTemplateSpec. // The VolumeClaimTemplates and Volume of StatefulSet reference the PipelineRun WorkspaceBinding VolumeClaimTempalte and the PVCs respectively. // The PVs created by the StatefulSet are scheduled to the same availability zone which avoids PV scheduling conflict. -func affinityAssistantStatefulSet(aaBehavior aa.AffinityAssistantBehavior, name string, pr *v1.PipelineRun, claimTemplates []corev1.PersistentVolumeClaim, claimNames []string, affinityAssistantImage string, defaultAATpl *pod.AffinityAssistantTemplate) *appsv1.StatefulSet { +func affinityAssistantStatefulSet(aaBehavior aa.AffinityAssistantBehavior, name string, pr *v1.PipelineRun, claimTemplates []corev1.PersistentVolumeClaim, claimNames []string, containerConfig aa.ContainerConfig, defaultAATpl *pod.AffinityAssistantTemplate) *appsv1.StatefulSet { // We want a singleton pod replicas := int32(1) @@ -296,9 +302,18 @@ func affinityAssistantStatefulSet(aaBehavior aa.AffinityAssistantBehavior, name mounts = append(mounts, corev1.VolumeMount{Name: claimTemplate.Name, MountPath: claimTemplate.Name}) } + securityContext := &corev1.SecurityContext{} + if containerConfig.SetSecurityContext { + securityContext = pipelinePod.LinuxSecurityContext + + if tpl.NodeSelector[pipelinePod.OsSelectorLabel] == "windows" { + securityContext = pipelinePod.WindowsSecurityContext + } + } + containers := []corev1.Container{{ Name: "affinity-assistant", - Image: affinityAssistantImage, + Image: containerConfig.Image, Args: []string{"tekton_run_indefinitely"}, // Set requests == limits to get QoS class _Guaranteed_. @@ -314,7 +329,8 @@ func affinityAssistantStatefulSet(aaBehavior aa.AffinityAssistantBehavior, name "memory": resource.MustParse("100Mi"), }, }, - VolumeMounts: mounts, + SecurityContext: securityContext, + VolumeMounts: mounts, }} var volumes []corev1.Volume diff --git a/pkg/reconciler/pipelinerun/affinity_assistant_test.go b/pkg/reconciler/pipelinerun/affinity_assistant_test.go index aa9c4fb7d16..30c5c273578 100644 --- a/pkg/reconciler/pipelinerun/affinity_assistant_test.go +++ b/pkg/reconciler/pipelinerun/affinity_assistant_test.go @@ -32,6 +32,7 @@ import ( "github.com/tektoncd/pipeline/pkg/apis/pipeline/pod" v1 "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1" aa "github.com/tektoncd/pipeline/pkg/internal/affinityassistant" + pipelinePod "github.com/tektoncd/pipeline/pkg/pod" "github.com/tektoncd/pipeline/pkg/reconciler/volumeclaim" "github.com/tektoncd/pipeline/pkg/workspace" "github.com/tektoncd/pipeline/test/diff" @@ -52,8 +53,14 @@ import ( ) var ( - podSpecFilter cmp.Option = cmpopts.IgnoreFields(corev1.PodSpec{}, "Containers", "Affinity") + podSpecFilter cmp.Option = cmpopts.IgnoreFields(corev1.PodSpec{}, "Affinity") podTemplateSpecFilter cmp.Option = cmpopts.IgnoreFields(corev1.PodTemplateSpec{}, "ObjectMeta") + podContainerFilter cmp.Option = cmpopts.IgnoreFields(corev1.Container{}, "Resources", "Args", "VolumeMounts") + + containerConfigWithoutSecurityContext = aa.ContainerConfig{ + Image: "nginx", + SetSecurityContext: false, + } ) var ( @@ -119,14 +126,31 @@ var testPRWithEmptyDir = &v1.PipelineRun{ }, } +var testPRWithWindowsOs = &v1.PipelineRun{ + ObjectMeta: metav1.ObjectMeta{Name: "pipelinerun-with-windows"}, + Spec: v1.PipelineRunSpec{ + TaskRunTemplate: v1.PipelineTaskRunTemplate{ + PodTemplate: &pod.PodTemplate{ + NodeSelector: map[string]string{pipelinePod.OsSelectorLabel: "windows"}, + }, + }, + Workspaces: []v1.WorkspaceBinding{{ + Name: "EmptyDir Workspace", + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }}, + }, +} + // TestCreateOrUpdateAffinityAssistantsAndPVCsPerPipelineRun tests to create and delete Affinity Assistants and PVCs // per pipelinerun for a given PipelineRun func TestCreateOrUpdateAffinityAssistantsAndPVCsPerPipelineRun(t *testing.T) { replicas := int32(1) + tests := []struct { name string pr *v1.PipelineRun expectStatefulSetSpec *appsv1.StatefulSetSpec + featureFlags map[string]string }{{ name: "PersistentVolumeClaim Workspace type", pr: testPRWithPVC, @@ -141,6 +165,10 @@ func TestCreateOrUpdateAffinityAssistantsAndPVCsPerPipelineRun(t *testing.T) { }, Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "affinity-assistant", + SecurityContext: &corev1.SecurityContext{}, + }}, Volumes: []corev1.Volume{{ Name: "workspace-0", VolumeSource: corev1.VolumeSource{ @@ -162,6 +190,14 @@ func TestCreateOrUpdateAffinityAssistantsAndPVCsPerPipelineRun(t *testing.T) { workspace.LabelComponent: workspace.ComponentNameAffinityAssistant, }, }, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "affinity-assistant", + SecurityContext: &corev1.SecurityContext{}, + }}, + }, + }, VolumeClaimTemplates: []corev1.PersistentVolumeClaim{{ ObjectMeta: metav1.ObjectMeta{Name: "pvc-b9eea16dce"}, }}, @@ -183,6 +219,10 @@ func TestCreateOrUpdateAffinityAssistantsAndPVCsPerPipelineRun(t *testing.T) { }}, Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "affinity-assistant", + SecurityContext: &corev1.SecurityContext{}, + }}, Volumes: []corev1.Volume{{ Name: "workspace-0", VolumeSource: corev1.VolumeSource{ @@ -204,17 +244,79 @@ func TestCreateOrUpdateAffinityAssistantsAndPVCsPerPipelineRun(t *testing.T) { workspace.LabelComponent: workspace.ComponentNameAffinityAssistant, }, }, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "affinity-assistant", + SecurityContext: &corev1.SecurityContext{}, + }}, + }, + }, + }, + }, { + name: "securityContext feature enabled and os is Windows", + pr: testPRWithWindowsOs, + featureFlags: map[string]string{ + "set-security-context": "true", + }, + expectStatefulSetSpec: &appsv1.StatefulSetSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + pipeline.PipelineRunLabelKey: testPRWithWindowsOs.Name, + workspace.LabelInstance: "affinity-assistant-01cecfbdec", + workspace.LabelComponent: workspace.ComponentNameAffinityAssistant, + }, + }, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + NodeSelector: map[string]string{pipelinePod.OsSelectorLabel: "windows"}, + Containers: []corev1.Container{{ + Name: "affinity-assistant", + SecurityContext: pipelinePod.WindowsSecurityContext, + }}, + }, + }, + }, + }, { + name: "securityContext feature enabled and os is Linux", + pr: testPRWithEmptyDir, + featureFlags: map[string]string{ + "set-security-context": "true", + }, + expectStatefulSetSpec: &appsv1.StatefulSetSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + pipeline.PipelineRunLabelKey: testPRWithEmptyDir.Name, + workspace.LabelInstance: "affinity-assistant-c655a0c8a2", + workspace.LabelComponent: workspace.ComponentNameAffinityAssistant, + }, + }, + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "affinity-assistant", + SecurityContext: pipelinePod.LinuxSecurityContext, + }}, + }, + }, }, }} for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - configMap := map[string]string{ + featureFlags := map[string]string{ "disable-affinity-assistant": "true", "coschedule": "pipelineruns", } + + for k, v := range tc.featureFlags { + featureFlags[k] = v + } + kubeClientSet := fakek8s.NewSimpleClientset() - ctx := cfgtesting.SetFeatureFlags(context.Background(), t, configMap) + ctx := cfgtesting.SetFeatureFlags(context.Background(), t, featureFlags) c := Reconciler{ KubeClientSet: kubeClientSet, pvcHandler: volumeclaim.NewPVCHandler(kubeClientSet, zap.NewExample().Sugar()), @@ -266,6 +368,10 @@ func TestCreateOrUpdateAffinityAssistantsAndPVCsPerWorkspaceOrDisabled(t *testin }, Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "affinity-assistant", + SecurityContext: &corev1.SecurityContext{}, + }}, Volumes: []corev1.Volume{{ Name: "workspace-0", VolumeSource: corev1.VolumeSource{ @@ -291,6 +397,10 @@ func TestCreateOrUpdateAffinityAssistantsAndPVCsPerWorkspaceOrDisabled(t *testin }, Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "affinity-assistant", + SecurityContext: &corev1.SecurityContext{}, + }}, Volumes: []corev1.Volume{{ Name: "workspace-0", VolumeSource: corev1.VolumeSource{ @@ -321,6 +431,10 @@ func TestCreateOrUpdateAffinityAssistantsAndPVCsPerWorkspaceOrDisabled(t *testin }, Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "affinity-assistant", + SecurityContext: &corev1.SecurityContext{}, + }}, Volumes: []corev1.Volume{{ Name: "workspace-0", VolumeSource: corev1.VolumeSource{ @@ -340,6 +454,10 @@ func TestCreateOrUpdateAffinityAssistantsAndPVCsPerWorkspaceOrDisabled(t *testin }, Template: corev1.PodTemplateSpec{ Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "affinity-assistant", + SecurityContext: &corev1.SecurityContext{}, + }}, Volumes: []corev1.Volume{{ Name: "workspace-0", VolumeSource: corev1.VolumeSource{ @@ -477,7 +595,7 @@ func TestCreateOrUpdateAffinityAssistantsAndPVCs_Failure(t *testing.T) { } // TestCreateOrUpdateAffinityAssistantWhenNodeIsCordoned tests an existing Affinity Assistant can identify the node failure and -// can migrate the affinity assistant pod to a healthy node so that the existing pipelineRun runs to competition +// can migrate the affinity assistant pod to a healthy node so that the existing pipelineRun runs to compleition func TestCreateOrUpdateAffinityAssistantWhenNodeIsCordoned(t *testing.T) { expectedAffinityAssistantName := GetAffinityAssistantName(workspacePVCName, testPRWithPVC.Name) @@ -627,7 +745,7 @@ func TestPipelineRunPodTemplatesArePropagatedToAffinityAssistant(t *testing.T) { }, } - stsWithOverridenTemplateFields := affinityAssistantStatefulSet(aa.AffinityAssistantPerWorkspace, "test-assistant", prWithCustomPodTemplate, []corev1.PersistentVolumeClaim{}, []string{}, "nginx", nil) + stsWithOverridenTemplateFields := affinityAssistantStatefulSet(aa.AffinityAssistantPerWorkspace, "test-assistant", prWithCustomPodTemplate, []corev1.PersistentVolumeClaim{}, []string{}, containerConfigWithoutSecurityContext, nil) if len(stsWithOverridenTemplateFields.Spec.Template.Spec.Tolerations) != 1 { t.Errorf("expected Tolerations in the StatefulSet") @@ -680,7 +798,7 @@ func TestDefaultPodTemplatesArePropagatedToAffinityAssistant(t *testing.T) { }, } - stsWithOverridenTemplateFields := affinityAssistantStatefulSet(aa.AffinityAssistantPerWorkspace, "test-assistant", prWithCustomPodTemplate, []corev1.PersistentVolumeClaim{}, []string{}, "nginx", defaultTpl) + stsWithOverridenTemplateFields := affinityAssistantStatefulSet(aa.AffinityAssistantPerWorkspace, "test-assistant", prWithCustomPodTemplate, []corev1.PersistentVolumeClaim{}, []string{}, containerConfigWithoutSecurityContext, defaultTpl) if len(stsWithOverridenTemplateFields.Spec.Template.Spec.Tolerations) != 1 { t.Errorf("expected Tolerations in the StatefulSet") @@ -736,7 +854,7 @@ func TestMergedPodTemplatesArePropagatedToAffinityAssistant(t *testing.T) { }, } - stsWithOverridenTemplateFields := affinityAssistantStatefulSet(aa.AffinityAssistantPerWorkspace, "test-assistant", prWithCustomPodTemplate, []corev1.PersistentVolumeClaim{}, []string{}, "nginx", defaultTpl) + stsWithOverridenTemplateFields := affinityAssistantStatefulSet(aa.AffinityAssistantPerWorkspace, "test-assistant", prWithCustomPodTemplate, []corev1.PersistentVolumeClaim{}, []string{}, containerConfigWithoutSecurityContext, defaultTpl) if len(stsWithOverridenTemplateFields.Spec.Template.Spec.Tolerations) != 1 { t.Errorf("expected Tolerations from spec in the StatefulSet") @@ -779,7 +897,7 @@ func TestOnlySelectPodTemplateFieldsArePropagatedToAffinityAssistant(t *testing. }, } - stsWithOverridenTemplateFields := affinityAssistantStatefulSet(aa.AffinityAssistantPerWorkspace, "test-assistant", prWithCustomPodTemplate, []corev1.PersistentVolumeClaim{}, []string{}, "nginx", nil) + stsWithOverridenTemplateFields := affinityAssistantStatefulSet(aa.AffinityAssistantPerWorkspace, "test-assistant", prWithCustomPodTemplate, []corev1.PersistentVolumeClaim{}, []string{}, containerConfigWithoutSecurityContext, nil) if len(stsWithOverridenTemplateFields.Spec.Template.Spec.Tolerations) != 1 { t.Errorf("expected Tolerations from spec in the StatefulSet") @@ -799,7 +917,7 @@ func TestThatTheAffinityAssistantIsWithoutNodeSelectorAndTolerations(t *testing. Spec: v1.PipelineRunSpec{}, } - stsWithoutTolerationsAndNodeSelector := affinityAssistantStatefulSet(aa.AffinityAssistantPerWorkspace, "test-assistant", prWithoutCustomPodTemplate, []corev1.PersistentVolumeClaim{}, []string{}, "nginx", nil) + stsWithoutTolerationsAndNodeSelector := affinityAssistantStatefulSet(aa.AffinityAssistantPerWorkspace, "test-assistant", prWithoutCustomPodTemplate, []corev1.PersistentVolumeClaim{}, []string{}, containerConfigWithoutSecurityContext, nil) if len(stsWithoutTolerationsAndNodeSelector.Spec.Template.Spec.Tolerations) != 0 { t.Errorf("unexpected Tolerations in the StatefulSet") @@ -1281,7 +1399,7 @@ func validateStatefulSetSpec(t *testing.T, ctx context.Context, c Reconciler, ex if err != nil { t.Fatalf("unexpected error when retrieving StatefulSet: %v", err) } - if d := cmp.Diff(expectStatefulSetSpec, &aa.Spec, podSpecFilter, podTemplateSpecFilter); d != "" { + if d := cmp.Diff(expectStatefulSetSpec, &aa.Spec, podSpecFilter, podTemplateSpecFilter, podContainerFilter); d != "" { t.Errorf("StatefulSetSpec diff: %s", diff.PrintWantGot(d)) } } else if !apierrors.IsNotFound(err) {