From e32e49bf77cd24cb8bdd4c007db851f52d355333 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Mon, 30 Dec 2019 13:51:21 +0100 Subject: [PATCH 1/6] Initial version PolicyEmptyDirSizeLimit --- policies/config.go | 7 +++ policies/config_test.go | 41 ++++++++++++++ policies/pod/empty_dir_size_limit.go | 69 +++++++++++++++++++++++ policies/pod/empty_dir_size_limit_test.go | 18 ++++++ server/policies.go | 7 +++ server/webhook_test.go | 6 ++ 6 files changed, 148 insertions(+) create mode 100644 policies/config_test.go create mode 100644 policies/pod/empty_dir_size_limit.go create mode 100644 policies/pod/empty_dir_size_limit_test.go diff --git a/policies/config.go b/policies/config.go index d7b4649..fe3cd8e 100644 --- a/policies/config.go +++ b/policies/config.go @@ -12,6 +12,11 @@ package policies +type MutateEmptyDirSizeLimit struct { + MaximumSizeLimit string `yaml:"maximum_size_limit"` + DefaultSizeLimit string `yaml:"default_size_limit"` +} + // Config contains configuration for Policies type Config struct { // PolicyRequireIngressExemptionClasses contains the Ingress classes that an exemption is required for @@ -22,4 +27,6 @@ type Config struct { // PolicyDefaultSeccompPolicy contains the seccomp policy that you want to be applied on Pods by default. // Defaults to 'runtime/default' PolicyDefaultSeccompPolicy string `yaml:"policy_default_seccomp_policy"` + + MutateEmptyDirSizeLimit MutateEmptyDirSizeLimit `yaml:"mutate_empty_dir_size_limit"` } diff --git a/policies/config_test.go b/policies/config_test.go new file mode 100644 index 0000000..16d1518 --- /dev/null +++ b/policies/config_test.go @@ -0,0 +1,41 @@ +package policies + +import ( + "reflect" + "testing" + + "gopkg.in/yaml.v2" +) + +func TestMutateEmptyDirSizeLimit(t *testing.T) { + specs := map[string]struct { + src string + exp MutateEmptyDirSizeLimit + }{ + + "all good": { + src: ` +mutate_empty_dir_size_limit: + maximum_size_limit: "1Gi" + default_size_limit: "512Mi" +`, + exp: MutateEmptyDirSizeLimit{ + MaximumSizeLimit: "1Gi", + DefaultSizeLimit: "512Mi", + }, + }, + } + for msg, spec := range specs { + t.Run(msg, func(t *testing.T) { + var cfg Config + err := yaml.Unmarshal([]byte(spec.src), &cfg) + if err != nil { + t.Fatalf("unexpected error: %+v", err) + } + if exp, got := spec.exp, cfg.MutateEmptyDirSizeLimit; !reflect.DeepEqual(exp, got) { + t.Errorf("expected %v but got %v", exp, got) + } + }) + } + +} diff --git a/policies/pod/empty_dir_size_limit.go b/policies/pod/empty_dir_size_limit.go new file mode 100644 index 0000000..b88e435 --- /dev/null +++ b/policies/pod/empty_dir_size_limit.go @@ -0,0 +1,69 @@ +// Copyright 2019 Cruise LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// https://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.package ingress + +package pod + +import ( + "context" + "fmt" + + "github.com/cruise-automation/k-rail/policies" + "github.com/cruise-automation/k-rail/resource" + admissionv1beta1 "k8s.io/api/admission/v1beta1" + apiresource "k8s.io/apimachinery/pkg/api/resource" +) + +type PolicyEmptyDirSizeLimit struct { + MaxSize, DefaultSize apiresource.Quantity +} + +func (p PolicyEmptyDirSizeLimit) Name() string { + return "pod_empty_dir_size_limit" +} + +const violationText = "Empty dir size limit: size limit is required for Pods that use emptyDir" + +func (p PolicyEmptyDirSizeLimit) Validate(ctx context.Context, config policies.Config, ar *admissionv1beta1.AdmissionRequest) ([]policies.ResourceViolation, []policies.PatchOperation) { + var resourceViolations []policies.ResourceViolation + + podResource := resource.GetPodResource(ar) + if podResource == nil { + return resourceViolations, nil + } + + var patches []policies.PatchOperation + + for i, volume := range podResource.PodSpec.Volumes { + if volume.EmptyDir == nil { + continue + } + if volume.EmptyDir.SizeLimit == nil || volume.EmptyDir.SizeLimit.IsZero() { + patches = append(patches, policies.PatchOperation{ + Op: "replace", + Path: fmt.Sprintf("/spec/volumes/%d/emptyDir/sizeLimit", i), + Value: p.DefaultSize.String(), + }) + continue + } + + if volume.EmptyDir.SizeLimit.Cmp(p.MaxSize) > 0 { + resourceViolations = append(resourceViolations, policies.ResourceViolation{ + Namespace: ar.Namespace, + ResourceName: podResource.ResourceName, + ResourceKind: podResource.ResourceKind, + Violation: violationText, + Policy: p.Name(), + }) + } + } + return resourceViolations, patches +} diff --git a/policies/pod/empty_dir_size_limit_test.go b/policies/pod/empty_dir_size_limit_test.go new file mode 100644 index 0000000..322e8f9 --- /dev/null +++ b/policies/pod/empty_dir_size_limit_test.go @@ -0,0 +1,18 @@ +package pod + +//func TestPolicyEmptyDirSizeLimit(t *testing.T) { +// specs := map[string]struct { +// max, def resource.Q +// exp bool +// }{ +// +// "": {true}, +// } +// for msg, spec := range specs { +// t.Run(msg, func(t *testing.T) { +// p +// _ = spec.exp +// }) +// } +// +//} diff --git a/server/policies.go b/server/policies.go index e37ba72..1a8aee4 100644 --- a/server/policies.go +++ b/server/policies.go @@ -20,6 +20,7 @@ import ( "github.com/cruise-automation/k-rail/policies/pod" log "github.com/sirupsen/logrus" admissionv1beta1 "k8s.io/api/admission/v1beta1" + "k8s.io/apimachinery/pkg/api/resource" ) // Policy specifies how a Policy is implemented @@ -39,6 +40,12 @@ func (s *Server) registerPolicies() { s.registerPolicy(pod.PolicyNoExec{}) s.registerPolicy(pod.PolicyBindMounts{}) s.registerPolicy(pod.PolicyDockerSock{}) + // fail fast on startup not on first request + limit := s.Config.PolicyConfig.MutateEmptyDirSizeLimit + s.registerPolicy(pod.PolicyEmptyDirSizeLimit{ + MaxSize: resource.MustParse(limit.MaximumSizeLimit), + DefaultSize: resource.MustParse(limit.DefaultSizeLimit)}, + ) s.registerPolicy(pod.PolicyImageImmutableReference{}) s.registerPolicy(pod.PolicyNoTiller{}) s.registerPolicy(pod.PolicyTrustedRepository{}) diff --git a/server/webhook_test.go b/server/webhook_test.go index 3c10aa3..8f3990a 100644 --- a/server/webhook_test.go +++ b/server/webhook_test.go @@ -56,6 +56,12 @@ func test_setup() (Server, []test) { ReportOnly: false, }, }, + PolicyConfig: policies.Config{ + MutateEmptyDirSizeLimit: policies.MutateEmptyDirSizeLimit{ + MaximumSizeLimit: "2Gi", + DefaultSizeLimit: "1Gi", + }, + }, }, Exemptions: compiledExemptions, } From 9fba29ff10395bf162510c510304d1350d389012 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Sat, 4 Jan 2020 14:59:01 +0100 Subject: [PATCH 2/6] Replace yaml parser; add config validation --- go.mod | 4 +-- policies/config.go | 52 ++++++++++++++++++++++----- policies/config_test.go | 54 +++++++++++++++++++++++----- policies/exemption.go | 12 +++---- policies/pod/empty_dir_size_limit.go | 7 ++-- server/config.go | 10 +++--- server/policies.go | 8 +---- server/server.go | 2 +- server/webhook_test.go | 5 +-- 9 files changed, 110 insertions(+), 44 deletions(-) diff --git a/go.mod b/go.mod index 5bceac2..3064c91 100644 --- a/go.mod +++ b/go.mod @@ -21,11 +21,11 @@ require ( golang.org/x/text v0.3.0 // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/inf.v0 v0.9.0 // indirect - gopkg.in/yaml.v2 v2.2.2 + gopkg.in/yaml.v2 v2.2.2 // indirect k8s.io/api v0.0.0-20190301173355-16f65c82b8fa k8s.io/apimachinery v0.0.0-20190301173222-2f7e9cae4418 k8s.io/klog v0.0.0-20181108234604-8139d8cb77af // indirect - sigs.k8s.io/yaml v1.1.0 // indirect + sigs.k8s.io/yaml v1.1.0 ) replace git.apache.org/thrift.git => github.com/apache/thrift v0.12.0 diff --git a/policies/config.go b/policies/config.go index fe3cd8e..e563776 100644 --- a/policies/config.go +++ b/policies/config.go @@ -12,21 +12,57 @@ package policies -type MutateEmptyDirSizeLimit struct { - MaximumSizeLimit string `yaml:"maximum_size_limit"` - DefaultSizeLimit string `yaml:"default_size_limit"` -} +import ( + "encoding/json" + "errors" + "fmt" + + apiresource "k8s.io/apimachinery/pkg/api/resource" +) // Config contains configuration for Policies type Config struct { // PolicyRequireIngressExemptionClasses contains the Ingress classes that an exemption is required for // to use. Typically this would include your public ingress classes. - PolicyRequireIngressExemptionClasses []string `yaml:"policy_require_ingress_exemption_classes"` + PolicyRequireIngressExemptionClasses []string `json:"policy_require_ingress_exemption_classes"` // PolicyTrustedRepositoryRegexes contains regexes that match image repositories that you want to allow. - PolicyTrustedRepositoryRegexes []string `yaml:"policy_trusted_repository_regexes"` + PolicyTrustedRepositoryRegexes []string `json:"policy_trusted_repository_regexes"` // PolicyDefaultSeccompPolicy contains the seccomp policy that you want to be applied on Pods by default. // Defaults to 'runtime/default' - PolicyDefaultSeccompPolicy string `yaml:"policy_default_seccomp_policy"` + PolicyDefaultSeccompPolicy string `json:"policy_default_seccomp_policy"` + + MutateEmptyDirSizeLimit MutateEmptyDirSizeLimit `json:"mutate_empty_dir_size_limit"` +} + +type MutateEmptyDirSizeLimit struct { + MaximumSizeLimit apiresource.Quantity `json:"maximum_size_limit"` + DefaultSizeLimit apiresource.Quantity `json:"default_size_limit"` +} + +func (m *MutateEmptyDirSizeLimit) UnmarshalJSON(value []byte) error { + var v map[string]json.RawMessage + if err := json.Unmarshal(value, &v); err != nil { + return err + } - MutateEmptyDirSizeLimit MutateEmptyDirSizeLimit `yaml:"mutate_empty_dir_size_limit"` + if max, ok := v["maximum_size_limit"]; ok { + if err := m.MaximumSizeLimit.UnmarshalJSON(max); err != nil { + return fmt.Errorf("maximum_size_limit failed: %s", err) + } + } + if def, ok := v["default_size_limit"]; ok { + if err := m.DefaultSizeLimit.UnmarshalJSON(def); err != nil { + return fmt.Errorf("default_size_limit failed: %s", err) + } + } + if m.DefaultSizeLimit.IsZero() { + return errors.New("default size must not be empty") + } + if m.MaximumSizeLimit.IsZero() { + return errors.New("max size must not be empty") + } + if m.DefaultSizeLimit.Cmp(m.MaximumSizeLimit) > 0 { + return errors.New("default size must not be greater than max size") + } + return nil } diff --git a/policies/config_test.go b/policies/config_test.go index 16d1518..f60d096 100644 --- a/policies/config_test.go +++ b/policies/config_test.go @@ -4,13 +4,15 @@ import ( "reflect" "testing" - "gopkg.in/yaml.v2" + apiresource "k8s.io/apimachinery/pkg/api/resource" + "sigs.k8s.io/yaml" ) func TestMutateEmptyDirSizeLimit(t *testing.T) { specs := map[string]struct { - src string - exp MutateEmptyDirSizeLimit + src string + exp *MutateEmptyDirSizeLimit + expErr bool }{ "all good": { @@ -19,20 +21,54 @@ mutate_empty_dir_size_limit: maximum_size_limit: "1Gi" default_size_limit: "512Mi" `, - exp: MutateEmptyDirSizeLimit{ - MaximumSizeLimit: "1Gi", - DefaultSizeLimit: "512Mi", + exp: &MutateEmptyDirSizeLimit{ + MaximumSizeLimit: apiresource.MustParse("1Gi"), + DefaultSizeLimit: apiresource.MustParse("512Mi"), }, }, + "default > max": { + src: ` +mutate_empty_dir_size_limit: + maximum_size_limit: "1Gi" + default_size_limit: "2Gi" +`, + expErr: true, + }, + "default not set": { + src: ` +mutate_empty_dir_size_limit: + maximum_size_limit: "1Gi" +`, + expErr: true, + }, + "max not set": { + src: ` +mutate_empty_dir_size_limit: + default_size_limit: "2Gi" +`, + expErr: true, + }, + "unsupported type": { + src: ` +mutate_empty_dir_size_limit: + default_size_limit: "2ALX" + maximum_size_limit: "2ALX" +`, + expErr: true, + }, } for msg, spec := range specs { t.Run(msg, func(t *testing.T) { var cfg Config - err := yaml.Unmarshal([]byte(spec.src), &cfg) - if err != nil { + switch err := yaml.Unmarshal([]byte(spec.src), &cfg); { + case spec.expErr && err != nil: + return + case spec.expErr: + t.Fatal("expected error") + case !spec.expErr && err != nil: t.Fatalf("unexpected error: %+v", err) } - if exp, got := spec.exp, cfg.MutateEmptyDirSizeLimit; !reflect.DeepEqual(exp, got) { + if exp, got := *spec.exp, cfg.MutateEmptyDirSizeLimit; !reflect.DeepEqual(exp, got) { t.Errorf("expected %v but got %v", exp, got) } }) diff --git a/policies/exemption.go b/policies/exemption.go index 203caa3..d7f26d6 100644 --- a/policies/exemption.go +++ b/policies/exemption.go @@ -19,17 +19,17 @@ import ( "github.com/gobwas/glob" log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" authenticationv1 "k8s.io/api/authentication/v1" + "sigs.k8s.io/yaml" ) // RawExemption is the configuration for a policy exemption type RawExemption struct { - ResourceName string `yaml:"resource_name"` - Namespace string `yaml:"namespace"` - Username string `yaml:"username"` - Group string `yaml:"group"` - ExemptPolicies []string `yaml:"exempt_policies"` + ResourceName string `json:"resource_name"` + Namespace string `json:"namespace"` + Username string `json:"username"` + Group string `json:"group"` + ExemptPolicies []string `json:"exempt_policies"` } // CompiledExemption is the compiled configuration for a policy exemption diff --git a/policies/pod/empty_dir_size_limit.go b/policies/pod/empty_dir_size_limit.go index b88e435..4b6f5c5 100644 --- a/policies/pod/empty_dir_size_limit.go +++ b/policies/pod/empty_dir_size_limit.go @@ -19,11 +19,9 @@ import ( "github.com/cruise-automation/k-rail/policies" "github.com/cruise-automation/k-rail/resource" admissionv1beta1 "k8s.io/api/admission/v1beta1" - apiresource "k8s.io/apimachinery/pkg/api/resource" ) type PolicyEmptyDirSizeLimit struct { - MaxSize, DefaultSize apiresource.Quantity } func (p PolicyEmptyDirSizeLimit) Name() string { @@ -40,6 +38,7 @@ func (p PolicyEmptyDirSizeLimit) Validate(ctx context.Context, config policies.C return resourceViolations, nil } + cfg := config.MutateEmptyDirSizeLimit var patches []policies.PatchOperation for i, volume := range podResource.PodSpec.Volumes { @@ -50,12 +49,12 @@ func (p PolicyEmptyDirSizeLimit) Validate(ctx context.Context, config policies.C patches = append(patches, policies.PatchOperation{ Op: "replace", Path: fmt.Sprintf("/spec/volumes/%d/emptyDir/sizeLimit", i), - Value: p.DefaultSize.String(), + Value: cfg.DefaultSizeLimit.String(), }) continue } - if volume.EmptyDir.SizeLimit.Cmp(p.MaxSize) > 0 { + if volume.EmptyDir.SizeLimit.Cmp(cfg.MaximumSizeLimit) > 0 { resourceViolations = append(resourceViolations, policies.ResourceViolation{ Namespace: ar.Namespace, ResourceName: podResource.ResourceName, diff --git a/server/config.go b/server/config.go index b4b38bf..39dd875 100644 --- a/server/config.go +++ b/server/config.go @@ -19,17 +19,17 @@ import ( type PolicySettings struct { Name string Enabled bool - ReportOnly bool `yaml:"report_only"` + ReportOnly bool `json:"report_only"` } type Config struct { - LogLevel string `yaml:"log_level"` - BlacklistedNamespaces []string `yaml:"blacklisted_namespaces"` + LogLevel string `json:"log_level"` + BlacklistedNamespaces []string `json:"blacklisted_namespaces"` TLS struct { Cert string Key string } - GlobalReportOnly bool `yaml:"global_report_only"` + GlobalReportOnly bool `json:"global_report_only"` Policies []PolicySettings - PolicyConfig policies.Config `yaml:"policy_config"` + PolicyConfig policies.Config `json:"policy_config"` } diff --git a/server/policies.go b/server/policies.go index 1a8aee4..afa6d1b 100644 --- a/server/policies.go +++ b/server/policies.go @@ -20,7 +20,6 @@ import ( "github.com/cruise-automation/k-rail/policies/pod" log "github.com/sirupsen/logrus" admissionv1beta1 "k8s.io/api/admission/v1beta1" - "k8s.io/apimachinery/pkg/api/resource" ) // Policy specifies how a Policy is implemented @@ -40,12 +39,7 @@ func (s *Server) registerPolicies() { s.registerPolicy(pod.PolicyNoExec{}) s.registerPolicy(pod.PolicyBindMounts{}) s.registerPolicy(pod.PolicyDockerSock{}) - // fail fast on startup not on first request - limit := s.Config.PolicyConfig.MutateEmptyDirSizeLimit - s.registerPolicy(pod.PolicyEmptyDirSizeLimit{ - MaxSize: resource.MustParse(limit.MaximumSizeLimit), - DefaultSize: resource.MustParse(limit.DefaultSizeLimit)}, - ) + s.registerPolicy(pod.PolicyEmptyDirSizeLimit{}) s.registerPolicy(pod.PolicyImageImmutableReference{}) s.registerPolicy(pod.PolicyNoTiller{}) s.registerPolicy(pod.PolicyTrustedRepository{}) diff --git a/server/server.go b/server/server.go index 74a57c5..b2f4870 100644 --- a/server/server.go +++ b/server/server.go @@ -25,7 +25,7 @@ import ( "github.com/cruise-automation/k-rail/policies" "github.com/gorilla/mux" log "github.com/sirupsen/logrus" - yaml "gopkg.in/yaml.v2" + "sigs.k8s.io/yaml" ) const ( diff --git a/server/webhook_test.go b/server/webhook_test.go index 8f3990a..b19271d 100644 --- a/server/webhook_test.go +++ b/server/webhook_test.go @@ -22,6 +22,7 @@ import ( admissionv1beta1 "k8s.io/api/admission/v1beta1" authenticationv1 "k8s.io/api/authentication/v1" corev1 "k8s.io/api/core/v1" + apiresource "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) @@ -58,8 +59,8 @@ func test_setup() (Server, []test) { }, PolicyConfig: policies.Config{ MutateEmptyDirSizeLimit: policies.MutateEmptyDirSizeLimit{ - MaximumSizeLimit: "2Gi", - DefaultSizeLimit: "1Gi", + MaximumSizeLimit: apiresource.MustParse("2Gi"), + DefaultSizeLimit: apiresource.MustParse("1Gi"), }, }, }, From e262eabe9d924b7603639a853579a391688a7d8d Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Mon, 6 Jan 2020 15:51:23 +0100 Subject: [PATCH 3/6] Support volume patch path for different resource kinds --- policies/pod/empty_dir_size_limit.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/policies/pod/empty_dir_size_limit.go b/policies/pod/empty_dir_size_limit.go index 4b6f5c5..2dacc14 100644 --- a/policies/pod/empty_dir_size_limit.go +++ b/policies/pod/empty_dir_size_limit.go @@ -33,7 +33,7 @@ const violationText = "Empty dir size limit: size limit is required for Pods tha func (p PolicyEmptyDirSizeLimit) Validate(ctx context.Context, config policies.Config, ar *admissionv1beta1.AdmissionRequest) ([]policies.ResourceViolation, []policies.PatchOperation) { var resourceViolations []policies.ResourceViolation - podResource := resource.GetPodResource(ar) + podResource := resource.GetPodResource(ar, ctx) if podResource == nil { return resourceViolations, nil } @@ -48,7 +48,7 @@ func (p PolicyEmptyDirSizeLimit) Validate(ctx context.Context, config policies.C if volume.EmptyDir.SizeLimit == nil || volume.EmptyDir.SizeLimit.IsZero() { patches = append(patches, policies.PatchOperation{ Op: "replace", - Path: fmt.Sprintf("/spec/volumes/%d/emptyDir/sizeLimit", i), + Path: fmt.Sprintf(volumePatchPath(podResource.ResourceKind)+"/%d/emptyDir/sizeLimit", i), Value: cfg.DefaultSizeLimit.String(), }) continue @@ -66,3 +66,16 @@ func (p PolicyEmptyDirSizeLimit) Validate(ctx context.Context, config policies.C } return resourceViolations, patches } + +const templateVolumePath = "/spec/template/spec/volumes" + +func volumePatchPath(podKind string) string { + nonTemplateKinds := map[string]string{ + "Pod": "/spec/volumes", + "CronJob": "/spec/jobTemplate/spec/template/spec/volumes", + } + if pathPath, ok := nonTemplateKinds[podKind]; ok { + return pathPath + } + return templateVolumePath +} From 10a98ac311217e43c011b0282d5e91e5d68d2594 Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Mon, 6 Jan 2020 17:40:15 +0100 Subject: [PATCH 4/6] Improve test coverate for PolicyEmptyDirSizeLimit and minor error msg update --- policies/pod/empty_dir_size_limit.go | 2 +- policies/pod/empty_dir_size_limit_test.go | 227 ++++++++++++++++++++-- 2 files changed, 212 insertions(+), 17 deletions(-) diff --git a/policies/pod/empty_dir_size_limit.go b/policies/pod/empty_dir_size_limit.go index 2dacc14..b27eca7 100644 --- a/policies/pod/empty_dir_size_limit.go +++ b/policies/pod/empty_dir_size_limit.go @@ -28,7 +28,7 @@ func (p PolicyEmptyDirSizeLimit) Name() string { return "pod_empty_dir_size_limit" } -const violationText = "Empty dir size limit: size limit is required for Pods that use emptyDir" +const violationText = "Empty dir size limit: size limit exceeds the max value" func (p PolicyEmptyDirSizeLimit) Validate(ctx context.Context, config policies.Config, ar *admissionv1beta1.AdmissionRequest) ([]policies.ResourceViolation, []policies.PatchOperation) { var resourceViolations []policies.ResourceViolation diff --git a/policies/pod/empty_dir_size_limit_test.go b/policies/pod/empty_dir_size_limit_test.go index 322e8f9..fc0bcbb 100644 --- a/policies/pod/empty_dir_size_limit_test.go +++ b/policies/pod/empty_dir_size_limit_test.go @@ -1,18 +1,213 @@ package pod -//func TestPolicyEmptyDirSizeLimit(t *testing.T) { -// specs := map[string]struct { -// max, def resource.Q -// exp bool -// }{ -// -// "": {true}, -// } -// for msg, spec := range specs { -// t.Run(msg, func(t *testing.T) { -// p -// _ = spec.exp -// }) -// } -// -//} +import ( + "context" + "encoding/json" + "reflect" + "testing" + + "github.com/cruise-automation/k-rail/policies" + admissionv1beta1 "k8s.io/api/admission/v1beta1" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" + apiresource "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +func TestEmptyDirSizeLimit(t *testing.T) { + config := policies.Config{ + MutateEmptyDirSizeLimit: policies.MutateEmptyDirSizeLimit{ + DefaultSizeLimit: *apiresource.NewQuantity(1, apiresource.DecimalSI), + MaximumSizeLimit: *apiresource.NewQuantity(10, apiresource.DecimalSI), + }, + } + + specs := map[string]struct { + src v1.PodSpec + expViolations []policies.ResourceViolation + expPatches []policies.PatchOperation + }{ + "limit set within range": { + src: v1.PodSpec{ + Volumes: []v1.Volume{{ + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + SizeLimit: apiresource.NewQuantity(2, apiresource.DecimalSI)}, + }, + }}, + }, + }, + "limit set within range with multiple volumes": { + src: v1.PodSpec{ + Volumes: []v1.Volume{{ + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + SizeLimit: apiresource.NewQuantity(2, apiresource.DecimalSI)}, + }, + }, { + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + SizeLimit: apiresource.NewQuantity(3, apiresource.DecimalSI)}, + }, + }}, + }, + }, + "set default value when 0": { + src: v1.PodSpec{ + Volumes: []v1.Volume{{ + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + SizeLimit: apiresource.NewQuantity(0, apiresource.DecimalExponent), + }, + }}, + }, + }, + expPatches: []policies.PatchOperation{ + { + Path: "/spec/template/spec/volumes/0/emptyDir/sizeLimit", + Op: "replace", + Value: "1", + }, + }, + }, "set default value when empty": { + src: v1.PodSpec{ + Volumes: []v1.Volume{{ + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }}, + }, + }, + expPatches: []policies.PatchOperation{ + { + Path: "/spec/template/spec/volumes/0/emptyDir/sizeLimit", + Op: "replace", + Value: "1", + }, + }, + }, + "set default value when empty with multiple": { + src: v1.PodSpec{ + Volumes: []v1.Volume{{ + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }}, { + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }}, + }, + }, + expPatches: []policies.PatchOperation{ + { + Path: "/spec/template/spec/volumes/0/emptyDir/sizeLimit", + Op: "replace", + Value: "1", + }, + { + Path: "/spec/template/spec/volumes/1/emptyDir/sizeLimit", + Op: "replace", + Value: "1", + }, + }, + }, + "allow max limit size": { + src: v1.PodSpec{ + Volumes: []v1.Volume{{ + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + SizeLimit: apiresource.NewQuantity(10, apiresource.DecimalSI), + }, + }}, + }, + }, + }, + "prevent greater than max limit size": { + src: v1.PodSpec{ + Volumes: []v1.Volume{{ + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{ + SizeLimit: apiresource.NewQuantity(11, apiresource.DecimalSI), + }, + }}, + }, + }, + expViolations: []policies.ResourceViolation{ + { + ResourceName: "test", + ResourceKind: "Deployment", + Namespace: "test", + Violation: "Empty dir size limit: size limit exceeds the max value", + Policy: "pod_empty_dir_size_limit", + }, + }, + }, + "skip non empty dir volume": { + src: v1.PodSpec{ + Volumes: []v1.Volume{{ + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{}, + }}, + }, + }, + }, + } + for msg, spec := range specs { + t.Run(msg, func(t *testing.T) { + policy := PolicyEmptyDirSizeLimit{} + v, p := policy.Validate(context.TODO(), config, asFakeAdmissionRequest(spec.src)) + if exp, got := spec.expViolations, v; !reflect.DeepEqual(exp, got) { + t.Errorf("expected %#v but got %#v", exp, got) + } + if exp, got := spec.expPatches, p; !reflect.DeepEqual(exp, got) { + t.Errorf("expected %#v but got %#v", exp, got) + } + + }) + } +} + +func TestResourceVolumePatchPath(t *testing.T) { + specs := map[string]string{ + "Pod": "/spec/volumes", + "ReplicationController": "/spec/template/spec/volumes", + "Deployment": "/spec/template/spec/volumes", + "ReplicaSet": "/spec/template/spec/volumes", + "DaemonSet": "/spec/template/spec/volumes", + "StatefulSet": "/spec/template/spec/volumes", + "Job": "/spec/template/spec/volumes", + "CronJob": "/spec/jobTemplate/spec/template/spec/volumes", + } + for kind, exp := range specs { + t.Run(kind, func(t *testing.T) { + got := volumePatchPath(kind) + if exp != got { + t.Errorf("expected %q but got %q", exp, got) + } + }) + } + +} + +func asFakeAdmissionRequest(src v1.PodSpec) *admissionv1beta1.AdmissionRequest { + xxx := appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{Name: "test"}, + Spec: appsv1.DeploymentSpec{ + Template: v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{}, + Spec: src, + }, + }, + Status: appsv1.DeploymentStatus{}, + } + b, err := json.Marshal(&xxx) + if err != nil { + panic(err) + } + return &admissionv1beta1.AdmissionRequest{ + Resource: metav1.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}, + Name: "any", + Namespace: "test", + Object: runtime.RawExtension{Raw: b}, + } +} From 7811278780734bd2197f7fa77b6c506197729aeb Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Mon, 6 Jan 2020 17:51:42 +0100 Subject: [PATCH 5/6] Enable policy in helm chart --- deploy/helm/values.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 2815b08..4dfc9fd 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -73,6 +73,9 @@ config: - name: "pod_mutate_safe_to_evict" enabled: True report_only: False + - name: "pod_empty_dir_size_limit" + enabled: True + report_only: False - name: "pod_default_seccomp_policy" enabled: True report_only: False From f8487970d1bf6a833559bd00f793286205d81abd Mon Sep 17 00:00:00 2001 From: Alex Peters Date: Tue, 7 Jan 2020 11:06:06 +0100 Subject: [PATCH 6/6] Doc and helm config --- README.md | 23 ++++++++++++++++++++--- deploy/helm/values.yaml | 3 +++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2c68f44..fc7f43a 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,10 @@ k-rail is a workload policy enforcement tool for Kubernetes. It can help you sec * [No Exec](#no-exec) * [No Bind Mounts](#no-bind-mounts) * [No Docker Sock Mount](#no-docker-sock-mount) - * [Mutate Default Seccomp Profile](#mutate-default-seccomp-profile) + * [EmptyDir size limit](#emptyDir-size-limit) + [Policy configuration](#policy-configuration) + * [Mutate Default Seccomp Profile](#mutate-default-seccomp-profile) + + [Policy configuration](#policy-configuration-1) * [Immutable Image Reference](#immutable-image-reference) * [No Host Network](#no-host-network) * [No Host PID](#no-host-pid) @@ -28,11 +30,11 @@ k-rail is a workload policy enforcement tool for Kubernetes. It can help you sec * [No Privileged Container](#no-privileged-container) * [No Helm Tiller](#no-helm-tiller) * [Trusted Image Repository](#trusted-image-repository) - + [Policy configuration](#policy-configuration-1) + + [Policy configuration](#policy-configuration-2) * [Safe to Evict (DEPRECATED)](#safe-to-evict--deprecated) * [Mutate Safe to Evict](#mutate-safe-to-evict) * [Require Ingress Exemption](#require-ingress-exemption) - + [Policy configuration](#policy-configuration-2) + + [Policy configuration](#policy-configuration-3) - [Configuration](#configuration) * [Logging](#logging) * [Modes of operation](#modes-of-operation) @@ -230,6 +232,21 @@ The Docker socket bind mount provides API access to the host Docker daemon, whic **Note:** It is recommended to use the `No Bind Mounts` policy to disable all `hostPath` mounts rather than only this policy. +## EmptyDir size limit +By [default](https://kubernetes.io/docs/concepts/storage/volumes/#example-pod), an `emptyDir` lacks a `sizeLimit` parameter, and is disk-based; +a Pod with access to said `emptyDir` can consume the Node's entire disk (i.e. the limit is unbounded) until the offending Pod is deleted or evicted, which can constitute a denial-of-service condition at the affected Node (i.e. DiskPressure). +This policy +* sets the configured default size when none is set for an `emptyDir` volume +* reports a violation when the size is greater then the configured max size + +### Policy configuration +```yaml +policy_config: + mutate_empty_dir_size_limit: + maximum_size_limit: "1Gi" + default_size_limit: "512Mi" +``` + ## Mutate Default Seccomp Profile Sets a default seccomp profile (`runtime/default` or a configured one) for Pods if they have no existing seccomp configuration. The default seccomp policy for Docker and Containerd both block over 40 syscalls, [many of which](https://docs.docker.com/engine/security/seccomp/#significant-syscalls-blocked-by-the-default-profile) are potentially dangerous. The default policies are [usually very compatible](https://blog.jessfraz.com/post/containers-security-and-echo-chambers/#breaking-changes) with applications, too. diff --git a/deploy/helm/values.yaml b/deploy/helm/values.yaml index 4dfc9fd..bb48e68 100644 --- a/deploy/helm/values.yaml +++ b/deploy/helm/values.yaml @@ -36,6 +36,9 @@ config: - '^k8s.gcr.io/.*' # official k8s GCR repo - '^[A-Za-z0-9\-:@]+$' # official docker hub images policy_default_seccomp_policy: "runtime/default" + mutate_empty_dir_size_limit: + maximum_size_limit: "1Gi" + default_size_limit: "512Mi" policies: - name: "pod_no_exec" enabled: True