diff --git a/PROJECT b/PROJECT index f23d7108..8a104804 100644 --- a/PROJECT +++ b/PROJECT @@ -1,5 +1,8 @@ domain: operator.containers.carbonblack.io layout: go.kubebuilder.io/v3 +plugins: + manifests.sdk.operatorframework.io/v2: {} + scorecard.sdk.operatorframework.io/v2: {} projectName: cbcontainers repo: github.com/vmware/cbcontainers-operator resources: @@ -7,7 +10,6 @@ resources: crdVersion: v1beta1 controller: true domain: operator.containers.carbonblack.io - group: kind: CBContainersCluster path: github.com/vmware/cbcontainers-operator/api/v1 version: v1 @@ -15,11 +17,14 @@ resources: crdVersion: v1beta1 controller: true domain: operator.containers.carbonblack.io - group: kind: CBContainersHardening path: github.com/vmware/cbcontainers-operator/api/v1 version: v1 +- api: + crdVersion: v1beta1 + controller: true + domain: operator.containers.carbonblack.io + kind: CBContainersRuntime + path: github.com/vmware/cbcontainers-operator/api/v1 + version: v1 version: "3" -plugins: - manifests.sdk.operatorframework.io/v2: {} - scorecard.sdk.operatorframework.io/v2: {} diff --git a/README.md b/README.md index 294a63e9..19a7ee97 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ the State Reporter components that are responsible for reporting the cluster sta * Notice that without the first CR, the Hardening components won't be able to work. -```sh +```yaml apiVersion: operator.containers.carbonblack.io/v1 kind: CBContainersHardening metadata: @@ -89,6 +89,11 @@ spec: host: {EVENTS_HOST} ``` +#### 2.3 Apply the Carbon Black Container Runtime CR +cbcontainersruntimes.operator.containers.carbonblack.io + +TODO + ### Uninstalling the Carbon Black Cloud Container Operator ```sh make undeploy @@ -106,7 +111,7 @@ You can create a ClusterRole and bind it with ClusterRoleBinding to the service If you don't have such cluster role & cluster role binding configured, you can use the following: Cluster Role: -```sh +```yaml apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: @@ -115,7 +120,7 @@ rules: - nonResourceURLs: - /metrics verbs: - - get + - get ``` Cluster Role binding creation: @@ -127,7 +132,7 @@ kubectl create clusterrolebinding metrics --clusterrole=cbcontainers-metrics-rea Use the following ServiceMonitor to start scraping metrics from the CBContainers operator: * Make sure that your Prometheus custom resource service monitor selectors match it. -``` +```yaml apiVersion: monitoring.coreos.com/v1 kind: ServiceMonitor metadata: @@ -160,7 +165,7 @@ kubectl set env -n cbcontainers-dataplane deployment cbcontainers-operator HTTP_ In order to configure those environment variables for the Hardening Enforcer and the Hardening State Reporter components, update the Hardening CR using the proxy environment variables: -```sh +```yaml spec: enforcerSpec: env: diff --git a/api/v1/cbcontainerscluster_types.go b/api/v1/cbcontainerscluster_types.go index 5a4513e3..af31191f 100644 --- a/api/v1/cbcontainerscluster_types.go +++ b/api/v1/cbcontainerscluster_types.go @@ -25,28 +25,10 @@ import ( // CBContainersClusterSpec defines the desired state of CBContainersCluster type CBContainersClusterSpec struct { - Account string `json:"account,required"` - ClusterName string `json:"clusterName,required"` - ApiGatewaySpec CBContainersClusterApiGatewaySpec `json:"apiGatewaySpec,required"` - EventsGatewaySpec CBContainersClusterEventsGatewaySpec `json:"eventsGatewaySpec,required"` -} - -type CBContainersClusterEventsGatewaySpec struct { - Host string `json:"host,required"` - // +kubebuilder:default:=443 - Port int `json:"port,omitempty"` -} - -type CBContainersClusterApiGatewaySpec struct { - // +kubebuilder:default:="https" - Scheme string `json:"scheme,omitempty"` - Host string `json:"host,required"` - // +kubebuilder:default:=443 - Port int `json:"port,omitempty"` - // +kubebuilder:default:="containers" - Adapter string `json:"adapter,omitempty"` - // +kubebuilder:default:="cbcontainers-access-token" - AccessTokenSecretName string `json:"accessTokenSecretName,omitempty"` + Account string `json:"account,required"` + ClusterName string `json:"clusterName,required"` + ApiGatewaySpec CBContainersApiGatewaySpec `json:"apiGatewaySpec,required"` + EventsGatewaySpec CBContainersEventsGatewaySpec `json:"eventsGatewaySpec,required"` } // CBContainersClusterStatus defines the observed state of CBContainersCluster diff --git a/api/v1/cbcontainershardening_types.go b/api/v1/cbcontainershardening_types.go index 03faa3b9..9d5a0134 100644 --- a/api/v1/cbcontainershardening_types.go +++ b/api/v1/cbcontainershardening_types.go @@ -24,50 +24,15 @@ import ( // EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! // NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. -type CBContainersHardeningProbesSpec struct { - // +kubebuilder:default:="/ready" - ReadinessPath string `json:"readinessPath,omitempty"` - // +kubebuilder:default:="/alive" - LivenessPath string `json:"livenessPath,omitempty"` - // +kubebuilder:default:=8181 - Port int `json:"port,omitempty"` - // +kubebuilder:default:="HTTP" - Scheme coreV1.URIScheme `json:"scheme,omitempty"` - // +kubebuilder:default:=3 - InitialDelaySeconds int32 `json:"initialDelaySeconds,omitempty"` - // +kubebuilder:default:=1 - TimeoutSeconds int32 `json:"timeoutSeconds,omitempty"` - // +kubebuilder:default:=30 - PeriodSeconds int32 `json:"periodSeconds,omitempty"` - // +kubebuilder:default:=1 - SuccessThreshold int32 `json:"successThreshold,omitempty"` - // +kubebuilder:default:=3 - FailureThreshold int32 `json:"failureThreshold,omitempty"` -} - -type CBContainersHardeningPrometheusSpec struct { - // +kubebuilder:default:=false - Enabled *bool `json:"enabled,omitempty"` - // +kubebuilder:default:=7071 - Port int `json:"port,omitempty"` -} - -type CBContainersHardeningImageSpec struct { - Repository string `json:"repository,omitempty"` - Tag string `json:"tag,omitempty"` - // +kubebuilder:default:="Always" - PullPolicy coreV1.PullPolicy `json:"pullPolicy,omitempty"` -} - type CBContainersHardeningSpec struct { - Version string `json:"version,required"` + Version string `json:"version,required"` + EventsGatewaySpec CBContainersEventsGatewaySpec `json:"eventsGatewaySpec,required"` // +kubebuilder:default:="cbcontainers-access-token" AccessTokenSecretName string `json:"accessTokenSecretName,omitempty"` // +kubebuilder:default:=<> EnforcerSpec CBContainersHardeningEnforcerSpec `json:"enforcerSpec,omitempty"` // +kubebuilder:default:=<> StateReporterSpec CBContainersHardeningStateReporterSpec `json:"stateReporterSpec,omitempty"` - EventsGatewaySpec CBContainersHardeningEventsGatewaySpec `json:"eventsGatewaySpec,required"` } type CBContainersHardeningStateReporterSpec struct { @@ -78,19 +43,13 @@ type CBContainersHardeningStateReporterSpec struct { // +kubebuilder:default:=<> PodTemplateAnnotations map[string]string `json:"podTemplateAnnotations,omitempty"` // +kubebuilder:default:={repository:"cbartifactory/guardrails-state-reporter"} - Image CBContainersHardeningImageSpec `json:"image,omitempty"` + Image CBContainersImageSpec `json:"image,omitempty"` // +kubebuilder:default:=<> Env map[string]string `json:"env,omitempty"` // +kubebuilder:default:={requests: {memory: "64Mi", cpu: "30m"}, limits: {memory: "256Mi", cpu: "200m"}} Resources coreV1.ResourceRequirements `json:"resources,omitempty"` // +kubebuilder:default:=<> - Probes CBContainersHardeningProbesSpec `json:"probes,omitempty"` -} - -type CBContainersHardeningEventsGatewaySpec struct { - Host string `json:"host,required"` - // +kubebuilder:default:=443 - Port int `json:"port,omitempty"` + Probes CBContainersHTTPProbesSpec `json:"probes,omitempty"` } type CBContainersHardeningEnforcerSpec struct { @@ -105,13 +64,13 @@ type CBContainersHardeningEnforcerSpec struct { // +kubebuilder:default:=<> Env map[string]string `json:"env,omitempty"` // +kubebuilder:default:=<> - Prometheus CBContainersHardeningPrometheusSpec `json:"prometheus,omitempty"` + Prometheus CBContainersPrometheusSpec `json:"prometheus,omitempty"` // +kubebuilder:default:={repository:"cbartifactory/guardrails-enforcer"} - Image CBContainersHardeningImageSpec `json:"image,omitempty"` + Image CBContainersImageSpec `json:"image,omitempty"` // +kubebuilder:default:={requests: {memory: "64Mi", cpu: "30m"}, limits: {memory: "256Mi", cpu: "200m"}} Resources coreV1.ResourceRequirements `json:"resources,omitempty"` // +kubebuilder:default:=<> - Probes CBContainersHardeningProbesSpec `json:"probes,omitempty"` + Probes CBContainersHTTPProbesSpec `json:"probes,omitempty"` // +kubebuilder:default:=5 WebhookTimeoutSeconds int32 `json:"webhookTimeoutSeconds,omitempty"` } diff --git a/api/v1/cbcontainersruntime_types.go b/api/v1/cbcontainersruntime_types.go new file mode 100644 index 00000000..6ca90c92 --- /dev/null +++ b/api/v1/cbcontainersruntime_types.go @@ -0,0 +1,110 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1 + +import ( + coreV1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type CBContainersRuntimeResolverSpec struct { + EventsGatewaySpec CBContainersEventsGatewaySpec `json:"eventsGatewaySpec,required"` + // +kubebuilder:default:=<> + Labels map[string]string `json:"labels,omitempty"` + // +kubebuilder:default:=<> + DeploymentAnnotations map[string]string `json:"deploymentAnnotations,omitempty"` + // +kubebuilder:default:={prometheus.io/scrape: "false", prometheus.io/port: "7071"} + PodTemplateAnnotations map[string]string `json:"podTemplateAnnotations,omitempty"` + // +kubebuilder:default:=1 + ReplicasCount *int32 `json:"replicasCount,omitempty"` + // +kubebuilder:default:=<> + Env map[string]string `json:"env,omitempty"` + // +kubebuilder:default:={repository:"cbartifactory/runtime-kubernetes-resolver"} + Image CBContainersImageSpec `json:"image,omitempty"` + // +kubebuilder:default:={requests: {memory: "64Mi", cpu: "200m"}, limits: {memory: "128Mi", cpu: "600m"}} + Resources coreV1.ResourceRequirements `json:"resources,omitempty"` + // +kubebuilder:default:=<> + Probes CBContainersHTTPProbesSpec `json:"probes,omitempty"` + // +kubebuilder:default:=<> + Prometheus CBContainersPrometheusSpec `json:"prometheus,omitempty"` +} + +type CBContainersRuntimeSensorSpec struct { + // +kubebuilder:default:=<> + Labels map[string]string `json:"labels,omitempty"` + // +kubebuilder:default:=<> + DaemonSetAnnotations map[string]string `json:"daemonSetAnnotations,omitempty"` + // +kubebuilder:default:={prometheus.io/scrape: "false", prometheus.io/port: "7071"} + PodTemplateAnnotations map[string]string `json:"podTemplateAnnotations,omitempty"` + // +kubebuilder:default:=<> + Env map[string]string `json:"env,omitempty"` + // +kubebuilder:default:={repository:"cbartifactory/runtime-kubernetes-sensor"} + Image CBContainersImageSpec `json:"image,omitempty"` + // +kubebuilder:default:={requests: {memory: "1Gi", cpu: "400m"}, limits: {memory: "2Gi", cpu: "1"}} + Resources coreV1.ResourceRequirements `json:"resources,omitempty"` + // +kubebuilder:default:=<> + Probes CBContainersFileProbesSpec `json:"probes,omitempty"` + // +kubebuilder:default:=<> + Prometheus CBContainersPrometheusSpec `json:"prometheus,omitempty"` + // +kubebuilder:default:=2 + VerbosityLevel *int `json:"verbosity_level,omitempty"` +} + +// CBContainersRuntimeSpec defines the desired state of CBContainersRuntime +type CBContainersRuntimeSpec struct { + Version string `json:"version,required"` + // +kubebuilder:default:="cbcontainers-access-token" + AccessTokenSecretName string `json:"accessTokenSecretName,omitempty"` + // +kubebuilder:default:=<> + ResolverSpec CBContainersRuntimeResolverSpec `json:"resolverSpec,omitempty"` + // +kubebuilder:default:=<> + SensorSpec CBContainersRuntimeSensorSpec `json:"sensorSpec,omitempty"` + // +kubebuilder:default:=443 + InternalGrpcPort int32 `json:"internalGrpcPort,omitempty"` +} + +// CBContainersRuntimeStatus defines the observed state of CBContainersRuntime +type CBContainersRuntimeStatus struct { + // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + // Important: Run "make" to regenerate code after modifying this file +} + +// +kubebuilder:object:root=true +// +kubebuilder:subresource:status +// +kubebuilder:resource:scope=Cluster + +// CBContainersRuntime is the Schema for the cbcontainersruntimes API +type CBContainersRuntime struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec CBContainersRuntimeSpec `json:"spec,omitempty"` + Status CBContainersRuntimeStatus `json:"status,omitempty"` +} + +// +kubebuilder:object:root=true + +// CBContainersRuntimeList contains a list of CBContainersRuntime +type CBContainersRuntimeList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []CBContainersRuntime `json:"items"` +} + +func init() { + SchemeBuilder.Register(&CBContainersRuntime{}, &CBContainersRuntimeList{}) +} diff --git a/api/v1/gateway_types.go b/api/v1/gateway_types.go new file mode 100644 index 00000000..8b8da905 --- /dev/null +++ b/api/v1/gateway_types.go @@ -0,0 +1,19 @@ +package v1 + +type CBContainersEventsGatewaySpec struct { + Host string `json:"host,required"` + // +kubebuilder:default:=443 + Port int `json:"port,omitempty"` +} + +type CBContainersApiGatewaySpec struct { + Host string `json:"host,required"` + // +kubebuilder:default:="https" + Scheme string `json:"scheme,omitempty"` + // +kubebuilder:default:=443 + Port int `json:"port,omitempty"` + // +kubebuilder:default:="containers" + Adapter string `json:"adapter,omitempty"` + // +kubebuilder:default:="cbcontainers-access-token" + AccessTokenSecretName string `json:"accessTokenSecretName,omitempty"` +} diff --git a/api/v1/image_types.go b/api/v1/image_types.go new file mode 100644 index 00000000..9e8f04d1 --- /dev/null +++ b/api/v1/image_types.go @@ -0,0 +1,12 @@ +package v1 + +import ( + coreV1 "k8s.io/api/core/v1" +) + +type CBContainersImageSpec struct { + Repository string `json:"repository,omitempty"` + Tag string `json:"tag,omitempty"` + // +kubebuilder:default:="Always" + PullPolicy coreV1.PullPolicy `json:"pullPolicy,omitempty"` +} diff --git a/api/v1/metrics_types.go b/api/v1/metrics_types.go new file mode 100644 index 00000000..76691c0f --- /dev/null +++ b/api/v1/metrics_types.go @@ -0,0 +1,8 @@ +package v1 + +type CBContainersPrometheusSpec struct { + // +kubebuilder:default:=false + Enabled *bool `json:"enabled,omitempty"` + // +kubebuilder:default:=7071 + Port int `json:"port,omitempty"` +} diff --git a/api/v1/probe_types.go b/api/v1/probe_types.go new file mode 100644 index 00000000..37166301 --- /dev/null +++ b/api/v1/probe_types.go @@ -0,0 +1,38 @@ +package v1 + +import ( + coreV1 "k8s.io/api/core/v1" +) + +type CBContainersCommonProbesSpec struct { + // +kubebuilder:default:=3 + InitialDelaySeconds int32 `json:"initialDelaySeconds,omitempty"` + // +kubebuilder:default:=1 + TimeoutSeconds int32 `json:"timeoutSeconds,omitempty"` + // +kubebuilder:default:=30 + PeriodSeconds int32 `json:"periodSeconds,omitempty"` + // +kubebuilder:default:=1 + SuccessThreshold int32 `json:"successThreshold,omitempty"` + // +kubebuilder:default:=3 + FailureThreshold int32 `json:"failureThreshold,omitempty"` +} + +type CBContainersHTTPProbesSpec struct { + CBContainersCommonProbesSpec `json:",inline"` + // +kubebuilder:default:="/ready" + ReadinessPath string `json:"readinessPath,omitempty"` + // +kubebuilder:default:="/alive" + LivenessPath string `json:"livenessPath,omitempty"` + // +kubebuilder:default:=8181 + Port int `json:"port,omitempty"` + // +kubebuilder:default:="HTTP" + Scheme coreV1.URIScheme `json:"scheme,omitempty"` +} + +type CBContainersFileProbesSpec struct { + CBContainersCommonProbesSpec `json:",inline"` + // +kubebuilder:default:="/tmp/ready" + ReadinessPath string `json:"readinessPath,omitempty"` + // +kubebuilder:default:="/tmp/alive" + LivenessPath string `json:"livenessPath,omitempty"` +} diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 400cac7d..075979e7 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -24,6 +24,21 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CBContainersApiGatewaySpec) DeepCopyInto(out *CBContainersApiGatewaySpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersApiGatewaySpec. +func (in *CBContainersApiGatewaySpec) DeepCopy() *CBContainersApiGatewaySpec { + if in == nil { + return nil + } + out := new(CBContainersApiGatewaySpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CBContainersCluster) DeepCopyInto(out *CBContainersCluster) { *out = *in @@ -51,36 +66,6 @@ func (in *CBContainersCluster) DeepCopyObject() runtime.Object { return nil } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CBContainersClusterApiGatewaySpec) DeepCopyInto(out *CBContainersClusterApiGatewaySpec) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersClusterApiGatewaySpec. -func (in *CBContainersClusterApiGatewaySpec) DeepCopy() *CBContainersClusterApiGatewaySpec { - if in == nil { - return nil - } - out := new(CBContainersClusterApiGatewaySpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CBContainersClusterEventsGatewaySpec) DeepCopyInto(out *CBContainersClusterEventsGatewaySpec) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersClusterEventsGatewaySpec. -func (in *CBContainersClusterEventsGatewaySpec) DeepCopy() *CBContainersClusterEventsGatewaySpec { - if in == nil { - return nil - } - out := new(CBContainersClusterEventsGatewaySpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CBContainersClusterList) DeepCopyInto(out *CBContainersClusterList) { *out = *in @@ -145,6 +130,68 @@ func (in *CBContainersClusterStatus) DeepCopy() *CBContainersClusterStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CBContainersCommonProbesSpec) DeepCopyInto(out *CBContainersCommonProbesSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersCommonProbesSpec. +func (in *CBContainersCommonProbesSpec) DeepCopy() *CBContainersCommonProbesSpec { + if in == nil { + return nil + } + out := new(CBContainersCommonProbesSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CBContainersEventsGatewaySpec) DeepCopyInto(out *CBContainersEventsGatewaySpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersEventsGatewaySpec. +func (in *CBContainersEventsGatewaySpec) DeepCopy() *CBContainersEventsGatewaySpec { + if in == nil { + return nil + } + out := new(CBContainersEventsGatewaySpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CBContainersFileProbesSpec) DeepCopyInto(out *CBContainersFileProbesSpec) { + *out = *in + out.CBContainersCommonProbesSpec = in.CBContainersCommonProbesSpec +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersFileProbesSpec. +func (in *CBContainersFileProbesSpec) DeepCopy() *CBContainersFileProbesSpec { + if in == nil { + return nil + } + out := new(CBContainersFileProbesSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CBContainersHTTPProbesSpec) DeepCopyInto(out *CBContainersHTTPProbesSpec) { + *out = *in + out.CBContainersCommonProbesSpec = in.CBContainersCommonProbesSpec +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersHTTPProbesSpec. +func (in *CBContainersHTTPProbesSpec) DeepCopy() *CBContainersHTTPProbesSpec { + if in == nil { + return nil + } + out := new(CBContainersHTTPProbesSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CBContainersHardening) DeepCopyInto(out *CBContainersHardening) { *out = *in @@ -225,84 +272,133 @@ func (in *CBContainersHardeningEnforcerSpec) DeepCopy() *CBContainersHardeningEn } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CBContainersHardeningEventsGatewaySpec) DeepCopyInto(out *CBContainersHardeningEventsGatewaySpec) { +func (in *CBContainersHardeningList) DeepCopyInto(out *CBContainersHardeningList) { *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CBContainersHardening, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersHardeningEventsGatewaySpec. -func (in *CBContainersHardeningEventsGatewaySpec) DeepCopy() *CBContainersHardeningEventsGatewaySpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersHardeningList. +func (in *CBContainersHardeningList) DeepCopy() *CBContainersHardeningList { if in == nil { return nil } - out := new(CBContainersHardeningEventsGatewaySpec) + out := new(CBContainersHardeningList) in.DeepCopyInto(out) return out } +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CBContainersHardeningList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CBContainersHardeningImageSpec) DeepCopyInto(out *CBContainersHardeningImageSpec) { +func (in *CBContainersHardeningSpec) DeepCopyInto(out *CBContainersHardeningSpec) { *out = *in + out.EventsGatewaySpec = in.EventsGatewaySpec + in.EnforcerSpec.DeepCopyInto(&out.EnforcerSpec) + in.StateReporterSpec.DeepCopyInto(&out.StateReporterSpec) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersHardeningImageSpec. -func (in *CBContainersHardeningImageSpec) DeepCopy() *CBContainersHardeningImageSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersHardeningSpec. +func (in *CBContainersHardeningSpec) DeepCopy() *CBContainersHardeningSpec { if in == nil { return nil } - out := new(CBContainersHardeningImageSpec) + out := new(CBContainersHardeningSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CBContainersHardeningList) DeepCopyInto(out *CBContainersHardeningList) { +func (in *CBContainersHardeningStateReporterSpec) DeepCopyInto(out *CBContainersHardeningStateReporterSpec) { *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]CBContainersHardening, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val } } + if in.DeploymentAnnotations != nil { + in, out := &in.DeploymentAnnotations, &out.DeploymentAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.PodTemplateAnnotations != nil { + in, out := &in.PodTemplateAnnotations, &out.PodTemplateAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + out.Image = in.Image + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + in.Resources.DeepCopyInto(&out.Resources) + out.Probes = in.Probes } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersHardeningList. -func (in *CBContainersHardeningList) DeepCopy() *CBContainersHardeningList { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersHardeningStateReporterSpec. +func (in *CBContainersHardeningStateReporterSpec) DeepCopy() *CBContainersHardeningStateReporterSpec { if in == nil { return nil } - out := new(CBContainersHardeningList) + out := new(CBContainersHardeningStateReporterSpec) in.DeepCopyInto(out) return out } -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *CBContainersHardeningList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CBContainersHardeningStatus) DeepCopyInto(out *CBContainersHardeningStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersHardeningStatus. +func (in *CBContainersHardeningStatus) DeepCopy() *CBContainersHardeningStatus { + if in == nil { + return nil } - return nil + out := new(CBContainersHardeningStatus) + in.DeepCopyInto(out) + return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CBContainersHardeningProbesSpec) DeepCopyInto(out *CBContainersHardeningProbesSpec) { +func (in *CBContainersImageSpec) DeepCopyInto(out *CBContainersImageSpec) { *out = *in } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersHardeningProbesSpec. -func (in *CBContainersHardeningProbesSpec) DeepCopy() *CBContainersHardeningProbesSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersImageSpec. +func (in *CBContainersImageSpec) DeepCopy() *CBContainersImageSpec { if in == nil { return nil } - out := new(CBContainersHardeningProbesSpec) + out := new(CBContainersImageSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CBContainersHardeningPrometheusSpec) DeepCopyInto(out *CBContainersHardeningPrometheusSpec) { +func (in *CBContainersPrometheusSpec) DeepCopyInto(out *CBContainersPrometheusSpec) { *out = *in if in.Enabled != nil { in, out := &in.Enabled, &out.Enabled @@ -311,37 +407,79 @@ func (in *CBContainersHardeningPrometheusSpec) DeepCopyInto(out *CBContainersHar } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersHardeningPrometheusSpec. -func (in *CBContainersHardeningPrometheusSpec) DeepCopy() *CBContainersHardeningPrometheusSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersPrometheusSpec. +func (in *CBContainersPrometheusSpec) DeepCopy() *CBContainersPrometheusSpec { if in == nil { return nil } - out := new(CBContainersHardeningPrometheusSpec) + out := new(CBContainersPrometheusSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CBContainersHardeningSpec) DeepCopyInto(out *CBContainersHardeningSpec) { +func (in *CBContainersRuntime) DeepCopyInto(out *CBContainersRuntime) { *out = *in - in.EnforcerSpec.DeepCopyInto(&out.EnforcerSpec) - in.StateReporterSpec.DeepCopyInto(&out.StateReporterSpec) - out.EventsGatewaySpec = in.EventsGatewaySpec + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + in.Spec.DeepCopyInto(&out.Spec) + out.Status = in.Status } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersHardeningSpec. -func (in *CBContainersHardeningSpec) DeepCopy() *CBContainersHardeningSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersRuntime. +func (in *CBContainersRuntime) DeepCopy() *CBContainersRuntime { if in == nil { return nil } - out := new(CBContainersHardeningSpec) + out := new(CBContainersRuntime) in.DeepCopyInto(out) return out } +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CBContainersRuntime) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CBContainersHardeningStateReporterSpec) DeepCopyInto(out *CBContainersHardeningStateReporterSpec) { +func (in *CBContainersRuntimeList) DeepCopyInto(out *CBContainersRuntimeList) { *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]CBContainersRuntime, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersRuntimeList. +func (in *CBContainersRuntimeList) DeepCopy() *CBContainersRuntimeList { + if in == nil { + return nil + } + out := new(CBContainersRuntimeList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *CBContainersRuntimeList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CBContainersRuntimeResolverSpec) DeepCopyInto(out *CBContainersRuntimeResolverSpec) { + *out = *in + out.EventsGatewaySpec = in.EventsGatewaySpec if in.Labels != nil { in, out := &in.Labels, &out.Labels *out = make(map[string]string, len(*in)) @@ -363,7 +501,58 @@ func (in *CBContainersHardeningStateReporterSpec) DeepCopyInto(out *CBContainers (*out)[key] = val } } + if in.ReplicasCount != nil { + in, out := &in.ReplicasCount, &out.ReplicasCount + *out = new(int32) + **out = **in + } + if in.Env != nil { + in, out := &in.Env, &out.Env + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } out.Image = in.Image + in.Resources.DeepCopyInto(&out.Resources) + out.Probes = in.Probes + in.Prometheus.DeepCopyInto(&out.Prometheus) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersRuntimeResolverSpec. +func (in *CBContainersRuntimeResolverSpec) DeepCopy() *CBContainersRuntimeResolverSpec { + if in == nil { + return nil + } + out := new(CBContainersRuntimeResolverSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CBContainersRuntimeSensorSpec) DeepCopyInto(out *CBContainersRuntimeSensorSpec) { + *out = *in + if in.Labels != nil { + in, out := &in.Labels, &out.Labels + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.DaemonSetAnnotations != nil { + in, out := &in.DaemonSetAnnotations, &out.DaemonSetAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } + if in.PodTemplateAnnotations != nil { + in, out := &in.PodTemplateAnnotations, &out.PodTemplateAnnotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.Env != nil { in, out := &in.Env, &out.Env *out = make(map[string]string, len(*in)) @@ -371,31 +560,55 @@ func (in *CBContainersHardeningStateReporterSpec) DeepCopyInto(out *CBContainers (*out)[key] = val } } + out.Image = in.Image in.Resources.DeepCopyInto(&out.Resources) out.Probes = in.Probes + in.Prometheus.DeepCopyInto(&out.Prometheus) + if in.VerbosityLevel != nil { + in, out := &in.VerbosityLevel, &out.VerbosityLevel + *out = new(int) + **out = **in + } } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersHardeningStateReporterSpec. -func (in *CBContainersHardeningStateReporterSpec) DeepCopy() *CBContainersHardeningStateReporterSpec { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersRuntimeSensorSpec. +func (in *CBContainersRuntimeSensorSpec) DeepCopy() *CBContainersRuntimeSensorSpec { if in == nil { return nil } - out := new(CBContainersHardeningStateReporterSpec) + out := new(CBContainersRuntimeSensorSpec) in.DeepCopyInto(out) return out } // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *CBContainersHardeningStatus) DeepCopyInto(out *CBContainersHardeningStatus) { +func (in *CBContainersRuntimeSpec) DeepCopyInto(out *CBContainersRuntimeSpec) { *out = *in + in.ResolverSpec.DeepCopyInto(&out.ResolverSpec) + in.SensorSpec.DeepCopyInto(&out.SensorSpec) } -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersHardeningStatus. -func (in *CBContainersHardeningStatus) DeepCopy() *CBContainersHardeningStatus { +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersRuntimeSpec. +func (in *CBContainersRuntimeSpec) DeepCopy() *CBContainersRuntimeSpec { if in == nil { return nil } - out := new(CBContainersHardeningStatus) + out := new(CBContainersRuntimeSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CBContainersRuntimeStatus) DeepCopyInto(out *CBContainersRuntimeStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CBContainersRuntimeStatus. +func (in *CBContainersRuntimeStatus) DeepCopy() *CBContainersRuntimeStatus { + if in == nil { + return nil + } + out := new(CBContainersRuntimeStatus) in.DeepCopyInto(out) return out } diff --git a/cbcontainers/monitor/default_features_status_provider.go b/cbcontainers/monitor/default_features_status_provider.go index 22073282..ef6b7299 100644 --- a/cbcontainers/monitor/default_features_status_provider.go +++ b/cbcontainers/monitor/default_features_status_provider.go @@ -3,6 +3,7 @@ package monitor import ( "context" "fmt" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -31,5 +32,14 @@ func (provider DefaultFeaturesStatusProvider) HardeningEnabled() (bool, error) { } func (provider DefaultFeaturesStatusProvider) RuntimeEnabled() (bool, error) { - return false, nil + cbContainersRuntimeList := &cbcontainersv1.CBContainersRuntimeList{} + if err := provider.client.List(context.Background(), cbContainersRuntimeList); err != nil { + return false, fmt.Errorf("couldn't find CBContainersRuntime k8s object: %v", err) + } + + if cbContainersRuntimeList.Items == nil { + return false, fmt.Errorf("couldn't find CBContainersRuntime k8s object") + } + + return len(cbContainersRuntimeList.Items) > 0, nil } diff --git a/cbcontainers/monitor/default_features_status_provider_test.go b/cbcontainers/monitor/default_features_status_provider_test.go index 2f07e11f..5c69fa87 100644 --- a/cbcontainers/monitor/default_features_status_provider_test.go +++ b/cbcontainers/monitor/default_features_status_provider_test.go @@ -3,12 +3,13 @@ package monitor_test import ( "context" "fmt" + "testing" + "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" "github.com/vmware/cbcontainers-operator/cbcontainers/monitor" "github.com/vmware/cbcontainers-operator/cbcontainers/test_utils/mocks" - "testing" ) type SetupAndAssertDefaultFeaturesProvider func(*mocks.MockClient, *monitor.DefaultFeaturesStatusProvider) @@ -41,7 +42,7 @@ func TestHardeningEnabled(t *testing.T) { t.Run("When Client.List returns 0 items, should return false", func(t *testing.T) { testFeatures(t, func(client *mocks.MockClient, provider *monitor.DefaultFeaturesStatusProvider) { client.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersHardeningList{}). - Do(func(ctx context.Context, list *cbcontainersv1.CBContainersHardeningList) { + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersHardeningList, _ ...interface{}) { list.Items = make([]cbcontainersv1.CBContainersHardening, 0) }). Return(nil) @@ -54,7 +55,7 @@ func TestHardeningEnabled(t *testing.T) { t.Run("When Client.List returns 1 items, should return true", func(t *testing.T) { testFeatures(t, func(client *mocks.MockClient, provider *monitor.DefaultFeaturesStatusProvider) { client.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersHardeningList{}). - Do(func(ctx context.Context, list *cbcontainersv1.CBContainersHardeningList) { + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersHardeningList, _ ...interface{}) { list.Items = make([]cbcontainersv1.CBContainersHardening, 1) }). Return(nil) @@ -66,11 +67,45 @@ func TestHardeningEnabled(t *testing.T) { } func TestRuntimeEnabled(t *testing.T) { - t.Run("Always return false", func(t *testing.T) { + t.Run("When Client.List returns error, should return error", func(t *testing.T) { + testFeatures(t, func(client *mocks.MockClient, provider *monitor.DefaultFeaturesStatusProvider) { + client.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersRuntimeList{}).Return(fmt.Errorf("")) + _, err := provider.RuntimeEnabled() + require.Error(t, err) + }) + }) + + t.Run("When Client.List returns no items, should return error", func(t *testing.T) { + testFeatures(t, func(client *mocks.MockClient, provider *monitor.DefaultFeaturesStatusProvider) { + client.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersRuntimeList{}).Return(nil) + _, err := provider.RuntimeEnabled() + require.Error(t, err) + }) + }) + + t.Run("When Client.List returns 0 items, should return false", func(t *testing.T) { testFeatures(t, func(client *mocks.MockClient, provider *monitor.DefaultFeaturesStatusProvider) { + client.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersRuntimeList{}). + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersRuntimeList, _ ...interface{}) { + list.Items = make([]cbcontainersv1.CBContainersRuntime, 0) + }). + Return(nil) enabled, err := provider.RuntimeEnabled() require.NoError(t, err) require.False(t, enabled) }) }) + + t.Run("When Client.List returns 1 items, should return true", func(t *testing.T) { + testFeatures(t, func(client *mocks.MockClient, provider *monitor.DefaultFeaturesStatusProvider) { + client.EXPECT().List(gomock.Any(), &cbcontainersv1.CBContainersRuntimeList{}). + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersRuntimeList, _ ...interface{}) { + list.Items = make([]cbcontainersv1.CBContainersRuntime, 1) + }). + Return(nil) + enabled, err := provider.RuntimeEnabled() + require.NoError(t, err) + require.True(t, enabled) + }) + }) } diff --git a/cbcontainers/state/cluster/cluster_state_applier_test.go b/cbcontainers/state/cluster/cluster_state_applier_test.go index 730a30c2..aaa55be4 100644 --- a/cbcontainers/state/cluster/cluster_state_applier_test.go +++ b/cbcontainers/state/cluster/cluster_state_applier_test.go @@ -2,6 +2,9 @@ package cluster_test import ( "context" + "reflect" + "testing" + logrTesting "github.com/go-logr/logr/testing" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" @@ -16,9 +19,7 @@ import ( schedulingV1 "k8s.io/api/scheduling/v1" schedulingV1alpha1 "k8s.io/api/scheduling/v1alpha1" schedulingV1beta1 "k8s.io/api/scheduling/v1beta1" - "reflect" "sigs.k8s.io/controller-runtime/pkg/client" - "testing" ) const ( @@ -59,7 +60,7 @@ func testClusterStateApplier(t *testing.T, setup ClusterStateApplierTestSetup, k Spec: cbcontainersv1.CBContainersClusterSpec{ Account: Account, ClusterName: Cluster, - ApiGatewaySpec: cbcontainersv1.CBContainersClusterApiGatewaySpec{ + ApiGatewaySpec: cbcontainersv1.CBContainersApiGatewaySpec{ Scheme: ApiGateWayScheme, Host: ApiGateWayHost, Port: ApiGateWayPort, diff --git a/cbcontainers/state/common/common.go b/cbcontainers/state/common/common.go deleted file mode 100644 index 2ed3e51e..00000000 --- a/cbcontainers/state/common/common.go +++ /dev/null @@ -1,39 +0,0 @@ -package common - -import coreV1 "k8s.io/api/core/v1" - -func GetCommonDataPlaneEnvVars(accessKeySecretName string) []coreV1.EnvVar { - return []coreV1.EnvVar{ - getEnvVarFromConfigmap("OCTARINE_ACCOUNT", DataPlaneConfigmapAccountKey), - getEnvVarFromConfigmap("OCTARINE_DOMAIN", DataPlaneConfigmapClusterKey), - getEnvVarFromSecret("OCTARINE_ACCESS_TOKEN", accessKeySecretName), - getEnvVarFromConfigmap("OCTARINE_API_SCHEME", DataPlaneConfigmapApiSchemeKey), - getEnvVarFromConfigmap("OCTARINE_API_HOST", DataPlaneConfigmapApiHostKey), - getEnvVarFromConfigmap("OCTARINE_API_PORT", DataPlaneConfigmapApiPortKey), - getEnvVarFromConfigmap("OCTARINE_API_ADAPTER_NAME", DataPlaneConfigmapApiAdapterKey), - } -} - -func getEnvVarFromSecret(envName, accessKeySecretName string) coreV1.EnvVar { - return coreV1.EnvVar{ - Name: envName, - ValueFrom: &coreV1.EnvVarSource{ - SecretKeyRef: &coreV1.SecretKeySelector{ - LocalObjectReference: coreV1.LocalObjectReference{Name: accessKeySecretName}, - Key: AccessTokenSecretKeyName, - }, - }, - } -} - -func getEnvVarFromConfigmap(envName, configKeyName string) coreV1.EnvVar { - return coreV1.EnvVar{ - Name: envName, - ValueFrom: &coreV1.EnvVarSource{ - ConfigMapKeyRef: &coreV1.ConfigMapKeySelector{ - LocalObjectReference: coreV1.LocalObjectReference{Name: DataPlaneConfigmapName}, - Key: configKeyName, - }, - }, - } -} diff --git a/cbcontainers/state/common/common_test.go b/cbcontainers/state/common/common_test.go new file mode 100644 index 00000000..bd74a0d5 --- /dev/null +++ b/cbcontainers/state/common/common_test.go @@ -0,0 +1,444 @@ +package common + +import ( + "reflect" + "strconv" + "testing" + + "github.com/stretchr/testify/require" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + coreV1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +const ( + accessTokenSecretName = "access-token-secret" + + testName1 = "test_1" + testName2 = "test_2" + testName3 = "test_2" + testValue1 = "value_1" + testValue2 = "value_2" + testValue3 = "value_3" + + eventsGatewayHost = "test.com" + eventsGatewayPort = 443 +) + +func compareEnvVars(t *testing.T, expected map[string]coreV1.EnvVar, actual []coreV1.EnvVar) { + for _, envVar := range actual { + expectedEnvVar, ok := expected[envVar.Name] + require.True(t, ok) + require.True(t, reflect.DeepEqual(expectedEnvVar, envVar)) + } +} + +func TestWithDataPlaneCommonConfig(t *testing.T) { + expected := map[string]coreV1.EnvVar{ + accessTokenVarName: { + Name: accessTokenVarName, + ValueFrom: &coreV1.EnvVarSource{ + SecretKeyRef: &coreV1.SecretKeySelector{ + LocalObjectReference: coreV1.LocalObjectReference{Name: accessTokenSecretName}, + Key: AccessTokenSecretKeyName, + }, + }, + }, + apiSchemeVarName: { + Name: apiSchemeVarName, + ValueFrom: &coreV1.EnvVarSource{ + ConfigMapKeyRef: &coreV1.ConfigMapKeySelector{ + LocalObjectReference: coreV1.LocalObjectReference{Name: DataPlaneConfigmapName}, + Key: DataPlaneConfigmapApiSchemeKey, + }, + }, + }, + apiHostVarName: { + Name: apiHostVarName, + ValueFrom: &coreV1.EnvVarSource{ + ConfigMapKeyRef: &coreV1.ConfigMapKeySelector{ + LocalObjectReference: coreV1.LocalObjectReference{Name: DataPlaneConfigmapName}, + Key: DataPlaneConfigmapApiHostKey, + }, + }, + }, + apiPortVarName: { + Name: apiPortVarName, + ValueFrom: &coreV1.EnvVarSource{ + ConfigMapKeyRef: &coreV1.ConfigMapKeySelector{ + LocalObjectReference: coreV1.LocalObjectReference{Name: DataPlaneConfigmapName}, + Key: DataPlaneConfigmapApiPortKey, + }, + }, + }, + accountVarName: { + Name: accountVarName, + ValueFrom: &coreV1.EnvVarSource{ + ConfigMapKeyRef: &coreV1.ConfigMapKeySelector{ + LocalObjectReference: coreV1.LocalObjectReference{Name: DataPlaneConfigmapName}, + Key: DataPlaneConfigmapAccountKey, + }, + }, + }, + clusterVarName: { + Name: clusterVarName, + ValueFrom: &coreV1.EnvVarSource{ + ConfigMapKeyRef: &coreV1.ConfigMapKeySelector{ + LocalObjectReference: coreV1.LocalObjectReference{Name: DataPlaneConfigmapName}, + Key: DataPlaneConfigmapClusterKey, + }, + }, + }, + apiAdapterVarName: { + Name: apiAdapterVarName, + ValueFrom: &coreV1.EnvVarSource{ + ConfigMapKeyRef: &coreV1.ConfigMapKeySelector{ + LocalObjectReference: coreV1.LocalObjectReference{Name: DataPlaneConfigmapName}, + Key: DataPlaneConfigmapApiAdapterKey, + }, + }, + }, + } + actual := NewEnvVarBuilder(). + WithCommonDataPlane(accessTokenSecretName). + Build() + + compareEnvVars(t, expected, actual) +} + +func TestWithCustomConfig(t *testing.T) { + expected := map[string]coreV1.EnvVar{ + testName1: { + Name: testName1, + Value: testValue1, + }, + testName2: { + Name: testName2, + Value: testValue2, + }, + } + + envSlice := make([]coreV1.EnvVar, 0, len(expected)) + for _, envVar := range expected { + envSlice = append(envSlice, envVar) + } + + actual := NewEnvVarBuilder(). + WithCustom(envSlice...). + Build() + + compareEnvVars(t, expected, actual) +} + +func TestWithEventsGateway(t *testing.T) { + eventsGatewaySpec := &cbcontainersv1.CBContainersEventsGatewaySpec{ + Host: eventsGatewayHost, + Port: eventsGatewayPort, + } + expected := map[string]coreV1.EnvVar{ + eventGatewayHostVarName: { + Name: eventGatewayHostVarName, + Value: eventsGatewaySpec.Host, + }, + eventGatewayPortVarName: { + Name: eventGatewayPortVarName, + Value: strconv.Itoa(eventsGatewaySpec.Port), + }, + } + + actual := NewEnvVarBuilder(). + WithEventsGateway(eventsGatewaySpec). + Build() + + compareEnvVars(t, expected, actual) +} + +func TestWithSpecNoOverlap(t *testing.T) { + envSpec := map[string]string{ + testName1: testValue1, + testName2: testValue2, + } + eventsGatewaySpec := &cbcontainersv1.CBContainersEventsGatewaySpec{ + Host: eventsGatewayHost, + Port: eventsGatewayPort, + } + expected := map[string]coreV1.EnvVar{ + eventGatewayHostVarName: { + Name: eventGatewayHostVarName, + Value: eventsGatewaySpec.Host, + }, + eventGatewayPortVarName: { + Name: eventGatewayPortVarName, + Value: strconv.Itoa(eventsGatewaySpec.Port), + }, + testName1: { + Name: testName1, + Value: testValue1, + }, + testName2: { + Name: testName2, + Value: testValue2, + }, + } + + actual := NewEnvVarBuilder(). + WithEventsGateway(eventsGatewaySpec). + WithSpec(envSpec). + Build() + + compareEnvVars(t, expected, actual) +} + +func TestWithSpecOverlapping(t *testing.T) { + eventsGatewaySpec := &cbcontainersv1.CBContainersEventsGatewaySpec{ + Host: eventsGatewayHost, + Port: eventsGatewayPort, + } + envSpec := map[string]string{ + eventGatewayPortVarName: strconv.Itoa(eventsGatewaySpec.Port + 1), + testName2: testValue2, + } + expected := map[string]coreV1.EnvVar{ + eventGatewayHostVarName: { + Name: eventGatewayHostVarName, + Value: eventsGatewaySpec.Host, + }, + eventGatewayPortVarName: { + Name: eventGatewayPortVarName, + Value: strconv.Itoa(eventsGatewaySpec.Port + 1), + }, + testName2: { + Name: testName2, + Value: testValue2, + }, + } + + actual := NewEnvVarBuilder(). + WithEventsGateway(eventsGatewaySpec). + WithSpec(envSpec). + Build() + + compareEnvVars(t, expected, actual) +} + +func TestMutationFalse(t *testing.T) { + expected := map[string]coreV1.EnvVar{ + testName1: { + Name: testName1, + Value: testValue1, + }, + testName2: { + Name: testName2, + Value: testValue2, + }, + } + expectedSlice := make([]coreV1.EnvVar, 0, len(expected)) + for _, envVar := range expected { + expectedSlice = append(expectedSlice, envVar) + } + container := &coreV1.Container{ + Env: expectedSlice, + } + builder := NewEnvVarBuilder(). + WithCustom(expectedSlice...) + + MutateEnvVars(container, builder) + compareEnvVars(t, expected, container.Env) +} + +func TestMutationTrueSameSize(t *testing.T) { + expected := map[string]coreV1.EnvVar{ + testName1: { + Name: testName1, + Value: testValue1, + }, + testName2: { + Name: testName2, + Value: testValue2, + }, + } + expectedSlice := make([]coreV1.EnvVar, 0, len(expected)) + for _, envVar := range expected { + expectedSlice = append(expectedSlice, envVar) + } + container := &coreV1.Container{ + Env: []coreV1.EnvVar{ + { + Name: testName1, + Value: testValue1, + }, + { + Name: testName3, + Value: testValue3, + }, + }, + } + builder := NewEnvVarBuilder(). + WithCustom(expectedSlice...) + + MutateEnvVars(container, builder) + compareEnvVars(t, expected, container.Env) +} + +func TestMutationTrueDifferentSize(t *testing.T) { + expected := map[string]coreV1.EnvVar{ + testName1: { + Name: testName1, + Value: testValue1, + }, + testName2: { + Name: testName2, + Value: testValue2, + }, + } + expectedSlice := make([]coreV1.EnvVar, 0, len(expected)) + for _, envVar := range expected { + expectedSlice = append(expectedSlice, envVar) + } + container := &coreV1.Container{ + Env: []coreV1.EnvVar{ + { + Name: testName3, + Value: testValue3, + }, + }, + } + builder := NewEnvVarBuilder(). + WithCustom(expectedSlice...) + + MutateEnvVars(container, builder) + compareEnvVars(t, expected, container.Env) +} + +func TestMutateImageWithTag(t *testing.T) { + expectedImage := "cbartifactory/test:1.0.0" + expectedPullPolicy := coreV1.PullPolicy("IfNotPresent") + imageSpec := cbcontainersv1.CBContainersImageSpec{ + Repository: "cbartifactory/test", + Tag: "1.0.0", + PullPolicy: expectedPullPolicy, + } + container := &coreV1.Container{} + MutateImage(container, imageSpec, "3.0.0") + require.Equal(t, expectedImage, container.Image) + require.Equal(t, expectedPullPolicy, container.ImagePullPolicy) +} + +func TestMutateImageWithoutTag(t *testing.T) { + expectedImage := "cbartifactory/test:3.0.0" + expectedPullPolicy := coreV1.PullPolicy("IfNotPresent") + imageSpec := cbcontainersv1.CBContainersImageSpec{ + Repository: "cbartifactory/test", + Tag: "", + PullPolicy: expectedPullPolicy, + } + container := &coreV1.Container{} + MutateImage(container, imageSpec, "3.0.0") + require.Equal(t, expectedImage, container.Image) + require.Equal(t, expectedPullPolicy, container.ImagePullPolicy) +} + +const ( + expectedInitialDelay = 1 + expectedTimeout = 2 + expectedPeriod = 3 + expectedSuccessThreshold = 4 + expectedFailureThreshold = 5 + expectedPort = 8181 + expectedReadinessPath = "/ready" + expectedLivenessPath = "/alive" +) + +func TestMutateContainerHTTPProbes(t *testing.T) { + httpProbesSpec := cbcontainersv1.CBContainersHTTPProbesSpec{ + CBContainersCommonProbesSpec: cbcontainersv1.CBContainersCommonProbesSpec{ + InitialDelaySeconds: expectedInitialDelay, + TimeoutSeconds: expectedTimeout, + PeriodSeconds: expectedPeriod, + SuccessThreshold: expectedSuccessThreshold, + FailureThreshold: expectedFailureThreshold, + }, + ReadinessPath: expectedReadinessPath, + LivenessPath: expectedLivenessPath, + Port: expectedPort, + Scheme: coreV1.URISchemeHTTP, + } + + expectedReadinessProbe := &coreV1.Probe{ + Handler: coreV1.Handler{ + HTTPGet: &coreV1.HTTPGetAction{ + Path: expectedReadinessPath, + Port: intstr.FromInt(expectedPort), + Scheme: coreV1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: expectedInitialDelay, + TimeoutSeconds: expectedTimeout, + PeriodSeconds: expectedPeriod, + SuccessThreshold: expectedSuccessThreshold, + FailureThreshold: expectedFailureThreshold, + } + expectedLivenessProbe := &coreV1.Probe{ + Handler: coreV1.Handler{ + HTTPGet: &coreV1.HTTPGetAction{ + Path: expectedLivenessPath, + Port: intstr.FromInt(expectedPort), + Scheme: coreV1.URISchemeHTTP, + }, + }, + InitialDelaySeconds: expectedInitialDelay, + TimeoutSeconds: expectedTimeout, + PeriodSeconds: expectedPeriod, + SuccessThreshold: expectedSuccessThreshold, + FailureThreshold: expectedFailureThreshold, + } + + container := &coreV1.Container{} + MutateContainerHTTPProbes(container, httpProbesSpec) + require.True(t, reflect.DeepEqual(expectedLivenessProbe, container.LivenessProbe)) + require.True(t, reflect.DeepEqual(expectedReadinessProbe, container.ReadinessProbe)) +} + +func TestMutateContainerFileProbes(t *testing.T) { + fileProbesSpec := cbcontainersv1.CBContainersFileProbesSpec{ + CBContainersCommonProbesSpec: cbcontainersv1.CBContainersCommonProbesSpec{ + InitialDelaySeconds: expectedInitialDelay, + TimeoutSeconds: expectedTimeout, + PeriodSeconds: expectedPeriod, + SuccessThreshold: expectedSuccessThreshold, + FailureThreshold: expectedFailureThreshold, + }, + ReadinessPath: expectedReadinessPath, + LivenessPath: expectedLivenessPath, + } + + expectedReadinessProbe := &coreV1.Probe{ + Handler: coreV1.Handler{ + Exec: &coreV1.ExecAction{ + Command: []string{"cat", expectedReadinessPath}, + }, + }, + InitialDelaySeconds: expectedInitialDelay, + TimeoutSeconds: expectedTimeout, + PeriodSeconds: expectedPeriod, + SuccessThreshold: expectedSuccessThreshold, + FailureThreshold: expectedFailureThreshold, + } + expectedLivenessProbe := &coreV1.Probe{ + Handler: coreV1.Handler{ + Exec: &coreV1.ExecAction{ + Command: []string{"cat", expectedLivenessPath}, + }, + }, + InitialDelaySeconds: expectedInitialDelay, + TimeoutSeconds: expectedTimeout, + PeriodSeconds: expectedPeriod, + SuccessThreshold: expectedSuccessThreshold, + FailureThreshold: expectedFailureThreshold, + } + + container := &coreV1.Container{} + MutateContainerFileProbes(container, fileProbesSpec) + require.True(t, reflect.DeepEqual(expectedLivenessProbe, container.LivenessProbe)) + require.True(t, reflect.DeepEqual(expectedReadinessProbe, container.ReadinessProbe)) +} diff --git a/cbcontainers/state/common/env_var.go b/cbcontainers/state/common/env_var.go new file mode 100644 index 00000000..cf85bb81 --- /dev/null +++ b/cbcontainers/state/common/env_var.go @@ -0,0 +1,125 @@ +package common + +import ( + "reflect" + "strconv" + + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + coreV1 "k8s.io/api/core/v1" +) + +const ( + eventGatewayHostVarName = "OCTARINE_MESSAGEPROXY_HOST" + eventGatewayPortVarName = "OCTARINE_MESSAGEPROXY_PORT" + accountVarName = "OCTARINE_ACCOUNT" + clusterVarName = "OCTARINE_DOMAIN" + accessTokenVarName = "OCTARINE_ACCESS_TOKEN" + apiSchemeVarName = "OCTARINE_API_SCHEME" + apiHostVarName = "OCTARINE_API_HOST" + apiPortVarName = "OCTARINE_API_PORT" + apiAdapterVarName = "OCTARINE_API_ADAPTER_NAME" +) + +type EnvVarBuilder struct { + envVars map[string]coreV1.EnvVar +} + +func NewEnvVarBuilder() *EnvVarBuilder { + return &EnvVarBuilder{envVars: make(map[string]coreV1.EnvVar, 0)} +} + +// WithSpec Must be the last builder - to override all the predefined env vars +func (b *EnvVarBuilder) WithSpec(desiredEnvsValues map[string]string) *EnvVarBuilder { + for desiredEnvVarName, desiredEnvVarValue := range desiredEnvsValues { + b.envVars[desiredEnvVarName] = coreV1.EnvVar{Name: desiredEnvVarName, Value: desiredEnvVarValue} + } + + return b +} + +func (b *EnvVarBuilder) WithEventsGateway(eventsGatewaySpec *cbcontainersv1.CBContainersEventsGatewaySpec) *EnvVarBuilder { + b.envVars[eventGatewayHostVarName] = coreV1.EnvVar{Name: eventGatewayHostVarName, Value: eventsGatewaySpec.Host} + b.envVars[eventGatewayPortVarName] = coreV1.EnvVar{Name: eventGatewayPortVarName, Value: strconv.Itoa(eventsGatewaySpec.Port)} + + return b +} + +func (b *EnvVarBuilder) WithCustom(customEnvsToAdd ...coreV1.EnvVar) *EnvVarBuilder { + for _, customEnvVar := range customEnvsToAdd { + b.envVars[customEnvVar.Name] = customEnvVar + } + + return b +} + +func (b *EnvVarBuilder) WithEnvVarFromSecret(envName, accessKeySecretName string) *EnvVarBuilder { + envVar := coreV1.EnvVar{ + Name: envName, + ValueFrom: &coreV1.EnvVarSource{ + SecretKeyRef: &coreV1.SecretKeySelector{ + LocalObjectReference: coreV1.LocalObjectReference{Name: accessKeySecretName}, + Key: AccessTokenSecretKeyName, + }, + }, + } + b.envVars[envName] = envVar + + return b +} + +func (b *EnvVarBuilder) WithEnvVarFromConfigmap(envName, configKeyName string) *EnvVarBuilder { + envVar := coreV1.EnvVar{ + Name: envName, + ValueFrom: &coreV1.EnvVarSource{ + ConfigMapKeyRef: &coreV1.ConfigMapKeySelector{ + LocalObjectReference: coreV1.LocalObjectReference{Name: DataPlaneConfigmapName}, + Key: configKeyName, + }, + }, + } + b.envVars[envName] = envVar + + return b +} + +func (b *EnvVarBuilder) WithCommonDataPlane(accessKeySecretName string) *EnvVarBuilder { + return b.WithEnvVarFromSecret(accessTokenVarName, accessKeySecretName). + WithEnvVarFromConfigmap(accountVarName, DataPlaneConfigmapAccountKey). + WithEnvVarFromConfigmap(clusterVarName, DataPlaneConfigmapClusterKey). + WithEnvVarFromConfigmap(apiSchemeVarName, DataPlaneConfigmapApiSchemeKey). + WithEnvVarFromConfigmap(apiHostVarName, DataPlaneConfigmapApiHostKey). + WithEnvVarFromConfigmap(apiPortVarName, DataPlaneConfigmapApiPortKey). + WithEnvVarFromConfigmap(apiAdapterVarName, DataPlaneConfigmapApiAdapterKey) +} + +func (b *EnvVarBuilder) IsEqual(actualEnv []coreV1.EnvVar) bool { + if len(actualEnv) != len(b.envVars) { + return false + } + + for _, actualEnvVar := range actualEnv { + desiredEnvVar, ok := b.envVars[actualEnvVar.Name] + if !ok || !reflect.DeepEqual(actualEnvVar, desiredEnvVar) { + return false + } + } + + return true +} + +func (b *EnvVarBuilder) Build() []coreV1.EnvVar { + envVarsToReturn := make([]coreV1.EnvVar, 0, len(b.envVars)) + for _, desiredEnvVar := range b.envVars { + envVarsToReturn = append(envVarsToReturn, desiredEnvVar) + } + + return envVarsToReturn +} + +func MutateEnvVars(container *coreV1.Container, builder *EnvVarBuilder) { + if builder.IsEqual(container.Env) { + return + } + + container.Env = builder.Build() +} diff --git a/cbcontainers/state/common/image.go b/cbcontainers/state/common/image.go new file mode 100644 index 00000000..8909f53e --- /dev/null +++ b/cbcontainers/state/common/image.go @@ -0,0 +1,19 @@ +package common + +import ( + "fmt" + + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + coreV1 "k8s.io/api/core/v1" +) + +func MutateImage(container *coreV1.Container, desiredImage cbcontainersv1.CBContainersImageSpec, desiredVersion string) { + desiredTag := desiredImage.Tag + if desiredTag == "" { + desiredTag = desiredVersion + } + desiredFullImage := fmt.Sprintf("%s:%s", desiredImage.Repository, desiredTag) + + container.Image = desiredFullImage + container.ImagePullPolicy = desiredImage.PullPolicy +} diff --git a/cbcontainers/state/common/probes.go b/cbcontainers/state/common/probes.go new file mode 100644 index 00000000..d483031d --- /dev/null +++ b/cbcontainers/state/common/probes.go @@ -0,0 +1,63 @@ +package common + +import ( + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + coreV1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/util/intstr" +) + +func MutateContainerHTTPProbes(container *coreV1.Container, desiredProbes cbcontainersv1.CBContainersHTTPProbesSpec) { + mutateContainerCommonProbes(container) + + mutateHTTPProbe(container.ReadinessProbe, desiredProbes.ReadinessPath, desiredProbes) + mutateHTTPProbe(container.LivenessProbe, desiredProbes.LivenessPath, desiredProbes) +} + +func MutateContainerFileProbes(container *coreV1.Container, desiredProbes cbcontainersv1.CBContainersFileProbesSpec) { + mutateContainerCommonProbes(container) + + mutateFileProbe(container.ReadinessProbe, desiredProbes.ReadinessPath, desiredProbes) + mutateFileProbe(container.LivenessProbe, desiredProbes.LivenessPath, desiredProbes) +} + +func mutateContainerCommonProbes(container *coreV1.Container) { + if container.ReadinessProbe == nil { + container.ReadinessProbe = &coreV1.Probe{} + } + + if container.LivenessProbe == nil { + container.LivenessProbe = &coreV1.Probe{} + } +} + +func mutateHTTPProbe(probe *coreV1.Probe, desiredPath string, desiredProbes cbcontainersv1.CBContainersHTTPProbesSpec) { + if probe.Handler.HTTPGet == nil { + probe.Handler = coreV1.Handler{ + HTTPGet: &coreV1.HTTPGetAction{}, + } + } + + probe.HTTPGet.Path = desiredPath + probe.HTTPGet.Port = intstr.FromInt(desiredProbes.Port) + probe.HTTPGet.Scheme = desiredProbes.Scheme + mutateCommonProbe(probe, desiredProbes.CBContainersCommonProbesSpec) +} + +func mutateFileProbe(probe *coreV1.Probe, desiredPath string, desiredProbes cbcontainersv1.CBContainersFileProbesSpec) { + if probe.Handler.Exec == nil { + probe.Handler = coreV1.Handler{ + Exec: &coreV1.ExecAction{}, + } + } + + probe.Handler.Exec.Command = []string{"cat", desiredPath} + mutateCommonProbe(probe, desiredProbes.CBContainersCommonProbesSpec) +} + +func mutateCommonProbe(probe *coreV1.Probe, desiredProbes cbcontainersv1.CBContainersCommonProbesSpec) { + probe.InitialDelaySeconds = desiredProbes.InitialDelaySeconds + probe.TimeoutSeconds = desiredProbes.TimeoutSeconds + probe.PeriodSeconds = desiredProbes.PeriodSeconds + probe.SuccessThreshold = desiredProbes.SuccessThreshold + probe.FailureThreshold = desiredProbes.FailureThreshold +} diff --git a/cbcontainers/state/hardening/hardening_child_resource.go b/cbcontainers/state/hardening/hardening_child_resource.go index c29424ef..5091f573 100644 --- a/cbcontainers/state/hardening/hardening_child_resource.go +++ b/cbcontainers/state/hardening/hardening_child_resource.go @@ -2,12 +2,14 @@ package hardening import ( "context" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment" applymentOptions "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment/options" stateTypes "github.com/vmware/cbcontainers-operator/cbcontainers/state/types" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" ) type HardeningChildK8sObject interface { diff --git a/cbcontainers/state/hardening/hardening_state_applier.go b/cbcontainers/state/hardening/hardening_state_applier.go index 78394412..73a1a4c9 100644 --- a/cbcontainers/state/hardening/hardening_state_applier.go +++ b/cbcontainers/state/hardening/hardening_state_applier.go @@ -3,6 +3,7 @@ package hardening import ( "context" "fmt" + "github.com/go-logr/logr" cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" "github.com/vmware/cbcontainers-operator/cbcontainers/models" diff --git a/cbcontainers/state/hardening/hardening_state_applier_test.go b/cbcontainers/state/hardening/hardening_state_applier_test.go index 2e884066..4d80cd5a 100644 --- a/cbcontainers/state/hardening/hardening_state_applier_test.go +++ b/cbcontainers/state/hardening/hardening_state_applier_test.go @@ -2,6 +2,9 @@ package hardening_test import ( "context" + "reflect" + "testing" + logrTesting "github.com/go-logr/logr/testing" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" @@ -16,9 +19,7 @@ import ( admissionsV1 "k8s.io/api/admissionregistration/v1beta1" appsV1 "k8s.io/api/apps/v1" coreV1 "k8s.io/api/core/v1" - "reflect" "sigs.k8s.io/controller-runtime/pkg/client" - "testing" ) const ( @@ -86,7 +87,7 @@ func testHardeningStateApplier(t *testing.T, setup HardeningStateApplierTestSetu cbContainersHardening := &cbcontainersv1.CBContainersHardening{ Spec: cbcontainersv1.CBContainersHardeningSpec{ Version: Version, - EventsGatewaySpec: cbcontainersv1.CBContainersHardeningEventsGatewaySpec{ + EventsGatewaySpec: cbcontainersv1.CBContainersEventsGatewaySpec{ Host: EventsGateWayHost, }, }, diff --git a/cbcontainers/state/hardening/objects/enforcer_deployment.go b/cbcontainers/state/hardening/objects/enforcer_deployment.go index 0981aa25..ea084cd0 100644 --- a/cbcontainers/state/hardening/objects/enforcer_deployment.go +++ b/cbcontainers/state/hardening/objects/enforcer_deployment.go @@ -2,6 +2,7 @@ package objects import ( "fmt" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment" commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" @@ -50,7 +51,7 @@ func (obj *EnforcerDeploymentK8sObject) HardeningChildNamespacedName(_ *cbcontai } func (obj *EnforcerDeploymentK8sObject) MutateHardeningChildK8sObject(k8sObject client.Object, cbContainersHardening *cbcontainersv1.CBContainersHardening) error { - enforcerSpec := cbContainersHardening.Spec.EnforcerSpec + enforcerSpec := &cbContainersHardening.Spec.EnforcerSpec deployment, ok := k8sObject.(*appsV1.Deployment) if !ok { @@ -76,12 +77,12 @@ func (obj *EnforcerDeploymentK8sObject) MutateHardeningChildK8sObject(k8sObject deployment.Spec.Template.Spec.ImagePullSecrets = []coreV1.LocalObjectReference{{Name: commonState.RegistrySecretName}} obj.mutateAnnotations(deployment, enforcerSpec) obj.mutateVolumes(&deployment.Spec.Template.Spec) - obj.mutateContainersList(&deployment.Spec.Template.Spec, &cbContainersHardening.Spec.EnforcerSpec, &cbContainersHardening.Spec.EventsGatewaySpec, cbContainersHardening.Spec.Version, cbContainersHardening.Spec.AccessTokenSecretName) + obj.mutateContainersList(&deployment.Spec.Template.Spec, enforcerSpec, &cbContainersHardening.Spec.EventsGatewaySpec, cbContainersHardening.Spec.Version, cbContainersHardening.Spec.AccessTokenSecretName) return nil } -func (obj *EnforcerDeploymentK8sObject) mutateAnnotations(deployment *appsV1.Deployment, enforcerSpec cbcontainersv1.CBContainersHardeningEnforcerSpec) { +func (obj *EnforcerDeploymentK8sObject) mutateAnnotations(deployment *appsV1.Deployment, enforcerSpec *cbcontainersv1.CBContainersHardeningEnforcerSpec) { if deployment.ObjectMeta.Annotations == nil { deployment.ObjectMeta.Annotations = make(map[string]string) } @@ -116,7 +117,7 @@ func (obj *EnforcerDeploymentK8sObject) mutateVolumes(templatePodSpec *coreV1.Po templatePodSpec.Volumes[0].Secret.Optional = &DesiredTlsSecretVolumeOptionalValue } -func (obj *EnforcerDeploymentK8sObject) mutateContainersList(templatePodSpec *coreV1.PodSpec, enforcerSpec *cbcontainersv1.CBContainersHardeningEnforcerSpec, eventsGatewaySpec *cbcontainersv1.CBContainersHardeningEventsGatewaySpec, version, accessTokenSecretName string) { +func (obj *EnforcerDeploymentK8sObject) mutateContainersList(templatePodSpec *coreV1.PodSpec, enforcerSpec *cbcontainersv1.CBContainersHardeningEnforcerSpec, eventsGatewaySpec *cbcontainersv1.CBContainersEventsGatewaySpec, version, accessTokenSecretName string) { if len(templatePodSpec.Containers) != 1 { container := coreV1.Container{} templatePodSpec.Containers = []coreV1.Container{container} @@ -125,24 +126,31 @@ func (obj *EnforcerDeploymentK8sObject) mutateContainersList(templatePodSpec *co obj.mutateContainer(&templatePodSpec.Containers[0], enforcerSpec, eventsGatewaySpec, version, accessTokenSecretName) } -func (obj *EnforcerDeploymentK8sObject) mutateContainer(container *coreV1.Container, enforcerSpec *cbcontainersv1.CBContainersHardeningEnforcerSpec, eventsGatewaySpec *cbcontainersv1.CBContainersHardeningEventsGatewaySpec, version, accessTokenSecretName string) { +func (obj *EnforcerDeploymentK8sObject) mutateContainer(container *coreV1.Container, enforcerSpec *cbcontainersv1.CBContainersHardeningEnforcerSpec, eventsGatewaySpec *cbcontainersv1.CBContainersEventsGatewaySpec, version, accessTokenSecretName string) { container.Name = EnforcerName container.Resources = enforcerSpec.Resources obj.mutateEnforcerEnvVars(container, enforcerSpec, accessTokenSecretName, eventsGatewaySpec) - mutateImage(container, enforcerSpec.Image, version) - mutateContainerProbes(container, enforcerSpec.Probes) + commonState.MutateImage(container, enforcerSpec.Image, version) + commonState.MutateContainerHTTPProbes(container, enforcerSpec.Probes) obj.mutateSecurityContext(container) obj.mutateContainerPorts(container) obj.mutateVolumesMounts(container) } -func (obj *EnforcerDeploymentK8sObject) mutateEnforcerEnvVars(container *coreV1.Container, enforcerSpec *cbcontainersv1.CBContainersHardeningEnforcerSpec, accessTokenSecretName string, eventsGatewaySpec *cbcontainersv1.CBContainersHardeningEventsGatewaySpec) { - mutateEnvVars(container, enforcerSpec.Env, accessTokenSecretName, eventsGatewaySpec, - coreV1.EnvVar{Name: "GUARDRAILS_ENFORCER_KEY_FILE_PATH", Value: fmt.Sprintf("%s/key", DesiredTlsSecretVolumeMountPath)}, - coreV1.EnvVar{Name: "GUARDRAILS_ENFORCER_CERT_FILE_PATH", Value: fmt.Sprintf("%s/signed_cert", DesiredTlsSecretVolumeMountPath)}, - coreV1.EnvVar{Name: "GUARDRAILS_ENFORCER_PROMETHEUS_PORT", Value: fmt.Sprintf("%d", enforcerSpec.Prometheus.Port)}, - coreV1.EnvVar{Name: "GIN_MODE", Value: "release"}, - ) +func (obj *EnforcerDeploymentK8sObject) mutateEnforcerEnvVars(container *coreV1.Container, enforcerSpec *cbcontainersv1.CBContainersHardeningEnforcerSpec, accessTokenSecretName string, eventsGatewaySpec *cbcontainersv1.CBContainersEventsGatewaySpec) { + customEnvs := []coreV1.EnvVar{ + {Name: "GUARDRAILS_ENFORCER_KEY_FILE_PATH", Value: fmt.Sprintf("%s/key", DesiredTlsSecretVolumeMountPath)}, + {Name: "GUARDRAILS_ENFORCER_CERT_FILE_PATH", Value: fmt.Sprintf("%s/signed_cert", DesiredTlsSecretVolumeMountPath)}, + {Name: "GUARDRAILS_ENFORCER_PROMETHEUS_PORT", Value: fmt.Sprintf("%d", enforcerSpec.Prometheus.Port)}, + {Name: "GIN_MODE", Value: "release"}, + } + + envVarBuilder := commonState.NewEnvVarBuilder(). + WithCommonDataPlane(accessTokenSecretName). + WithEventsGateway(eventsGatewaySpec). + WithCustom(customEnvs...). + WithSpec(enforcerSpec.Env) + commonState.MutateEnvVars(container, envVarBuilder) } func (obj *EnforcerDeploymentK8sObject) mutateSecurityContext(container *coreV1.Container) { diff --git a/cbcontainers/state/hardening/objects/enforcer_service.go b/cbcontainers/state/hardening/objects/enforcer_service.go index 1e83df6a..193c225c 100644 --- a/cbcontainers/state/hardening/objects/enforcer_service.go +++ b/cbcontainers/state/hardening/objects/enforcer_service.go @@ -2,6 +2,7 @@ package objects import ( "fmt" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" coreV1 "k8s.io/api/core/v1" @@ -15,9 +16,7 @@ const ( DesiredServicePortValue = 443 ) -type EnforcerServiceK8sObject struct { - tlsSecretsValuesCreator TlsSecretsValuesCreator -} +type EnforcerServiceK8sObject struct{} func NewEnforcerServiceK8sObject() *EnforcerServiceK8sObject { return &EnforcerServiceK8sObject{} } diff --git a/cbcontainers/state/hardening/objects/enforcer_tls_secret.go b/cbcontainers/state/hardening/objects/enforcer_tls_secret.go index 3d08fb32..0de86aa6 100644 --- a/cbcontainers/state/hardening/objects/enforcer_tls_secret.go +++ b/cbcontainers/state/hardening/objects/enforcer_tls_secret.go @@ -2,6 +2,7 @@ package objects import ( "fmt" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" "github.com/vmware/cbcontainers-operator/cbcontainers/models" commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" @@ -36,7 +37,7 @@ func (obj *EnforcerTlsK8sObject) HardeningChildNamespacedName(_ *cbcontainersv1. return types.NamespacedName{Name: EnforcerTlsName, Namespace: commonState.DataPlaneNamespaceName} } -func (obj *EnforcerTlsK8sObject) MutateHardeningChildK8sObject(k8sObject client.Object, cbContainersHardening *cbcontainersv1.CBContainersHardening) error { +func (obj *EnforcerTlsK8sObject) MutateHardeningChildK8sObject(k8sObject client.Object, _ *cbcontainersv1.CBContainersHardening) error { secret, ok := k8sObject.(*coreV1.Secret) if !ok { return fmt.Errorf("expected Secret K8s object") diff --git a/cbcontainers/state/hardening/objects/objects_common.go b/cbcontainers/state/hardening/objects/objects_common.go deleted file mode 100644 index 333d44bd..00000000 --- a/cbcontainers/state/hardening/objects/objects_common.go +++ /dev/null @@ -1,105 +0,0 @@ -package objects - -import ( - "fmt" - cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" - commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" - coreV1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/util/intstr" - "reflect" - "strconv" -) - -func mutateEnvVars(container *coreV1.Container, desiredEnvsValues map[string]string, accessTokenSecretName string, eventsGatewaySpec *cbcontainersv1.CBContainersHardeningEventsGatewaySpec, customEnvsToAdd ...coreV1.EnvVar) { - desiredEnvVars := getDesiredEnvVars(desiredEnvsValues, accessTokenSecretName, eventsGatewaySpec, customEnvsToAdd) - - if !shouldChangeEnvVars(container, desiredEnvVars) { - return - } - - container.Env = make([]coreV1.EnvVar, 0, len(desiredEnvVars)) - for _, desiredEnvVar := range desiredEnvVars { - container.Env = append(container.Env, desiredEnvVar) - } -} - -func shouldChangeEnvVars(container *coreV1.Container, desiredEnvVars map[string]coreV1.EnvVar) bool { - if len(container.Env) != len(desiredEnvVars) { - return true - } - - for _, actualEnvVar := range container.Env { - desiredEnvVar, ok := desiredEnvVars[actualEnvVar.Name] - if !ok || !reflect.DeepEqual(actualEnvVar, desiredEnvVar) { - return true - } - } - - return false -} - -func getDesiredEnvVars(desiredEnvsValues map[string]string, accessTokenSecretName string, eventsGatewaySpec *cbcontainersv1.CBContainersHardeningEventsGatewaySpec, customEnvsToAdd []coreV1.EnvVar) map[string]coreV1.EnvVar { - desiredEnvVars := make(map[string]coreV1.EnvVar) - for desiredEnvVarName, desiredEnvVarValue := range desiredEnvsValues { - desiredEnvVars[desiredEnvVarName] = coreV1.EnvVar{Name: desiredEnvVarName, Value: desiredEnvVarValue} - } - envsToAdd := commonState.GetCommonDataPlaneEnvVars(accessTokenSecretName) - envsToAdd = append(envsToAdd, getEventsGateWayEnvVars(eventsGatewaySpec)...) - envsToAdd = append(envsToAdd, customEnvsToAdd...) - - for _, dataPlaneEnvVar := range envsToAdd { - if _, ok := desiredEnvVars[dataPlaneEnvVar.Name]; ok { - continue - } - desiredEnvVars[dataPlaneEnvVar.Name] = dataPlaneEnvVar - } - return desiredEnvVars -} - -func getEventsGateWayEnvVars(eventsGatewaySpec *cbcontainersv1.CBContainersHardeningEventsGatewaySpec) []coreV1.EnvVar { - return []coreV1.EnvVar{ - {Name: "OCTARINE_MESSAGEPROXY_HOST", Value: eventsGatewaySpec.Host}, - {Name: "OCTARINE_MESSAGEPROXY_PORT", Value: strconv.Itoa(eventsGatewaySpec.Port)}, - } -} - -func mutateImage(container *coreV1.Container, desiredImage cbcontainersv1.CBContainersHardeningImageSpec, desiredVersion string) { - desiredTag := desiredImage.Tag - if desiredTag == "" { - desiredTag = desiredVersion - } - desiredFullImage := fmt.Sprintf("%s:%s", desiredImage.Repository, desiredTag) - - container.Image = desiredFullImage - container.ImagePullPolicy = desiredImage.PullPolicy -} - -func mutateContainerProbes(container *coreV1.Container, desiredProbes cbcontainersv1.CBContainersHardeningProbesSpec) { - if container.ReadinessProbe == nil { - container.ReadinessProbe = &coreV1.Probe{} - } - - if container.LivenessProbe == nil { - container.LivenessProbe = &coreV1.Probe{} - } - - mutateProbe(container.ReadinessProbe, desiredProbes.ReadinessPath, desiredProbes) - mutateProbe(container.LivenessProbe, desiredProbes.LivenessPath, desiredProbes) -} - -func mutateProbe(probe *coreV1.Probe, desiredPath string, desiredProbes cbcontainersv1.CBContainersHardeningProbesSpec) { - if probe.Handler.HTTPGet == nil { - probe.Handler = coreV1.Handler{ - HTTPGet: &coreV1.HTTPGetAction{}, - } - } - - probe.HTTPGet.Path = desiredPath - probe.HTTPGet.Port = intstr.FromInt(desiredProbes.Port) - probe.HTTPGet.Scheme = desiredProbes.Scheme - probe.InitialDelaySeconds = desiredProbes.InitialDelaySeconds - probe.TimeoutSeconds = desiredProbes.TimeoutSeconds - probe.PeriodSeconds = desiredProbes.PeriodSeconds - probe.SuccessThreshold = desiredProbes.SuccessThreshold - probe.FailureThreshold = desiredProbes.FailureThreshold -} diff --git a/cbcontainers/state/hardening/objects/state_reporter_deployment.go b/cbcontainers/state/hardening/objects/state_reporter_deployment.go index d1cda8ea..5f83a9df 100644 --- a/cbcontainers/state/hardening/objects/state_reporter_deployment.go +++ b/cbcontainers/state/hardening/objects/state_reporter_deployment.go @@ -2,6 +2,7 @@ package objects import ( "fmt" + cbContainersV1 "github.com/vmware/cbcontainers-operator/api/v1" "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment" commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" @@ -78,7 +79,7 @@ func (obj *StateReporterDeploymentK8sObject) MutateHardeningChildK8sObject(k8sOb return nil } -func (obj *StateReporterDeploymentK8sObject) mutateContainersList(templatePodSpec *coreV1.PodSpec, stateReporterSpec *cbContainersV1.CBContainersHardeningStateReporterSpec, eventsGatewaySpec *cbContainersV1.CBContainersHardeningEventsGatewaySpec, version, accessTokenSecretName string) { +func (obj *StateReporterDeploymentK8sObject) mutateContainersList(templatePodSpec *coreV1.PodSpec, stateReporterSpec *cbContainersV1.CBContainersHardeningStateReporterSpec, eventsGatewaySpec *cbContainersV1.CBContainersEventsGatewaySpec, version, accessTokenSecretName string) { if len(templatePodSpec.Containers) != 1 { container := coreV1.Container{} templatePodSpec.Containers = []coreV1.Container{container} @@ -87,12 +88,18 @@ func (obj *StateReporterDeploymentK8sObject) mutateContainersList(templatePodSpe obj.mutateContainer(&templatePodSpec.Containers[0], stateReporterSpec, eventsGatewaySpec, version, accessTokenSecretName) } -func (obj *StateReporterDeploymentK8sObject) mutateContainer(container *coreV1.Container, stateReporterSpec *cbContainersV1.CBContainersHardeningStateReporterSpec, eventsGatewaySpec *cbContainersV1.CBContainersHardeningEventsGatewaySpec, version, accessTokenSecretName string) { +func (obj *StateReporterDeploymentK8sObject) mutateContainer(container *coreV1.Container, stateReporterSpec *cbContainersV1.CBContainersHardeningStateReporterSpec, eventsGatewaySpec *cbContainersV1.CBContainersEventsGatewaySpec, version, accessTokenSecretName string) { container.Name = StateReporterName container.Resources = stateReporterSpec.Resources - mutateEnvVars(container, stateReporterSpec.Env, accessTokenSecretName, eventsGatewaySpec) - mutateImage(container, stateReporterSpec.Image, version) - mutateContainerProbes(container, stateReporterSpec.Probes) + + envVarBuilder := commonState.NewEnvVarBuilder(). + WithCommonDataPlane(accessTokenSecretName). + WithEventsGateway(eventsGatewaySpec). + WithSpec(stateReporterSpec.Env) + commonState.MutateEnvVars(container, envVarBuilder) + + commonState.MutateImage(container, stateReporterSpec.Image, version) + commonState.MutateContainerHTTPProbes(container, stateReporterSpec.Probes) obj.mutateSecurityContext(container) } diff --git a/cbcontainers/state/runtime/mocks/generated.go b/cbcontainers/state/runtime/mocks/generated.go new file mode 100644 index 00000000..175343d3 --- /dev/null +++ b/cbcontainers/state/runtime/mocks/generated.go @@ -0,0 +1,3 @@ +package mocks + +//go:generate mockgen -destination mock_runtime_child_applier.go -package mocks github.com/vmware/cbcontainers-operator/cbcontainers/state/runtime RuntimeChildK8sObjectApplier diff --git a/cbcontainers/state/runtime/mocks/mock_runtime_child_applier.go b/cbcontainers/state/runtime/mocks/mock_runtime_child_applier.go new file mode 100644 index 00000000..4e021957 --- /dev/null +++ b/cbcontainers/state/runtime/mocks/mock_runtime_child_applier.go @@ -0,0 +1,75 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/vmware/cbcontainers-operator/cbcontainers/state/runtime (interfaces: RuntimeChildK8sObjectApplier) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/vmware/cbcontainers-operator/api/v1" + options "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment/options" + runtime "github.com/vmware/cbcontainers-operator/cbcontainers/state/runtime" + client "sigs.k8s.io/controller-runtime/pkg/client" +) + +// MockRuntimeChildK8sObjectApplier is a mock of RuntimeChildK8sObjectApplier interface. +type MockRuntimeChildK8sObjectApplier struct { + ctrl *gomock.Controller + recorder *MockRuntimeChildK8sObjectApplierMockRecorder +} + +// MockRuntimeChildK8sObjectApplierMockRecorder is the mock recorder for MockRuntimeChildK8sObjectApplier. +type MockRuntimeChildK8sObjectApplierMockRecorder struct { + mock *MockRuntimeChildK8sObjectApplier +} + +// NewMockRuntimeChildK8sObjectApplier creates a new mock instance. +func NewMockRuntimeChildK8sObjectApplier(ctrl *gomock.Controller) *MockRuntimeChildK8sObjectApplier { + mock := &MockRuntimeChildK8sObjectApplier{ctrl: ctrl} + mock.recorder = &MockRuntimeChildK8sObjectApplierMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRuntimeChildK8sObjectApplier) EXPECT() *MockRuntimeChildK8sObjectApplierMockRecorder { + return m.recorder +} + +// ApplyRuntimeChildK8sObject mocks base method. +func (m *MockRuntimeChildK8sObjectApplier) ApplyRuntimeChildK8sObject(arg0 context.Context, arg1 *v1.CBContainersRuntime, arg2 client.Client, arg3 runtime.RuntimeChildK8sObject, arg4 ...*options.ApplyOptions) (bool, client.Object, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1, arg2, arg3} + for _, a := range arg4 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "ApplyRuntimeChildK8sObject", varargs...) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(client.Object) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// ApplyRuntimeChildK8sObject indicates an expected call of ApplyRuntimeChildK8sObject. +func (mr *MockRuntimeChildK8sObjectApplierMockRecorder) ApplyRuntimeChildK8sObject(arg0, arg1, arg2, arg3 interface{}, arg4 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1, arg2, arg3}, arg4...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyRuntimeChildK8sObject", reflect.TypeOf((*MockRuntimeChildK8sObjectApplier)(nil).ApplyRuntimeChildK8sObject), varargs...) +} + +// DeleteK8sObjectIfExists mocks base method. +func (m *MockRuntimeChildK8sObjectApplier) DeleteK8sObjectIfExists(arg0 context.Context, arg1 *v1.CBContainersRuntime, arg2 client.Client, arg3 runtime.RuntimeChildK8sObject) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteK8sObjectIfExists", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// DeleteK8sObjectIfExists indicates an expected call of DeleteK8sObjectIfExists. +func (mr *MockRuntimeChildK8sObjectApplierMockRecorder) DeleteK8sObjectIfExists(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteK8sObjectIfExists", reflect.TypeOf((*MockRuntimeChildK8sObjectApplier)(nil).DeleteK8sObjectIfExists), arg0, arg1, arg2, arg3) +} diff --git a/cbcontainers/state/runtime/objects/resolver_deployment.go b/cbcontainers/state/runtime/objects/resolver_deployment.go new file mode 100644 index 00000000..9961cb10 --- /dev/null +++ b/cbcontainers/state/runtime/objects/resolver_deployment.go @@ -0,0 +1,186 @@ +package objects + +import ( + "fmt" + + cbContainersV1 "github.com/vmware/cbcontainers-operator/api/v1" + "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment" + commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" + appsV1 "k8s.io/api/apps/v1" + coreV1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + ResolverName = "cbcontainers-runtime-resolver" + resolverLabelKey = "app.kubernetes.io/name" + + desiredDeploymentGRPCPortName = "grpc" + desiredInitializationTimeoutMinutes = 3 +) + +var ( + resolverAllowPrivilegeEscalation = false + resolverReadOnlyRootFilesystem = true + resolverRunAsUser int64 = 0 + resolverCapabilitiesToDrop = []coreV1.Capability{"ALL"} + resolverCapabilitiesToAdd = []coreV1.Capability{"NET_BIND_SERVICE"} +) + +type ResolverDeploymentK8sObject struct{} + +func NewResolverDeploymentK8sObject() *ResolverDeploymentK8sObject { + return &ResolverDeploymentK8sObject{} +} + +func (obj *ResolverDeploymentK8sObject) EmptyK8sObject() client.Object { + return &appsV1.Deployment{} +} + +func (obj *ResolverDeploymentK8sObject) RuntimeChildNamespacedName(_ *cbContainersV1.CBContainersRuntime) types.NamespacedName { + return types.NamespacedName{Name: ResolverName, Namespace: commonState.DataPlaneNamespaceName} +} + +func (obj *ResolverDeploymentK8sObject) MutateRuntimeChildK8sObject(k8sObject client.Object, cbContainersRuntime *cbContainersV1.CBContainersRuntime) error { + resolverSpec := &cbContainersRuntime.Spec.ResolverSpec + deployment, ok := k8sObject.(*appsV1.Deployment) + if !ok { + return fmt.Errorf("expected Deployment K8s object") + } + + desiredLabels := resolverSpec.Labels + if desiredLabels == nil { + desiredLabels = make(map[string]string) + } + desiredLabels[resolverLabelKey] = ResolverName + + if deployment.Spec.Selector == nil { + deployment.Spec.Selector = &metav1.LabelSelector{} + } + + if deployment.ObjectMeta.Annotations == nil { + deployment.ObjectMeta.Annotations = make(map[string]string) + } + + if deployment.Spec.Template.ObjectMeta.Annotations == nil { + deployment.Spec.Template.ObjectMeta.Annotations = make(map[string]string) + } + + deployment.Spec.Replicas = resolverSpec.ReplicasCount + deployment.ObjectMeta.Labels = desiredLabels + deployment.Spec.Selector.MatchLabels = desiredLabels + deployment.Spec.Template.ObjectMeta.Labels = desiredLabels + deployment.Spec.Template.Spec.ServiceAccountName = commonState.DataPlaneServiceAccountName + deployment.Spec.Template.Spec.PriorityClassName = commonState.DataPlanePriorityClassName + deployment.Spec.Template.Spec.ImagePullSecrets = []coreV1.LocalObjectReference{{Name: commonState.RegistrySecretName}} + obj.mutateAnnotations(deployment, resolverSpec) + obj.mutateContainersList(&deployment.Spec.Template.Spec, + resolverSpec, + &cbContainersRuntime.Spec.ResolverSpec.EventsGatewaySpec, + cbContainersRuntime.Spec.Version, + cbContainersRuntime.Spec.AccessTokenSecretName, + cbContainersRuntime.Spec.InternalGrpcPort, + ) + + return nil +} + +func (obj *ResolverDeploymentK8sObject) mutateAnnotations(deployment *appsV1.Deployment, resolverSpec *cbContainersV1.CBContainersRuntimeResolverSpec) { + if deployment.ObjectMeta.Annotations == nil { + deployment.ObjectMeta.Annotations = make(map[string]string) + } + + applyment.EnforceMapContains(deployment.ObjectMeta.Annotations, resolverSpec.DeploymentAnnotations) + + if deployment.Spec.Template.ObjectMeta.Annotations == nil { + deployment.Spec.Template.ObjectMeta.Annotations = make(map[string]string) + } + + applyment.EnforceMapContains(deployment.Spec.Template.ObjectMeta.Annotations, map[string]string{ + "prometheus.io/scrape": fmt.Sprint(*resolverSpec.Prometheus.Enabled), + "prometheus.io/port": fmt.Sprint(resolverSpec.Prometheus.Port), + }) + applyment.EnforceMapContains(deployment.Spec.Template.ObjectMeta.Annotations, resolverSpec.PodTemplateAnnotations) +} + +func (obj *ResolverDeploymentK8sObject) mutateContainersList( + templatePodSpec *coreV1.PodSpec, + resolverSpec *cbContainersV1.CBContainersRuntimeResolverSpec, + eventsGatewaySpec *cbContainersV1.CBContainersEventsGatewaySpec, + version, + accessTokenSecretName string, + desiredGRPCPortValue int32) { + + if len(templatePodSpec.Containers) != 1 { + container := coreV1.Container{} + templatePodSpec.Containers = []coreV1.Container{container} + } + + obj.mutateContainer(&templatePodSpec.Containers[0], resolverSpec, eventsGatewaySpec, + version, accessTokenSecretName, desiredGRPCPortValue) +} + +func (obj *ResolverDeploymentK8sObject) mutateContainer( + container *coreV1.Container, + resolverSpec *cbContainersV1.CBContainersRuntimeResolverSpec, + eventsGatewaySpec *cbContainersV1.CBContainersEventsGatewaySpec, + version, + accessTokenSecretName string, + desiredGRPCPortValue int32) { + + container.Name = ResolverName + container.Resources = resolverSpec.Resources + commonState.MutateImage(container, resolverSpec.Image, version) + commonState.MutateContainerHTTPProbes(container, resolverSpec.Probes) + obj.mutateEnvVars(container, resolverSpec, eventsGatewaySpec, accessTokenSecretName, desiredGRPCPortValue) + obj.mutateContainerPorts(container, desiredGRPCPortValue) + obj.mutateSecurityContext(container) +} + +func (obj *ResolverDeploymentK8sObject) mutateContainerPorts(container *coreV1.Container, desiredGRPCPortValue int32) { + if container.Ports == nil || len(container.Ports) != 1 { + container.Ports = []coreV1.ContainerPort{{}} + } + + container.Ports[0].Name = desiredDeploymentGRPCPortName + container.Ports[0].ContainerPort = desiredGRPCPortValue +} + +func (obj *ResolverDeploymentK8sObject) mutateEnvVars( + container *coreV1.Container, + resolverSpec *cbContainersV1.CBContainersRuntimeResolverSpec, + eventsGatewaySpec *cbContainersV1.CBContainersEventsGatewaySpec, + accessTokenSecretName string, + desiredGRPCPortValue int32) { + + customEnvs := []coreV1.EnvVar{ + {Name: "RUNTIME_KUBERNETES_RESOLVER_GRPC_PORT", Value: fmt.Sprintf("%d", desiredGRPCPortValue)}, + {Name: "RUNTIME_KUBERNETES_RESOLVER_PROMETHEUS_PORT", Value: fmt.Sprintf("%d", resolverSpec.Prometheus.Port)}, + {Name: "RUNTIME_KUBERNETES_RESOLVER_PROBES_PORT", Value: fmt.Sprintf("%d", resolverSpec.Probes.Port)}, + {Name: "RUNTIME_KUBERNETES_RESOLVER_INITIALIZATION_TIMEOUT_MINUTES", Value: fmt.Sprintf("%d", desiredInitializationTimeoutMinutes)}, + {Name: "GIN_MODE", Value: "release"}, + } + + envVarBuilder := commonState.NewEnvVarBuilder(). + WithCommonDataPlane(accessTokenSecretName). + WithEventsGateway(eventsGatewaySpec). + WithCustom(customEnvs...). + WithSpec(resolverSpec.Env) + commonState.MutateEnvVars(container, envVarBuilder) +} + +func (obj *ResolverDeploymentK8sObject) mutateSecurityContext(container *coreV1.Container) { + if container.SecurityContext == nil { + container.SecurityContext = &coreV1.SecurityContext{} + } + + container.SecurityContext.AllowPrivilegeEscalation = &resolverAllowPrivilegeEscalation + container.SecurityContext.ReadOnlyRootFilesystem = &resolverReadOnlyRootFilesystem + container.SecurityContext.RunAsUser = &resolverRunAsUser + container.SecurityContext.Capabilities = &coreV1.Capabilities{ + Drop: resolverCapabilitiesToDrop, + Add: resolverCapabilitiesToAdd, + } +} diff --git a/cbcontainers/state/runtime/objects/resolver_service.go b/cbcontainers/state/runtime/objects/resolver_service.go new file mode 100644 index 00000000..66ac5717 --- /dev/null +++ b/cbcontainers/state/runtime/objects/resolver_service.go @@ -0,0 +1,56 @@ +package objects + +import ( + "fmt" + + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" + coreV1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + DesiredServiceGRPCPortName = "https" +) + +type ResolverServiceK8sObject struct{} + +func NewResolverServiceK8sObject() *ResolverServiceK8sObject { return &ResolverServiceK8sObject{} } + +func (obj *ResolverServiceK8sObject) EmptyK8sObject() client.Object { + return &coreV1.Service{} +} + +func (obj *ResolverServiceK8sObject) RuntimeChildNamespacedName(_ *cbcontainersv1.CBContainersRuntime) types.NamespacedName { + return types.NamespacedName{Name: ResolverName, Namespace: commonState.DataPlaneNamespaceName} +} + +func (obj *ResolverServiceK8sObject) MutateRuntimeChildK8sObject(k8sObject client.Object, cbContainersRuntime *cbcontainersv1.CBContainersRuntime) error { + service, ok := k8sObject.(*coreV1.Service) + if !ok { + return fmt.Errorf("expected Service K8s object") + } + + resolverSpec := cbContainersRuntime.Spec.ResolverSpec + + service.Labels = resolverSpec.Labels + service.Spec.Type = coreV1.ServiceTypeClusterIP + service.Spec.Selector = map[string]string{ + resolverLabelKey: ResolverName, + } + obj.mutatePorts(service, cbContainersRuntime.Spec.InternalGrpcPort) + + return nil +} + +func (obj *ResolverServiceK8sObject) mutatePorts(service *coreV1.Service, desiredGRPCPortValue int32) { + if service.Spec.Ports == nil || len(service.Spec.Ports) != 1 { + service.Spec.Ports = []coreV1.ServicePort{{}} + } + + service.Spec.Ports[0].Name = DesiredServiceGRPCPortName + service.Spec.Ports[0].TargetPort = intstr.FromString(desiredDeploymentGRPCPortName) + service.Spec.Ports[0].Port = desiredGRPCPortValue +} diff --git a/cbcontainers/state/runtime/objects/sensor_daemon_set.go b/cbcontainers/state/runtime/objects/sensor_daemon_set.go new file mode 100644 index 00000000..b4a3eb72 --- /dev/null +++ b/cbcontainers/state/runtime/objects/sensor_daemon_set.go @@ -0,0 +1,174 @@ +package objects + +import ( + "fmt" + + cbContainersV1 "github.com/vmware/cbcontainers-operator/api/v1" + "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment" + commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" + appsV1 "k8s.io/api/apps/v1" + coreV1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + SensorName = "cbcontainers-runtime-sensor" + sensorLabelKey = "app.kubernetes.io/name" + + sensorVerbosityFlag = "-v" + sensorRunCommand = "/run_sensor.sh" + + sensorDNSPolicy = coreV1.DNSClusterFirstWithHostNet + sensorHostNetwork = true + sensorHostPID = true + + desiredConnectionTimeoutSeconds = 60 +) + +var ( + sensorIsPrivileged = true + sensorRunAsUser int64 = 0 + + resolverAddress = fmt.Sprintf("%s.%s.svc.cluster.local", ResolverName, commonState.DataPlaneNamespaceName) +) + +type SensorDaemonSetK8sObject struct{} + +func NewSensorDaemonSetK8sObject() *SensorDaemonSetK8sObject { + return &SensorDaemonSetK8sObject{} +} + +func (obj *SensorDaemonSetK8sObject) EmptyK8sObject() client.Object { + return &appsV1.DaemonSet{} +} + +func (obj *SensorDaemonSetK8sObject) RuntimeChildNamespacedName(_ *cbContainersV1.CBContainersRuntime) types.NamespacedName { + return types.NamespacedName{Name: SensorName, Namespace: commonState.DataPlaneNamespaceName} +} + +func (obj *SensorDaemonSetK8sObject) MutateRuntimeChildK8sObject(k8sObject client.Object, cbContainersRuntime *cbContainersV1.CBContainersRuntime) error { + sensorSpec := &cbContainersRuntime.Spec.SensorSpec + daemonSet, ok := k8sObject.(*appsV1.DaemonSet) + if !ok { + return fmt.Errorf("expected DaemonSet K8s object") + } + + desiredLabels := sensorSpec.Labels + if desiredLabels == nil { + desiredLabels = make(map[string]string) + } + desiredLabels[sensorLabelKey] = SensorName + + if daemonSet.Spec.Selector == nil { + daemonSet.Spec.Selector = &metav1.LabelSelector{} + } + + if daemonSet.ObjectMeta.Annotations == nil { + daemonSet.ObjectMeta.Annotations = make(map[string]string) + } + + if daemonSet.Spec.Template.ObjectMeta.Annotations == nil { + daemonSet.Spec.Template.ObjectMeta.Annotations = make(map[string]string) + } + + daemonSet.ObjectMeta.Labels = desiredLabels + daemonSet.Spec.Selector.MatchLabels = desiredLabels + daemonSet.Spec.Template.ObjectMeta.Labels = desiredLabels + daemonSet.Spec.Template.Spec.ServiceAccountName = commonState.DataPlaneServiceAccountName + daemonSet.Spec.Template.Spec.PriorityClassName = commonState.DataPlanePriorityClassName + daemonSet.Spec.Template.Spec.ImagePullSecrets = []coreV1.LocalObjectReference{{Name: commonState.RegistrySecretName}} + + daemonSet.Spec.Template.Spec.DNSPolicy = sensorDNSPolicy + daemonSet.Spec.Template.Spec.HostNetwork = sensorHostNetwork + daemonSet.Spec.Template.Spec.HostPID = sensorHostPID + + obj.mutateAnnotations(daemonSet, sensorSpec) + obj.mutateContainersList(&daemonSet.Spec.Template.Spec, + sensorSpec, + cbContainersRuntime.Spec.Version, + cbContainersRuntime.Spec.AccessTokenSecretName, + cbContainersRuntime.Spec.InternalGrpcPort, + ) + + return nil +} + +func (obj *SensorDaemonSetK8sObject) mutateAnnotations(daemonSet *appsV1.DaemonSet, sensorSpec *cbContainersV1.CBContainersRuntimeSensorSpec) { + if daemonSet.ObjectMeta.Annotations == nil { + daemonSet.ObjectMeta.Annotations = make(map[string]string) + } + + applyment.EnforceMapContains(daemonSet.ObjectMeta.Annotations, sensorSpec.DaemonSetAnnotations) + + if daemonSet.Spec.Template.ObjectMeta.Annotations == nil { + daemonSet.Spec.Template.ObjectMeta.Annotations = make(map[string]string) + } + + applyment.EnforceMapContains(daemonSet.Spec.Template.ObjectMeta.Annotations, map[string]string{ + "prometheus.io/scrape": fmt.Sprint(*sensorSpec.Prometheus.Enabled), + "prometheus.io/port": fmt.Sprint(sensorSpec.Prometheus.Port), + }) + applyment.EnforceMapContains(daemonSet.Spec.Template.ObjectMeta.Annotations, sensorSpec.PodTemplateAnnotations) +} + +func (obj *SensorDaemonSetK8sObject) mutateContainersList( + templatePodSpec *coreV1.PodSpec, + sensorSpec *cbContainersV1.CBContainersRuntimeSensorSpec, + version, + accessTokenSecretName string, + desiredGRPCPortValue int32) { + + if len(templatePodSpec.Containers) != 1 { + container := coreV1.Container{} + templatePodSpec.Containers = []coreV1.Container{container} + } + + obj.mutateContainer(&templatePodSpec.Containers[0], sensorSpec, version, accessTokenSecretName, desiredGRPCPortValue) +} + +func (obj *SensorDaemonSetK8sObject) mutateContainer( + container *coreV1.Container, + sensorSpec *cbContainersV1.CBContainersRuntimeSensorSpec, + version, + accessTokenSecretName string, + desiredGRPCPortValue int32) { + + container.Name = SensorName + container.Resources = sensorSpec.Resources + container.Args = []string{sensorVerbosityFlag, fmt.Sprintf("%d", *sensorSpec.VerbosityLevel)} + container.Command = []string{sensorRunCommand} + commonState.MutateImage(container, sensorSpec.Image, version) + commonState.MutateContainerFileProbes(container, sensorSpec.Probes) + obj.mutateEnvVars(container, sensorSpec, accessTokenSecretName, desiredGRPCPortValue) + obj.mutateSecurityContext(container) +} + +func (obj *SensorDaemonSetK8sObject) mutateEnvVars( + container *coreV1.Container, + sensorSpec *cbContainersV1.CBContainersRuntimeSensorSpec, + accessTokenSecretName string, + desiredGRPCPortValue int32) { + customEnvs := []coreV1.EnvVar{ + {Name: "RUNTIME_KUBERNETES_SENSOR_GRPC_PORT", Value: fmt.Sprintf("%d", desiredGRPCPortValue)}, + {Name: "RUNTIME_KUBERNETES_SENSOR_RESOLVER_ADDRESS", Value: resolverAddress}, + {Name: "RUNTIME_KUBERNETES_SENSOR_RESOLVER_CONNECTION_TIMEOUT_SECONDS", Value: fmt.Sprintf("%d", desiredConnectionTimeoutSeconds)}, + {Name: "RUNTIME_KUBERNETES_SENSOR_LIVENESS_PATH", Value: sensorSpec.Probes.LivenessPath}, + {Name: "RUNTIME_KUBERNETES_SENSOR_READINESS_PATH", Value: sensorSpec.Probes.ReadinessPath}, + } + + envVarBuilder := commonState.NewEnvVarBuilder(). + WithCustom(customEnvs...). + WithSpec(sensorSpec.Env) + commonState.MutateEnvVars(container, envVarBuilder) +} + +func (obj *SensorDaemonSetK8sObject) mutateSecurityContext(container *coreV1.Container) { + if container.SecurityContext == nil { + container.SecurityContext = &coreV1.SecurityContext{} + } + + container.SecurityContext.Privileged = &sensorIsPrivileged + container.SecurityContext.RunAsUser = &sensorRunAsUser +} diff --git a/cbcontainers/state/runtime/runtime_child_resource.go b/cbcontainers/state/runtime/runtime_child_resource.go new file mode 100644 index 00000000..b1ff6593 --- /dev/null +++ b/cbcontainers/state/runtime/runtime_child_resource.go @@ -0,0 +1,55 @@ +package runtime + +import ( + "context" + + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment" + applymentOptions "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment/options" + stateTypes "github.com/vmware/cbcontainers-operator/cbcontainers/state/types" +) + +type RuntimeChildK8sObject interface { + MutateRuntimeChildK8sObject(k8sObject client.Object, cbContainersRuntime *cbcontainersv1.CBContainersRuntime) error + RuntimeChildNamespacedName(cbContainersRuntime *cbcontainersv1.CBContainersRuntime) types.NamespacedName + stateTypes.DesiredK8sObjectInitializer +} + +type DefaultRuntimeChildK8sObjectApplier struct{} + +func NewDefaultRuntimeChildK8sObjectApplier() *DefaultRuntimeChildK8sObjectApplier { + return &DefaultRuntimeChildK8sObjectApplier{} +} + +func (applier *DefaultRuntimeChildK8sObjectApplier) ApplyRuntimeChildK8sObject(ctx context.Context, cbContainersRuntime *cbcontainersv1.CBContainersRuntime, client client.Client, runtimeChildK8sObject RuntimeChildK8sObject, applyOptionsList ...*applymentOptions.ApplyOptions) (bool, client.Object, error) { + runtimeChildWrapper := NewCBContainersRuntimeChildK8sObject(cbContainersRuntime, runtimeChildK8sObject) + return applyment.ApplyDesiredK8sObject(ctx, client, runtimeChildWrapper, applyOptionsList...) +} + +func (applier *DefaultRuntimeChildK8sObjectApplier) DeleteK8sObjectIfExists(ctx context.Context, cbContainersRuntime *cbcontainersv1.CBContainersRuntime, client client.Client, runtimeChildK8sObject RuntimeChildK8sObject) (bool, error) { + runtimeChildWrapper := NewCBContainersRuntimeChildK8sObject(cbContainersRuntime, runtimeChildK8sObject) + return applyment.DeleteK8sObjectIfExists(ctx, client, runtimeChildWrapper) +} + +type CBContainersRuntimeChildK8sObject struct { + cbContainersRuntime *cbcontainersv1.CBContainersRuntime + RuntimeChildK8sObject +} + +func NewCBContainersRuntimeChildK8sObject(cbContainersRuntime *cbcontainersv1.CBContainersRuntime, runtimeChildK8sObject RuntimeChildK8sObject) *CBContainersRuntimeChildK8sObject { + return &CBContainersRuntimeChildK8sObject{ + cbContainersRuntime: cbContainersRuntime, + RuntimeChildK8sObject: runtimeChildK8sObject, + } +} + +func (runtimeChildWrapper *CBContainersRuntimeChildK8sObject) NamespacedName() types.NamespacedName { + return runtimeChildWrapper.RuntimeChildNamespacedName(runtimeChildWrapper.cbContainersRuntime) +} + +func (runtimeChildWrapper *CBContainersRuntimeChildK8sObject) MutateK8sObject(k8sObject client.Object) error { + return runtimeChildWrapper.MutateRuntimeChildK8sObject(k8sObject, runtimeChildWrapper.cbContainersRuntime) +} diff --git a/cbcontainers/state/runtime/runtime_state_applier.go b/cbcontainers/state/runtime/runtime_state_applier.go new file mode 100644 index 00000000..ee4f9c07 --- /dev/null +++ b/cbcontainers/state/runtime/runtime_state_applier.go @@ -0,0 +1,77 @@ +package runtime + +import ( + "context" + + "github.com/go-logr/logr" + "sigs.k8s.io/controller-runtime/pkg/client" + + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + applymentOptions "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment/options" + runtimeObjects "github.com/vmware/cbcontainers-operator/cbcontainers/state/runtime/objects" +) + +type RuntimeChildK8sObjectApplier interface { + ApplyRuntimeChildK8sObject(ctx context.Context, cbContainersRuntime *cbcontainersv1.CBContainersRuntime, client client.Client, runtimeChildK8sObject RuntimeChildK8sObject, applyOptionsList ...*applymentOptions.ApplyOptions) (bool, client.Object, error) + DeleteK8sObjectIfExists(ctx context.Context, cbContainersRuntime *cbcontainersv1.CBContainersRuntime, client client.Client, runtimeChildK8sObject RuntimeChildK8sObject) (bool, error) +} + +type RuntimeStateApplier struct { + resolverDeployment *runtimeObjects.ResolverDeploymentK8sObject + resolverService *runtimeObjects.ResolverServiceK8sObject + sensorDaemonSet *runtimeObjects.SensorDaemonSetK8sObject + childApplier RuntimeChildK8sObjectApplier + log logr.Logger +} + +func NewRuntimeStateApplier(log logr.Logger, childApplier RuntimeChildK8sObjectApplier) *RuntimeStateApplier { + return &RuntimeStateApplier{ + resolverDeployment: runtimeObjects.NewResolverDeploymentK8sObject(), + resolverService: runtimeObjects.NewResolverServiceK8sObject(), + sensorDaemonSet: runtimeObjects.NewSensorDaemonSetK8sObject(), + childApplier: childApplier, + log: log, + } +} + +func (c *RuntimeStateApplier) ApplyDesiredState(ctx context.Context, cbContainersRuntime *cbcontainersv1.CBContainersRuntime, client client.Client, setOwner applymentOptions.OwnerSetter) (bool, error) { + applyOptions := applymentOptions.NewApplyOptions().SetOwnerSetter(setOwner) + + mutatedResolver, err := c.applyResolver(ctx, cbContainersRuntime, client, applyOptions) + if err != nil { + return false, err + } + c.log.Info("Applied runtime kubernetes resolver objects", "Mutated", mutatedResolver) + + mutatedSensor, err := c.applySensor(ctx, cbContainersRuntime, client, applyOptions) + if err != nil { + return false, err + } + c.log.Info("Applied runtime kubernetes sensor objects", "Mutated", mutatedSensor) + + return mutatedResolver || mutatedSensor, nil +} + +func (c *RuntimeStateApplier) applyResolver(ctx context.Context, cbContainersRuntime *cbcontainersv1.CBContainersRuntime, client client.Client, applyOptions *applymentOptions.ApplyOptions) (bool, error) { + mutatedService, _, err := c.childApplier.ApplyRuntimeChildK8sObject(ctx, cbContainersRuntime, client, c.resolverService, applyOptions) + if err != nil { + return false, err + } + c.log.Info("Applied kubernetes resolver service", "Mutated", mutatedService) + + mutatedDeployment, _, err := c.childApplier.ApplyRuntimeChildK8sObject(ctx, cbContainersRuntime, client, c.resolverDeployment, applyOptions) + if err != nil { + return false, err + } + c.log.Info("Applied runtime kubernetes resolver deployment", "Mutated", mutatedDeployment) + return mutatedService || mutatedDeployment, nil +} + +func (c *RuntimeStateApplier) applySensor(ctx context.Context, cbContainersRuntime *cbcontainersv1.CBContainersRuntime, client client.Client, applyOptions *applymentOptions.ApplyOptions) (bool, error) { + mutatedDaemonSet, _, err := c.childApplier.ApplyRuntimeChildK8sObject(ctx, cbContainersRuntime, client, c.sensorDaemonSet, applyOptions) + if err != nil { + return false, err + } + c.log.Info("Applied runtime kubernetes sensor daemon set", "Mutated", mutatedDaemonSet) + return mutatedDaemonSet, nil +} diff --git a/cbcontainers/state/runtime/runtime_state_applier_test.go b/cbcontainers/state/runtime/runtime_state_applier_test.go new file mode 100644 index 00000000..2af17a10 --- /dev/null +++ b/cbcontainers/state/runtime/runtime_state_applier_test.go @@ -0,0 +1,149 @@ +package runtime_test + +import ( + "context" + "reflect" + "testing" + + logrTesting "github.com/go-logr/logr/testing" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment/options" + commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" + "github.com/vmware/cbcontainers-operator/cbcontainers/state/runtime" + "github.com/vmware/cbcontainers-operator/cbcontainers/state/runtime/mocks" + "github.com/vmware/cbcontainers-operator/cbcontainers/state/runtime/objects" + "github.com/vmware/cbcontainers-operator/cbcontainers/test_utils" + testUtilsMocks "github.com/vmware/cbcontainers-operator/cbcontainers/test_utils/mocks" + appsV1 "k8s.io/api/apps/v1" + coreV1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + NumberOfExpectedAppliedObjects = 3 +) + +type AppliedK8sObjectsChanger func(K8sObjectDetails, client.Object) + +var ( + Version = test_utils.RandomString() + + ResolverDeploymentDetails = K8sObjectDetails{ + Namespace: commonState.DataPlaneNamespaceName, + Name: objects.ResolverName, + ObjectType: reflect.TypeOf(&appsV1.Deployment{}), + } + + MutateDeploymentReadyReplicas = func(details K8sObjectDetails, object client.Object, readyReplicas int32) { + if details != ResolverDeploymentDetails { + return + } + + enforcerDeployment := object.(*appsV1.Deployment) + enforcerDeployment.Status.ReadyReplicas = readyReplicas + } + + MutateDeploymentToBeWithReadyReplica AppliedK8sObjectsChanger = func(details K8sObjectDetails, object client.Object) { + MutateDeploymentReadyReplicas(details, object, 1) + } +) + +type RuntimeStateApplierTestMocks struct { + client *testUtilsMocks.MockClient + childApplier *mocks.MockRuntimeChildK8sObjectApplier + cbContainersRuntime *cbcontainersv1.CBContainersRuntime +} + +type RuntimeStateApplierTestSetup func(*RuntimeStateApplierTestMocks) + +type K8sObjectDetails struct { + Namespace string + Name string + ObjectType reflect.Type +} + +func testRuntimeStateApplier(t *testing.T, setup RuntimeStateApplierTestSetup) (bool, error) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + cbContainersRuntime := &cbcontainersv1.CBContainersRuntime{ + Spec: cbcontainersv1.CBContainersRuntimeSpec{ + Version: Version, + }, + } + + mockObjects := &RuntimeStateApplierTestMocks{ + client: testUtilsMocks.NewMockClient(ctrl), + childApplier: mocks.NewMockRuntimeChildK8sObjectApplier(ctrl), + cbContainersRuntime: cbContainersRuntime, + } + + setup(mockObjects) + + return runtime.NewRuntimeStateApplier(&logrTesting.TestLogger{T: t}, mockObjects.childApplier).ApplyDesiredState(context.Background(), cbContainersRuntime, mockObjects.client, nil) +} + +func getAppliedAndDeletedObjects(t *testing.T, appliedK8sObjectsChangers ...AppliedK8sObjectsChanger) ([]K8sObjectDetails, []K8sObjectDetails, error) { + appliedObjects := make([]K8sObjectDetails, 0) + deletedObjects := make([]K8sObjectDetails, 0) + + _, err := testRuntimeStateApplier(t, func(mocks *RuntimeStateApplierTestMocks) { + mocks.childApplier.EXPECT().ApplyRuntimeChildK8sObject(gomock.Any(), mocks.cbContainersRuntime, mocks.client, gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, cr *cbcontainersv1.CBContainersRuntime, client client.Client, childObject runtime.RuntimeChildK8sObject, options ...*options.ApplyOptions) (bool, client.Object, error) { + namespacedName := childObject.RuntimeChildNamespacedName(cr) + k8sObject := childObject.EmptyK8sObject() + objType := reflect.TypeOf(k8sObject) + objectDetails := K8sObjectDetails{Namespace: namespacedName.Namespace, Name: namespacedName.Name, ObjectType: objType} + appliedObjects = append(appliedObjects, objectDetails) + + for _, changeAppliedK8sObjects := range appliedK8sObjectsChangers { + changeAppliedK8sObjects(objectDetails, k8sObject) + } + + return true, k8sObject, nil + }).AnyTimes() + + mocks.childApplier.EXPECT().DeleteK8sObjectIfExists(gomock.Any(), mocks.cbContainersRuntime, mocks.client, gomock.Any()). + DoAndReturn(func(ctx context.Context, cr *cbcontainersv1.CBContainersRuntime, client client.Client, obj runtime.RuntimeChildK8sObject) (bool, error) { + namespacedName := obj.RuntimeChildNamespacedName(cr) + objType := reflect.TypeOf(obj.EmptyK8sObject()) + deletedObjects = append(deletedObjects, K8sObjectDetails{Namespace: namespacedName.Namespace, Name: namespacedName.Name, ObjectType: objType}) + return true, nil + }).AnyTimes() + }) + + return appliedObjects, deletedObjects, err +} + +func getAndAssertAppliedAndDeletedObjects(t *testing.T) ([]K8sObjectDetails, []K8sObjectDetails) { + appliedObjects, deletedObjects, err := getAppliedAndDeletedObjects(t, MutateDeploymentToBeWithReadyReplica) + + require.NoError(t, err) + require.Len(t, appliedObjects, NumberOfExpectedAppliedObjects) + return appliedObjects, deletedObjects +} + +func TestResolverServiceIsApplied(t *testing.T) { + appliedObjects, _ := getAndAssertAppliedAndDeletedObjects(t) + require.Contains(t, appliedObjects, K8sObjectDetails{ + Namespace: commonState.DataPlaneNamespaceName, + Name: objects.ResolverName, + ObjectType: reflect.TypeOf(&coreV1.Service{}), + }) +} + +func TestEnforcerDeploymentIsApplied(t *testing.T) { + appliedObjects, _ := getAndAssertAppliedAndDeletedObjects(t) + require.Contains(t, appliedObjects, ResolverDeploymentDetails) +} + +func TestSensorDaemonsetIsApplied(t *testing.T) { + appliedObjects, _ := getAndAssertAppliedAndDeletedObjects(t) + require.Contains(t, appliedObjects, K8sObjectDetails{ + Namespace: commonState.DataPlaneNamespaceName, + Name: objects.SensorName, + ObjectType: reflect.TypeOf(&appsV1.DaemonSet{}), + }) +} diff --git a/config/crd/bases/operator.containers.carbonblack.io_cbcontainersruntimes.yaml b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersruntimes.yaml new file mode 100644 index 00000000..b1ff64d7 --- /dev/null +++ b/config/crd/bases/operator.containers.carbonblack.io_cbcontainersruntimes.yaml @@ -0,0 +1,258 @@ + +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.4.1 + creationTimestamp: null + name: cbcontainersruntimes.operator.containers.carbonblack.io +spec: + group: operator.containers.carbonblack.io + names: + kind: CBContainersRuntime + listKind: CBContainersRuntimeList + plural: cbcontainersruntimes + singular: cbcontainersruntime + scope: Cluster + subresources: + status: {} + validation: + openAPIV3Schema: + description: CBContainersRuntime is the Schema for the cbcontainersruntimes + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: CBContainersRuntimeSpec defines the desired state of CBContainersRuntime + properties: + accessTokenSecretName: + type: string + internalGrpcPort: + format: int32 + type: integer + resolverSpec: + properties: + deploymentAnnotations: + additionalProperties: + type: string + type: object + env: + additionalProperties: + type: string + type: object + eventsGatewaySpec: + properties: + host: + type: string + port: + type: integer + required: + - host + type: object + image: + properties: + pullPolicy: + description: PullPolicy describes a policy for if/when to pull + a container image + type: string + repository: + type: string + tag: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + podTemplateAnnotations: + additionalProperties: + type: string + type: object + probes: + properties: + failureThreshold: + format: int32 + type: integer + initialDelaySeconds: + format: int32 + type: integer + livenessPath: + type: string + periodSeconds: + format: int32 + type: integer + port: + type: integer + readinessPath: + type: string + scheme: + description: URIScheme identifies the scheme used for connection + to a host for Get actions + type: string + successThreshold: + format: int32 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + prometheus: + properties: + enabled: + type: boolean + port: + type: integer + type: object + replicasCount: + format: int32 + type: integer + resources: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + required: + - eventsGatewaySpec + type: object + sensorSpec: + properties: + daemonSetAnnotations: + additionalProperties: + type: string + type: object + env: + additionalProperties: + type: string + type: object + image: + properties: + pullPolicy: + description: PullPolicy describes a policy for if/when to pull + a container image + type: string + repository: + type: string + tag: + type: string + type: object + labels: + additionalProperties: + type: string + type: object + podTemplateAnnotations: + additionalProperties: + type: string + type: object + probes: + properties: + failureThreshold: + format: int32 + type: integer + initialDelaySeconds: + format: int32 + type: integer + livenessPath: + type: string + periodSeconds: + format: int32 + type: integer + readinessPath: + type: string + successThreshold: + format: int32 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + prometheus: + properties: + enabled: + type: boolean + port: + type: integer + type: object + resources: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of compute + resources required. If Requests is omitted for a container, + it defaults to Limits if that is explicitly specified, otherwise + to an implementation-defined value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + verbosity_level: + type: integer + type: object + version: + type: string + required: + - version + type: object + status: + description: CBContainersRuntimeStatus defines the observed state of CBContainersRuntime + type: object + type: object + version: v1 + versions: + - name: v1 + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 176f4a75..5d9f52be 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -4,6 +4,7 @@ resources: - bases/operator.containers.carbonblack.io_cbcontainersclusters.yaml - bases/operator.containers.carbonblack.io_cbcontainershardenings.yaml +- bases/operator.containers.carbonblack.io_cbcontainersruntimes.yaml # +kubebuilder:scaffold:crdkustomizeresource patchesStrategicMerge: @@ -11,12 +12,14 @@ patchesStrategicMerge: # patches here are for enabling the conversion webhook for each CRD #- patches/webhook_in_cbcontainersclusters.yaml #- patches/webhook_in_cbcontainershardenings.yaml +#- patches/webhook_in_cbcontainersruntimes.yaml # +kubebuilder:scaffold:crdkustomizewebhookpatch # [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix. # patches here are for enabling the CA injection for each CRD #- patches/cainjection_in_cbcontainersclusters.yaml #- patches/cainjection_in_cbcontainershardenings.yaml +#- patches/cainjection_in_cbcontainersruntimes.yaml # +kubebuilder:scaffold:crdkustomizecainjectionpatch # the following config is for teaching kustomize how to do kustomization for CRDs. diff --git a/config/crd/patches/cainjection_in_cbcontainersruntimes.yaml b/config/crd/patches/cainjection_in_cbcontainersruntimes.yaml new file mode 100644 index 00000000..419005ba --- /dev/null +++ b/config/crd/patches/cainjection_in_cbcontainersruntimes.yaml @@ -0,0 +1,7 @@ +# The following patch adds a directive for certmanager to inject CA into the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) + name: cbcontainersruntimes.operator.containers.carbonblack.io diff --git a/config/crd/patches/webhook_in_cbcontainersruntimes.yaml b/config/crd/patches/webhook_in_cbcontainersruntimes.yaml new file mode 100644 index 00000000..1fd42d40 --- /dev/null +++ b/config/crd/patches/webhook_in_cbcontainersruntimes.yaml @@ -0,0 +1,14 @@ +# The following patch enables a conversion webhook for the CRD +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: cbcontainersruntimes.operator.containers.carbonblack.io +spec: + conversion: + strategy: Webhook + webhook: + clientConfig: + service: + namespace: system + name: webhook-service + path: /convert diff --git a/config/rbac/cbcontainersruntime_editor_role.yaml b/config/rbac/cbcontainersruntime_editor_role.yaml new file mode 100644 index 00000000..25d0b893 --- /dev/null +++ b/config/rbac/cbcontainersruntime_editor_role.yaml @@ -0,0 +1,24 @@ +# permissions for end users to edit cbcontainersruntimes. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cbcontainersruntime-editor-role +rules: +- apiGroups: + - operator.containers.carbonblack.io + resources: + - cbcontainersruntimes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - operator.containers.carbonblack.io + resources: + - cbcontainersruntimes/status + verbs: + - get diff --git a/config/rbac/cbcontainersruntime_viewer_role.yaml b/config/rbac/cbcontainersruntime_viewer_role.yaml new file mode 100644 index 00000000..7ab1c13f --- /dev/null +++ b/config/rbac/cbcontainersruntime_viewer_role.yaml @@ -0,0 +1,20 @@ +# permissions for end users to view cbcontainersruntimes. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: cbcontainersruntime-viewer-role +rules: +- apiGroups: + - operator.containers.carbonblack.io + resources: + - cbcontainersruntimes + verbs: + - get + - list + - watch +- apiGroups: + - operator.containers.carbonblack.io + resources: + - cbcontainersruntimes/status + verbs: + - get diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index edd2f5db..2d2d1dfd 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -42,6 +42,21 @@ rules: - get - list - watch +- apiGroups: + - apps + - "" + resources: + - daemonsets + - deployments + - services + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - apps - "" @@ -121,6 +136,32 @@ rules: - get - patch - update +- apiGroups: + - operator.containers.carbonblack.io + resources: + - cbcontainersruntimes + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - operator.containers.carbonblack.io + resources: + - cbcontainersruntimes/finalizers + verbs: + - update +- apiGroups: + - operator.containers.carbonblack.io + resources: + - cbcontainersruntimes/status + verbs: + - get + - patch + - update - apiGroups: - scheduling.k8s.io resources: diff --git a/config/samples/_v1_cbcontainersruntime.yaml b/config/samples/_v1_cbcontainersruntime.yaml new file mode 100644 index 00000000..881910fb --- /dev/null +++ b/config/samples/_v1_cbcontainersruntime.yaml @@ -0,0 +1,7 @@ +apiVersion: operator.containers.carbonblack.io/v1 +kind: CBContainersRuntime +metadata: + name: cbcontainersruntime-sample +spec: + # Add fields here + foo: bar diff --git a/controllers/cbcontainerscluster_controller.go b/controllers/cbcontainerscluster_controller.go index 53e0808e..bd24a6f4 100644 --- a/controllers/cbcontainerscluster_controller.go +++ b/controllers/cbcontainerscluster_controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + "github.com/go-logr/logr" "github.com/vmware/cbcontainers-operator/cbcontainers/models" applymentOptions "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment/options" @@ -126,7 +127,7 @@ func (r *CBContainersClusterReconciler) setDefaults(cbContainersCluster *cbconta } if cbContainersCluster.Spec.ApiGatewaySpec.AccessTokenSecretName == "" { - cbContainersCluster.Spec.ApiGatewaySpec.AccessTokenSecretName = "cbcontainers-access-token" + cbContainersCluster.Spec.ApiGatewaySpec.AccessTokenSecretName = defaultAccessToken } if cbContainersCluster.Spec.EventsGatewaySpec.Port == 0 { diff --git a/controllers/cbcontainerscluster_controller_test.go b/controllers/cbcontainerscluster_controller_test.go index 7b984343..a5a7bfcb 100644 --- a/controllers/cbcontainerscluster_controller_test.go +++ b/controllers/cbcontainerscluster_controller_test.go @@ -3,6 +3,8 @@ package controllers_test import ( "context" "fmt" + "testing" + logrTesting "github.com/go-logr/logr/testing" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" @@ -17,7 +19,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" ctrlRuntime "sigs.k8s.io/controller-runtime" - "testing" ) type SetupClusterControllerTest func(*ClusterControllerTestMocks) @@ -39,7 +40,7 @@ var ( ClusterCustomResourceItems = []cbcontainersv1.CBContainersCluster{ { Spec: cbcontainersv1.CBContainersClusterSpec{ - ApiGatewaySpec: cbcontainersv1.CBContainersClusterApiGatewaySpec{ + ApiGatewaySpec: cbcontainersv1.CBContainersApiGatewaySpec{ AccessTokenSecretName: ClusterAccessTokenSecretName, }, }, @@ -76,7 +77,7 @@ func testCBContainersClusterController(t *testing.T, setups ...SetupClusterContr func setupClusterCustomResource(testMocks *ClusterControllerTestMocks) { testMocks.client.EXPECT().List(testMocks.ctx, &cbcontainersv1.CBContainersClusterList{}). - Do(func(ctx context.Context, list *cbcontainersv1.CBContainersClusterList) { + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersClusterList, _ ...interface{}) { list.Items = ClusterCustomResourceItems }). Return(nil) @@ -113,7 +114,7 @@ func TestNotFindingAnyClusterResourceShouldReturnNil(t *testing.T) { func TestFindingMoreThanOneClusterResourceShouldReturnError(t *testing.T) { _, err := testCBContainersClusterController(t, func(testMocks *ClusterControllerTestMocks) { testMocks.client.EXPECT().List(testMocks.ctx, &cbcontainersv1.CBContainersClusterList{}). - Do(func(ctx context.Context, list *cbcontainersv1.CBContainersClusterList) { + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersClusterList, _ ...interface{}) { list.Items = append(list.Items, cbcontainersv1.CBContainersCluster{}) list.Items = append(list.Items, cbcontainersv1.CBContainersCluster{}) }). diff --git a/controllers/cbcontainershardening_controller.go b/controllers/cbcontainershardening_controller.go index 285e253d..4f98d947 100644 --- a/controllers/cbcontainershardening_controller.go +++ b/controllers/cbcontainershardening_controller.go @@ -19,23 +19,20 @@ package controllers import ( "context" "fmt" + + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" applymentOptions "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment/options" admissionsV1 "k8s.io/api/admissionregistration/v1beta1" appsV1 "k8s.io/api/apps/v1" coreV1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/go-logr/logr" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - - cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" ) -var falseRef = false - type HardeningStateApplier interface { ApplyDesiredState(ctx context.Context, cbContainersHardening *cbcontainersv1.CBContainersHardening, client client.Client, setOwner applymentOptions.OwnerSetter) (bool, error) } @@ -107,7 +104,7 @@ func (r *CBContainersHardeningReconciler) Reconcile(ctx context.Context, req ctr func (r *CBContainersHardeningReconciler) setDefaults(cbContainersHardening *cbcontainersv1.CBContainersHardening) error { if cbContainersHardening.Spec.AccessTokenSecretName == "" { - cbContainersHardening.Spec.AccessTokenSecretName = "cbcontainers-access-token" + cbContainersHardening.Spec.AccessTokenSecretName = defaultAccessToken } if cbContainersHardening.Spec.EnforcerSpec.Labels == nil { @@ -131,15 +128,15 @@ func (r *CBContainersHardeningReconciler) setDefaults(cbContainersHardening *cbc cbContainersHardening.Spec.EnforcerSpec.Env = make(map[string]string) } - r.setDefaultPrometheus(&cbContainersHardening.Spec.EnforcerSpec.Prometheus) + setDefaultPrometheus(&cbContainersHardening.Spec.EnforcerSpec.Prometheus) - r.setDefaultImage(&cbContainersHardening.Spec.EnforcerSpec.Image, "cbartifactory/guardrails-enforcer") + setDefaultImage(&cbContainersHardening.Spec.EnforcerSpec.Image, "cbartifactory/guardrails-enforcer") - if err := r.setDefaultResourceRequirements(&cbContainersHardening.Spec.EnforcerSpec.Resources, "64Mi", "30m", "256Mi", "200m"); err != nil { + if err := setDefaultResourceRequirements(&cbContainersHardening.Spec.EnforcerSpec.Resources, "64Mi", "30m", "256Mi", "200m"); err != nil { return err } - r.setDefaultProbes(&cbContainersHardening.Spec.EnforcerSpec.Probes) + setDefaultHTTPProbes(&cbContainersHardening.Spec.EnforcerSpec.Probes) if cbContainersHardening.Spec.EnforcerSpec.WebhookTimeoutSeconds == 0 { cbContainersHardening.Spec.EnforcerSpec.WebhookTimeoutSeconds = 5 @@ -161,13 +158,13 @@ func (r *CBContainersHardeningReconciler) setDefaults(cbContainersHardening *cbc cbContainersHardening.Spec.StateReporterSpec.Env = make(map[string]string) } - r.setDefaultImage(&cbContainersHardening.Spec.StateReporterSpec.Image, "cbartifactory/guardrails-state-reporter") + setDefaultImage(&cbContainersHardening.Spec.StateReporterSpec.Image, "cbartifactory/guardrails-state-reporter") - if err := r.setDefaultResourceRequirements(&cbContainersHardening.Spec.StateReporterSpec.Resources, "64Mi", "30m", "256Mi", "200m"); err != nil { + if err := setDefaultResourceRequirements(&cbContainersHardening.Spec.StateReporterSpec.Resources, "64Mi", "30m", "256Mi", "200m"); err != nil { return err } - r.setDefaultProbes(&cbContainersHardening.Spec.StateReporterSpec.Probes) + setDefaultHTTPProbes(&cbContainersHardening.Spec.StateReporterSpec.Probes) if cbContainersHardening.Spec.EventsGatewaySpec.Port == 0 { cbContainersHardening.Spec.EventsGatewaySpec.Port = 443 @@ -176,109 +173,6 @@ func (r *CBContainersHardeningReconciler) setDefaults(cbContainersHardening *cbc return nil } -func (r *CBContainersHardeningReconciler) setDefaultProbes(probesSpec *cbcontainersv1.CBContainersHardeningProbesSpec) { - if probesSpec.ReadinessPath == "" { - probesSpec.ReadinessPath = "/ready" - } - - if probesSpec.LivenessPath == "" { - probesSpec.LivenessPath = "/alive" - } - - if probesSpec.Port == 0 { - probesSpec.Port = 8181 - } - - if probesSpec.Scheme == "" { - probesSpec.Scheme = coreV1.URISchemeHTTP - } - - if probesSpec.InitialDelaySeconds == 0 { - probesSpec.InitialDelaySeconds = 3 - } - - if probesSpec.TimeoutSeconds == 0 { - probesSpec.TimeoutSeconds = 1 - } - - if probesSpec.PeriodSeconds == 0 { - probesSpec.PeriodSeconds = 30 - } - - if probesSpec.SuccessThreshold == 0 { - probesSpec.SuccessThreshold = 1 - } - - if probesSpec.FailureThreshold == 0 { - probesSpec.FailureThreshold = 3 - } -} - -func (r *CBContainersHardeningReconciler) setDefaultPrometheus(prometheusSpec *cbcontainersv1.CBContainersHardeningPrometheusSpec) { - if prometheusSpec.Enabled == nil { - prometheusSpec.Enabled = &falseRef - } - - if prometheusSpec.Port == 0 { - prometheusSpec.Port = 7071 - } -} - -func (r *CBContainersHardeningReconciler) setDefaultImage(imageSpec *cbcontainersv1.CBContainersHardeningImageSpec, imageName string) { - if imageSpec.Repository == "" { - imageSpec.Repository = imageName - } - - if imageSpec.PullPolicy == "" { - imageSpec.PullPolicy = "Always" - } -} - -func (r *CBContainersHardeningReconciler) setDefaultResourceRequirements(resources *coreV1.ResourceRequirements, requestMemory, requestCpu, limitMemory, limitCpu string) error { - if resources.Requests == nil { - resources.Requests = make(coreV1.ResourceList) - } - - if err := r.setDefaultsResourcesList(resources.Requests, requestMemory, requestCpu); err != nil { - return err - } - - if resources.Limits == nil { - resources.Limits = make(coreV1.ResourceList) - } - - if err := r.setDefaultsResourcesList(resources.Limits, limitMemory, limitCpu); err != nil { - return err - } - - return nil -} - -func (r *CBContainersHardeningReconciler) setDefaultsResourcesList(list coreV1.ResourceList, memory, cpu string) error { - if err := r.setDefaultResource(list, coreV1.ResourceMemory, memory); err != nil { - return err - } - - if err := r.setDefaultResource(list, coreV1.ResourceCPU, cpu); err != nil { - return err - } - - return nil -} - -func (r *CBContainersHardeningReconciler) setDefaultResource(list coreV1.ResourceList, resourceName coreV1.ResourceName, value string) error { - if _, ok := list[resourceName]; !ok { - quantity, err := resource.ParseQuantity(value) - if err != nil { - return err - } - - list[resourceName] = quantity - } - - return nil -} - // SetupWithManager sets up the controller with the Manager. func (r *CBContainersHardeningReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). diff --git a/controllers/cbcontainershardening_controller_test.go b/controllers/cbcontainershardening_controller_test.go index fe4af784..6948ed32 100644 --- a/controllers/cbcontainershardening_controller_test.go +++ b/controllers/cbcontainershardening_controller_test.go @@ -3,6 +3,8 @@ package controllers_test import ( "context" "fmt" + "testing" + logrTesting "github.com/go-logr/logr/testing" "github.com/golang/mock/gomock" "github.com/stretchr/testify/require" @@ -13,7 +15,6 @@ import ( "github.com/vmware/cbcontainers-operator/controllers/mocks" "k8s.io/apimachinery/pkg/runtime" ctrlRuntime "sigs.k8s.io/controller-runtime" - "testing" ) type SetupHardeningControllerTest func(*HardeningControllerTestMocks) @@ -61,7 +62,7 @@ func testCBContainersHardeningController(t *testing.T, setups ...SetupHardeningC func setupHardeningCustomResource(testMocks *HardeningControllerTestMocks) { testMocks.client.EXPECT().List(testMocks.ctx, &cbcontainersv1.CBContainersHardeningList{}). - Do(func(ctx context.Context, list *cbcontainersv1.CBContainersHardeningList) { + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersHardeningList, _ ...interface{}) { list.Items = HardeningCustomResourceItems }). Return(nil) @@ -87,7 +88,7 @@ func TestNotFindingAnyHardeningResourceShouldReturnNil(t *testing.T) { func TestFindingMoreThanOneHardeningResourceShouldReturnError(t *testing.T) { _, err := testCBContainersHardeningController(t, func(testMocks *HardeningControllerTestMocks) { testMocks.client.EXPECT().List(testMocks.ctx, &cbcontainersv1.CBContainersHardeningList{}). - Do(func(ctx context.Context, list *cbcontainersv1.CBContainersHardeningList) { + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersHardeningList, _ ...interface{}) { list.Items = append(list.Items, cbcontainersv1.CBContainersHardening{}) list.Items = append(list.Items, cbcontainersv1.CBContainersHardening{}) }). diff --git a/controllers/cbcontainersruntime_controller.go b/controllers/cbcontainersruntime_controller.go new file mode 100644 index 00000000..97437720 --- /dev/null +++ b/controllers/cbcontainersruntime_controller.go @@ -0,0 +1,189 @@ +/* +Copyright 2021. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + appsV1 "k8s.io/api/apps/v1" + coreV1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + operatorcontainerscarbonblackiov1 "github.com/vmware/cbcontainers-operator/api/v1" + applymentOptions "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment/options" +) + +type RuntimeStateApplier interface { + ApplyDesiredState(ctx context.Context, cbContainersRuntime *operatorcontainerscarbonblackiov1.CBContainersRuntime, client client.Client, setOwner applymentOptions.OwnerSetter) (bool, error) +} + +// CBContainersRuntimeReconciler reconciles a CBContainersRuntime object +type CBContainersRuntimeReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme + RuntimeStateApplier RuntimeStateApplier +} + +func (r *CBContainersRuntimeReconciler) getContainersRuntimeObject(ctx context.Context) (*operatorcontainerscarbonblackiov1.CBContainersRuntime, error) { + cbContainersRuntimeList := &operatorcontainerscarbonblackiov1.CBContainersRuntimeList{} + if err := r.List(ctx, cbContainersRuntimeList); err != nil { + return nil, fmt.Errorf("couldn't list CBContainersRuntime k8s objects: %v", err) + } + + if cbContainersRuntimeList.Items == nil || len(cbContainersRuntimeList.Items) == 0 { + return nil, nil + } + + if len(cbContainersRuntimeList.Items) > 1 { + return nil, fmt.Errorf("there is more than 1 CBContainersRuntime k8s object, please delete unwanted resources") + } + + return &cbContainersRuntimeList.Items[0], nil +} + +// +kubebuilder:rbac:groups=operator.containers.carbonblack.io,resources=cbcontainersruntimes,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=operator.containers.carbonblack.io,resources=cbcontainersruntimes/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=operator.containers.carbonblack.io,resources=cbcontainersruntimes/finalizers,verbs=update +// +kubebuilder:rbac:groups={apps,core},resources={deployments,services,daemonsets},verbs=get;list;watch;create;update;patch;delete + +func (r *CBContainersRuntimeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + r.Log.Info("\n\n") + r.Log.Info("Got reconcile request", "namespaced name", req.NamespacedName) + r.Log.Info("Starting reconciling") + + r.Log.Info("Getting CBContainersRuntime object") + cbContainersRuntime, err := r.getContainersRuntimeObject(ctx) + if err != nil { + return ctrl.Result{}, err + } + + if cbContainersRuntime == nil { + return ctrl.Result{}, nil + } + + if err := r.setDefaults(cbContainersRuntime); err != nil { + return ctrl.Result{}, fmt.Errorf("faild to set defaults to CR: %v", err) + } + + setOwner := func(controlledResource metav1.Object) error { + return ctrl.SetControllerReference(cbContainersRuntime, controlledResource, r.Scheme) + } + + r.Log.Info("Applying desired state") + stateWasChanged, err := r.RuntimeStateApplier.ApplyDesiredState(ctx, cbContainersRuntime, r.Client, setOwner) + if err != nil { + return ctrl.Result{}, err + } + + r.Log.Info("Finished reconciling", "Requiring", stateWasChanged) + r.Log.Info("\n\n") + return ctrl.Result{Requeue: stateWasChanged}, nil +} + +func (r *CBContainersRuntimeReconciler) setDefaults(cbContainersRuntime *operatorcontainerscarbonblackiov1.CBContainersRuntime) error { + if cbContainersRuntime.Spec.AccessTokenSecretName == "" { + cbContainersRuntime.Spec.AccessTokenSecretName = defaultAccessToken + } + + if cbContainersRuntime.Spec.ResolverSpec.Labels == nil { + cbContainersRuntime.Spec.ResolverSpec.Labels = make(map[string]string) + } + + if cbContainersRuntime.Spec.ResolverSpec.DeploymentAnnotations == nil { + cbContainersRuntime.Spec.ResolverSpec.DeploymentAnnotations = make(map[string]string) + } + + if cbContainersRuntime.Spec.ResolverSpec.PodTemplateAnnotations == nil { + cbContainersRuntime.Spec.ResolverSpec.PodTemplateAnnotations = make(map[string]string) + } + + if cbContainersRuntime.Spec.ResolverSpec.ReplicasCount == nil { + defaultReplicaCount := int32(1) + cbContainersRuntime.Spec.ResolverSpec.ReplicasCount = &defaultReplicaCount + } + + if cbContainersRuntime.Spec.ResolverSpec.Env == nil { + cbContainersRuntime.Spec.ResolverSpec.Env = make(map[string]string) + } + + if cbContainersRuntime.Spec.ResolverSpec.EventsGatewaySpec.Port == 0 { + cbContainersRuntime.Spec.ResolverSpec.EventsGatewaySpec.Port = 443 + } + + setDefaultPrometheus(&cbContainersRuntime.Spec.ResolverSpec.Prometheus) + + setDefaultImage(&cbContainersRuntime.Spec.ResolverSpec.Image, "cbartifactory/runtime-kubernetes-resolver") + + if err := setDefaultResourceRequirements(&cbContainersRuntime.Spec.ResolverSpec.Resources, "64Mi", "200m", "128Mi", "600m"); err != nil { + return err + } + + setDefaultHTTPProbes(&cbContainersRuntime.Spec.ResolverSpec.Probes) + + if cbContainersRuntime.Spec.SensorSpec.Labels == nil { + cbContainersRuntime.Spec.SensorSpec.Labels = make(map[string]string) + } + + if cbContainersRuntime.Spec.SensorSpec.DaemonSetAnnotations == nil { + cbContainersRuntime.Spec.SensorSpec.DaemonSetAnnotations = make(map[string]string) + } + + if cbContainersRuntime.Spec.SensorSpec.PodTemplateAnnotations == nil { + cbContainersRuntime.Spec.SensorSpec.PodTemplateAnnotations = make(map[string]string) + } + + if cbContainersRuntime.Spec.SensorSpec.Env == nil { + cbContainersRuntime.Spec.SensorSpec.Env = make(map[string]string) + } + + setDefaultPrometheus(&cbContainersRuntime.Spec.SensorSpec.Prometheus) + + setDefaultImage(&cbContainersRuntime.Spec.SensorSpec.Image, "cbartifactory/runtime-kubernetes-sensor") + + if err := setDefaultResourceRequirements(&cbContainersRuntime.Spec.SensorSpec.Resources, "64Mi", "30m", "256Mi", "200m"); err != nil { + return err + } + + setDefaultFileProbes(&cbContainersRuntime.Spec.SensorSpec.Probes) + + if cbContainersRuntime.Spec.SensorSpec.VerbosityLevel == nil { + defaultVerbosity := 2 + cbContainersRuntime.Spec.SensorSpec.VerbosityLevel = &defaultVerbosity + } + + if cbContainersRuntime.Spec.InternalGrpcPort == 0 { + cbContainersRuntime.Spec.InternalGrpcPort = 443 + } + + return nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *CBContainersRuntimeReconciler) SetupWithManager(mgr ctrl.Manager) error { + return ctrl.NewControllerManagedBy(mgr). + For(&operatorcontainerscarbonblackiov1.CBContainersRuntime{}). + Owns(&appsV1.Deployment{}). + Owns(&coreV1.Service{}). + Owns(&appsV1.DaemonSet{}). + Complete(r) +} diff --git a/controllers/cbcontainersruntime_controller_test.go b/controllers/cbcontainersruntime_controller_test.go new file mode 100644 index 00000000..5f29d9d1 --- /dev/null +++ b/controllers/cbcontainersruntime_controller_test.go @@ -0,0 +1,127 @@ +package controllers_test + +import ( + "context" + "fmt" + "testing" + + logrTesting "github.com/go-logr/logr/testing" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + "github.com/vmware/cbcontainers-operator/cbcontainers/test_utils" + testUtilsMocks "github.com/vmware/cbcontainers-operator/cbcontainers/test_utils/mocks" + "github.com/vmware/cbcontainers-operator/controllers" + "github.com/vmware/cbcontainers-operator/controllers/mocks" + "k8s.io/apimachinery/pkg/runtime" + ctrlRuntime "sigs.k8s.io/controller-runtime" +) + +type SetupRuntimeControllerTest func(*RuntimeControllerTestMocks) + +type RuntimeControllerTestMocks struct { + client *testUtilsMocks.MockClient + RuntimeStateApplier *mocks.MockRuntimeStateApplier + ctx context.Context +} + +var ( + RuntimeVersion = test_utils.RandomString() + + RuntimeCustomResourceItems = []cbcontainersv1.CBContainersRuntime{ + { + Spec: cbcontainersv1.CBContainersRuntimeSpec{Version: RuntimeVersion}, + }, + } +) + +func testCBContainersRuntimeController(t *testing.T, setups ...SetupRuntimeControllerTest) (ctrlRuntime.Result, error) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + mocksObjects := &RuntimeControllerTestMocks{ + ctx: context.TODO(), + client: testUtilsMocks.NewMockClient(ctrl), + RuntimeStateApplier: mocks.NewMockRuntimeStateApplier(ctrl), + } + + for _, setup := range setups { + setup(mocksObjects) + } + + controller := &controllers.CBContainersRuntimeReconciler{ + Client: mocksObjects.client, + Log: &logrTesting.TestLogger{T: t}, + Scheme: &runtime.Scheme{}, + + RuntimeStateApplier: mocksObjects.RuntimeStateApplier, + } + + return controller.Reconcile(mocksObjects.ctx, ctrlRuntime.Request{}) +} + +func setupRuntimeCustomResource(testMocks *RuntimeControllerTestMocks) { + testMocks.client.EXPECT().List(testMocks.ctx, &cbcontainersv1.CBContainersRuntimeList{}). + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersRuntimeList, _ ...interface{}) { + list.Items = RuntimeCustomResourceItems + }). + Return(nil) +} + +func TestListRuntimeResourcesErrorShouldReturnError(t *testing.T) { + _, err := testCBContainersRuntimeController(t, func(testMocks *RuntimeControllerTestMocks) { + testMocks.client.EXPECT().List(testMocks.ctx, &cbcontainersv1.CBContainersRuntimeList{}).Return(fmt.Errorf("")) + }) + + require.Error(t, err) +} + +func TestNotFindingAnyRuntimeResourceShouldReturnNil(t *testing.T) { + result, err := testCBContainersRuntimeController(t, func(testMocks *RuntimeControllerTestMocks) { + testMocks.client.EXPECT().List(testMocks.ctx, &cbcontainersv1.CBContainersRuntimeList{}).Return(nil) + }) + + require.NoError(t, err) + require.Equal(t, result, ctrlRuntime.Result{}) +} + +func TestFindingMoreThanOneRuntimeResourceShouldReturnError(t *testing.T) { + _, err := testCBContainersRuntimeController(t, func(testMocks *RuntimeControllerTestMocks) { + testMocks.client.EXPECT().List(testMocks.ctx, &cbcontainersv1.CBContainersRuntimeList{}). + Do(func(ctx context.Context, list *cbcontainersv1.CBContainersRuntimeList, _ ...interface{}) { + list.Items = append(list.Items, cbcontainersv1.CBContainersRuntime{}) + list.Items = append(list.Items, cbcontainersv1.CBContainersRuntime{}) + }). + Return(nil) + }) + + require.Error(t, err) +} + +func TestRuntimeReconcile(t *testing.T) { + t.Run("When state applier returns error, reconcile should return error", func(t *testing.T) { + _, err := testCBContainersRuntimeController(t, setupRuntimeCustomResource, func(testMocks *RuntimeControllerTestMocks) { + testMocks.RuntimeStateApplier.EXPECT().ApplyDesiredState(testMocks.ctx, &RuntimeCustomResourceItems[0], testMocks.client, gomock.Any()).Return(false, fmt.Errorf("")) + }) + + require.Error(t, err) + }) + + t.Run("When state applier returns state was changed, reconcile should return Requeue true", func(t *testing.T) { + result, err := testCBContainersRuntimeController(t, setupRuntimeCustomResource, func(testMocks *RuntimeControllerTestMocks) { + testMocks.RuntimeStateApplier.EXPECT().ApplyDesiredState(testMocks.ctx, &RuntimeCustomResourceItems[0], testMocks.client, gomock.Any()).Return(true, nil) + }) + + require.NoError(t, err) + require.Equal(t, result, ctrlRuntime.Result{Requeue: true}) + }) + + t.Run("When state applier returns state was not changed, reconcile should return default Requeue", func(t *testing.T) { + result, err := testCBContainersRuntimeController(t, setupRuntimeCustomResource, func(testMocks *RuntimeControllerTestMocks) { + testMocks.RuntimeStateApplier.EXPECT().ApplyDesiredState(testMocks.ctx, &RuntimeCustomResourceItems[0], testMocks.client, gomock.Any()).Return(false, nil) + }) + + require.NoError(t, err) + require.Equal(t, result, ctrlRuntime.Result{}) + }) +} diff --git a/controllers/defaults.go b/controllers/defaults.go new file mode 100644 index 00000000..4c370b65 --- /dev/null +++ b/controllers/defaults.go @@ -0,0 +1,132 @@ +package controllers + +import ( + "github.com/vmware/cbcontainers-operator/api/v1" + coreV1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" +) + +const ( + defaultAccessToken = "cbcontainers-access-token" +) + +var falseRef = false + +func setDefaultProbesCommon(probesSpec *v1.CBContainersCommonProbesSpec) { + if probesSpec.InitialDelaySeconds == 0 { + probesSpec.InitialDelaySeconds = 3 + } + + if probesSpec.TimeoutSeconds == 0 { + probesSpec.TimeoutSeconds = 1 + } + + if probesSpec.PeriodSeconds == 0 { + probesSpec.PeriodSeconds = 30 + } + + if probesSpec.SuccessThreshold == 0 { + probesSpec.SuccessThreshold = 1 + } + + if probesSpec.FailureThreshold == 0 { + probesSpec.FailureThreshold = 3 + } +} + +func setDefaultHTTPProbes(probesSpec *v1.CBContainersHTTPProbesSpec) { + if probesSpec.ReadinessPath == "" { + probesSpec.ReadinessPath = "/ready" + } + + if probesSpec.LivenessPath == "" { + probesSpec.LivenessPath = "/alive" + } + + if probesSpec.Port == 0 { + probesSpec.Port = 8181 + } + + if probesSpec.Scheme == "" { + probesSpec.Scheme = coreV1.URISchemeHTTP + } + + setDefaultProbesCommon(&probesSpec.CBContainersCommonProbesSpec) +} + +func setDefaultFileProbes(probesSpec *v1.CBContainersFileProbesSpec) { + if probesSpec.ReadinessPath == "" { + probesSpec.ReadinessPath = "/tmp/ready" + } + + if probesSpec.LivenessPath == "" { + probesSpec.LivenessPath = "/tmp/alive" + } + + setDefaultProbesCommon(&probesSpec.CBContainersCommonProbesSpec) +} + +func setDefaultPrometheus(prometheusSpec *v1.CBContainersPrometheusSpec) { + if prometheusSpec.Enabled == nil { + prometheusSpec.Enabled = &falseRef + } + + if prometheusSpec.Port == 0 { + prometheusSpec.Port = 7071 + } +} + +func setDefaultImage(imageSpec *v1.CBContainersImageSpec, imageName string) { + if imageSpec.Repository == "" { + imageSpec.Repository = imageName + } + + if imageSpec.PullPolicy == "" { + imageSpec.PullPolicy = "Always" + } +} + +func setDefaultResourceRequirements(resources *coreV1.ResourceRequirements, requestMemory, requestCpu, limitMemory, limitCpu string) error { + if resources.Requests == nil { + resources.Requests = make(coreV1.ResourceList) + } + + if err := setDefaultsResourcesList(resources.Requests, requestMemory, requestCpu); err != nil { + return err + } + + if resources.Limits == nil { + resources.Limits = make(coreV1.ResourceList) + } + + if err := setDefaultsResourcesList(resources.Limits, limitMemory, limitCpu); err != nil { + return err + } + + return nil +} + +func setDefaultsResourcesList(list coreV1.ResourceList, memory, cpu string) error { + if err := setDefaultResource(list, coreV1.ResourceMemory, memory); err != nil { + return err + } + + if err := setDefaultResource(list, coreV1.ResourceCPU, cpu); err != nil { + return err + } + + return nil +} + +func setDefaultResource(list coreV1.ResourceList, resourceName coreV1.ResourceName, value string) error { + if _, ok := list[resourceName]; !ok { + quantity, err := resource.ParseQuantity(value) + if err != nil { + return err + } + + list[resourceName] = quantity + } + + return nil +} diff --git a/controllers/mocks/generated.go b/controllers/mocks/generated.go index f20d91bf..90f59a7a 100644 --- a/controllers/mocks/generated.go +++ b/controllers/mocks/generated.go @@ -3,3 +3,4 @@ package mocks //go:generate mockgen -destination mock_cluster_state_applier.go -package mocks github.com/vmware/cbcontainers-operator/controllers ClusterStateApplier //go:generate mockgen -destination mock_cluster_processor.go -package mocks github.com/vmware/cbcontainers-operator/controllers ClusterProcessor //go:generate mockgen -destination mock_hardening_state_applier.go -package mocks github.com/vmware/cbcontainers-operator/controllers HardeningStateApplier +//go:generate mockgen -destination mock_runtime_state_applier.go -package mocks github.com/vmware/cbcontainers-operator/controllers RuntimeStateApplier diff --git a/controllers/mocks/mock_runtime_state_applier.go b/controllers/mocks/mock_runtime_state_applier.go new file mode 100644 index 00000000..25289085 --- /dev/null +++ b/controllers/mocks/mock_runtime_state_applier.go @@ -0,0 +1,53 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/vmware/cbcontainers-operator/controllers (interfaces: RuntimeStateApplier) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + v1 "github.com/vmware/cbcontainers-operator/api/v1" + options "github.com/vmware/cbcontainers-operator/cbcontainers/state/applyment/options" + client "sigs.k8s.io/controller-runtime/pkg/client" +) + +// MockRuntimeStateApplier is a mock of RuntimeStateApplier interface. +type MockRuntimeStateApplier struct { + ctrl *gomock.Controller + recorder *MockRuntimeStateApplierMockRecorder +} + +// MockRuntimeStateApplierMockRecorder is the mock recorder for MockRuntimeStateApplier. +type MockRuntimeStateApplierMockRecorder struct { + mock *MockRuntimeStateApplier +} + +// NewMockRuntimeStateApplier creates a new mock instance. +func NewMockRuntimeStateApplier(ctrl *gomock.Controller) *MockRuntimeStateApplier { + mock := &MockRuntimeStateApplier{ctrl: ctrl} + mock.recorder = &MockRuntimeStateApplierMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRuntimeStateApplier) EXPECT() *MockRuntimeStateApplierMockRecorder { + return m.recorder +} + +// ApplyDesiredState mocks base method. +func (m *MockRuntimeStateApplier) ApplyDesiredState(arg0 context.Context, arg1 *v1.CBContainersRuntime, arg2 client.Client, arg3 options.OwnerSetter) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ApplyDesiredState", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ApplyDesiredState indicates an expected call of ApplyDesiredState. +func (mr *MockRuntimeStateApplierMockRecorder) ApplyDesiredState(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ApplyDesiredState", reflect.TypeOf((*MockRuntimeStateApplier)(nil).ApplyDesiredState), arg0, arg1, arg2, arg3) +} diff --git a/go.mod b/go.mod index 7eb1940b..a07079d5 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/cloudflare/cfssl v1.4.1 github.com/go-logr/logr v0.4.0 github.com/go-resty/resty/v2 v2.5.0 - github.com/golang/mock v1.5.0 + github.com/golang/mock v1.6.0 github.com/golang/protobuf v1.5.2 github.com/stretchr/testify v1.7.0 google.golang.org/grpc v1.36.1 diff --git a/go.sum b/go.sum index 2cb1e39e..5fd71e39 100644 --- a/go.sum +++ b/go.sum @@ -164,8 +164,8 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.5.0 h1:jlYHihg//f7RRwuPfptm04yp4s7O6Kw8EZiVYIGcH0g= -github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -411,6 +411,7 @@ github.com/weppos/publicsuffix-go v0.5.0 h1:rutRtjBJViU/YjcI5d80t4JAVvDltS6bciJg github.com/weppos/publicsuffix-go v0.5.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= github.com/zmap/rc2 v0.0.0-20131011165748-24b9757f5521/go.mod h1:3YZ9o3WnatTIZhuOtot4IcUfzoKVjUHqu6WALIyI0nE= github.com/zmap/zcertificate v0.0.0-20180516150559-0e3d58b1bac4/go.mod h1:5iU54tB79AMBcySS0R2XIyZBAVmeHranShAFELYx7is= @@ -482,8 +483,9 @@ golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -512,8 +514,9 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4 h1:4nGaVu0QrbjT/AK2PRLuQfQuh6DJve+pELhqTdAj3x0= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -526,6 +529,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -562,8 +566,10 @@ golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -618,8 +624,9 @@ golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200616133436-c1934b75d054 h1:HHeAlu5H9b71C+Fx0K+1dGgVFN1DM1/wz4aoGOA5qS8= golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.1.1 h1:wGiQel/hW0NnEkJUk8lbzkX2gFJU6PFxf1v5OlCfuOs= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/main.go b/main.go index eeb48a32..f7ded393 100644 --- a/main.go +++ b/main.go @@ -20,10 +20,11 @@ import ( "context" "flag" "fmt" + "os" + "github.com/vmware/cbcontainers-operator/cbcontainers/monitor" commonState "github.com/vmware/cbcontainers-operator/cbcontainers/state/common" coreV1 "k8s.io/api/core/v1" - "os" clusterProcessors "github.com/vmware/cbcontainers-operator/cbcontainers/processors/cluster" @@ -38,9 +39,10 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" - cbcontainersv1 "github.com/vmware/cbcontainers-operator/api/v1" + operatorcontainerscarbonblackiov1 "github.com/vmware/cbcontainers-operator/api/v1" clusterState "github.com/vmware/cbcontainers-operator/cbcontainers/state/cluster" hardeningState "github.com/vmware/cbcontainers-operator/cbcontainers/state/hardening" + runtimeState "github.com/vmware/cbcontainers-operator/cbcontainers/state/runtime" certificatesUtils "github.com/vmware/cbcontainers-operator/cbcontainers/utils/certificates" "github.com/vmware/cbcontainers-operator/controllers" // +kubebuilder:scaffold:imports @@ -54,7 +56,7 @@ var ( func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) - utilruntime.Must(cbcontainersv1.AddToScheme(scheme)) + utilruntime.Must(operatorcontainerscarbonblackiov1.AddToScheme(scheme)) // +kubebuilder:scaffold:scheme } @@ -126,6 +128,17 @@ func main() { setupLog.Error(err, "unable to create controller", "controller", "CBContainersHardening") os.Exit(1) } + + cbContainersRuntimeLogger := ctrl.Log.WithName("controllers").WithName("CBContainersRuntime") + if err = (&controllers.CBContainersRuntimeReconciler{ + Client: mgr.GetClient(), + Log: cbContainersRuntimeLogger, + Scheme: mgr.GetScheme(), + RuntimeStateApplier: runtimeState.NewRuntimeStateApplier(cbContainersRuntimeLogger, runtimeState.NewDefaultRuntimeChildK8sObjectApplier()), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "CBContainersRuntime") + os.Exit(1) + } // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("health", healthz.Ping); err != nil {