Add support for Kubernetes 1.22 with backwards compatibility (#49)

* Initial adapter for validating webhook

* Extended unit tests to validate proper type is used

* Changed project structure to support v1 and v1beta1 apiextensions for CRDs

* Removed install and uninstall make targets

* Restored and fixed install/uninstall targets and merged manifests with the fix for empty object default. Split the CRDs in two folders. Fixed a bug with an optional value that had required children in runtime.

* Add some explanation to readmes for defaults and api versions

* Remove monitor from operator and its uses

* Fix api version in hardening controller as well. Added samples for 2 CRDs for easier debugging.

* Fixed an endless reconcile loop due to admission rules being overriden. Added delve config for debugging.

* Fixed CR comment

* Removed obsolete interface

* Changed SetWebhooks() so that we can modify the objects via pointer references and not have to set them every time...this avoids constant reconcile if kubernetes re-orders the items

* Fix runtime file generation (from rebase on main)

Makefile

@@ -16,8 +16,16 @@ BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL)
 IMG ?= controller:latest
 # Image URL to use all building/pushing image targets
 OPERATOR_REPLICAS ?= 1
-# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
-CRD_OPTIONS ?= "crd:trivialVersions=true,crdVersions=v1beta1"
+
+CRD_OPTIONS ?= "crd:crdVersions=v1"
+# Produce CRDs that work back to Kubernetes 1.11 (no version conversion) - supported in v1beta1 only
+CRD_OPTIONS_V1BETA1 ?= "crd:trivialVersions=true,crdVersions=v1beta1"
+
+PATH_TO_RELEASE := config/default
+PATH_TO_RELEASE := $(if $(findstring v1beta1, $(CRD_VERSION)), $(PATH_TO_RELEASE)_v1beta1, $(PATH_TO_RELEASE))
+
+PATH_TO_CRDS := config/crd
+PATH_TO_CRDS := $(if $(findstring v1beta1, $(CRD_VERSION)), $(PATH_TO_CRDS)_v1beta1, $(PATH_TO_CRDS))
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -37,6 +45,7 @@ all: build
 OS = $(shell go env GOOS)
 ARCH = $(shell go env GOARCH)
 
+
 # Run tests
 # Set default shell as bash
 SHELL := /bin/bash
@@ -54,17 +63,24 @@ manager: generate fmt vet
 run: generate fmt vet manifests
 	go run ./main.go
 
+# Run with Delve for development purposes against the configured Kubernetes cluster in ~/.kube/config
+# Delve is a debugger for the Go programming language. More info: https://github.com/go-delve/delve
+# Note: use kill -SIGINT $pid to stop delve if it hangs
+run-delve: generate fmt vet manifests
+	go build -gcflags "all=-trimpath=$(shell go env GOPATH) -N -l" -o bin/manager main.go
+	dlv --listen=:2345 --headless=true --api-version=2 --accept-multiclient exec ./bin/manager
+
 install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) build config/crd | kubectl apply -f -
+	$(KUSTOMIZE) build $(PATH_TO_CRDS) | kubectl apply -f -
 
 uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
-	$(KUSTOMIZE) build config/crd | kubectl delete -f -
+	$(KUSTOMIZE) build $(PATH_TO_CRDS) | kubectl delete -f -
 
-# Deploy controller in the configured Kubernetes cluster in ~/.kube/config
+# Generate and bundle all operator components in a single YAML file
 create_operator_spec: manifests kustomize
 	rm -f operator.yaml
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} && $(KUSTOMIZE) edit set replicas operator=${OPERATOR_REPLICAS}
-	- $(KUSTOMIZE) build config/default >> operator.yaml
+	- $(KUSTOMIZE) build $(PATH_TO_RELEASE) >> operator.yaml
 	git restore config/manager/kustomization.yaml
 
 # Deploy controller in the configured Kubernetes cluster in ~/.kube/config
@@ -80,16 +96,15 @@ undeploy: create_operator_spec
 # Generate manifests e.g. CRD, RBAC etc.
 manifests: controller-gen
 	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
-
-# Generate manifests e.g. CRD, RBAC etc.
-manifests_with_defaults: controller-gen
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+# This is needed since controller-gen does not support empty object/maps like {} but they are helpful to propagate default values up from nested objects
 	for filename in $$(ls config/crd/bases) ; do \
-        XT=config/crd/bases/$$filename ; \
-        sed 's/default: <>/default: {}/g' $$XT >> $$XT.temp ; \
-        rm -f $$XT ; \
-        mv $$XT.temp $$XT ; \
-    done
+		XT=config/crd/bases/$$filename ; \
+		sed 's/default: <>/default: {}/g' $$XT >> $$XT.temp ; \
+		rm -f $$XT ; \
+		mv $$XT.temp $$XT ; \
+	done
+# The above modification is not needed for v1beta1 as defaults are not supported there at all
+	$(CONTROLLER_GEN) $(CRD_OPTIONS_V1BETA1) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd_v1beta1/bases
 
 # Run go fmt against code
 fmt:

PROJECT

@@ -7,21 +7,21 @@ projectName: cbcontainers
 repo: github.com/vmware/cbcontainers-operator
 resources:
 - api:
-    crdVersion: v1beta1
+    crdVersion: v1
   controller: true
   domain: operator.containers.carbonblack.io
   kind: CBContainersCluster
   path: github.com/vmware/cbcontainers-operator/api/v1
   version: v1
 - api:
-    crdVersion: v1beta1
+    crdVersion: v1
   controller: true
   domain: operator.containers.carbonblack.io
   kind: CBContainersHardening
   path: github.com/vmware/cbcontainers-operator/api/v1
   version: v1
 - api:
-    crdVersion: v1beta1
+    crdVersion: v1
   controller: true
   domain: operator.containers.carbonblack.io
   kind: CBContainersRuntime

README.md

@@ -15,7 +15,10 @@ The Carbon Black Cloud Container Operator utilizes the operator-framework to cre
 ## Operator Deployment
 
 ### Prerequisites
-Kubernetes 1.13+ 
+Kubernetes 1.13+ is supported.
+
+By default, the operator utilizes CustomResourceDefinitions v1, which requires Kubernetes 1.16+.
+Deploying an operator with CustomResourceDefinitions v1beta1 (deprecated in Kubernetes 1.16, removed in Kubernetes 1.22) can be done - see the relevant section below.
 
 ### Create the operator image
 ```
@@ -28,7 +31,7 @@ make docker-build docker-push IMG={IMAGE_NAME}
 make deploy IMG={IMAGE_NAME}
 ```
 
-* View [Developer Guide](docs/developers.md#deploying-the-operator-without-using-an-image) to see how deploy the operator without using an image
+* View [Developer Guide](docs/developers.md) to see how deploy the operator without using an image
 
 ## Data Plane Deployment
 
@@ -185,3 +188,18 @@ Finding the API-server IP:
 ```sh
 kubectl -n default get service kubernetes -o=jsonpath='{..clusterIP}'
 ```
+
+## Utilizing v1beta1 CustomResourceDefinition versions
+The operator supports Kubernetes clusters from v1.13+. 
+The CustomResourceDefinition APIs were in beta stage in those cluster and were later promoted to GA in v1.16. They are no longer served as of v1.22 of Kubernetes.
+
+To maintain compatibility, this operator offers 2 sets of CustomResoruceDefinitions - one under the `apiextensions/v1beta1` API and one under `apiextensons/v1`.
+
+By default, all operations in the repository like `deploy` or `install` work with the v1 version of the `apiextensions` API. Utilizing `v1beta1` is supported by passing the `CRD_VERSION=v1beta1` option when running make.
+Note that both `apiextensions/v1` and `apiextensions/v1beta1` versions of the CRDs are generated and maintained by `make` - only commands that use the final output work with 1 version at a time.
+
+For example, this command will deploy the operator resources on the current cluster but utilizing the `apiextensions/v1beta1` API version for them.
+
+```
+make deploy CRD_VERSION=v1beta1
+```

api/v1/cbcontainersruntime_types.go

@@ -68,9 +68,8 @@ type CBContainersRuntimeSensorSpec struct {
 type CBContainersRuntimeSpec struct {
 	Version string `json:"version,required"`
 	// +kubebuilder:default:="cbcontainers-access-token"
-	AccessTokenSecretName string `json:"accessTokenSecretName,omitempty"`
-	// +kubebuilder:default:=<>
-	ResolverSpec CBContainersRuntimeResolverSpec `json:"resolverSpec,omitempty"`
+	AccessTokenSecretName string                          `json:"accessTokenSecretName,omitempty"`
+	ResolverSpec          CBContainersRuntimeResolverSpec `json:"resolverSpec,omitempty"`
 	// +kubebuilder:default:=<>
 	SensorSpec CBContainersRuntimeSensorSpec `json:"sensorSpec,omitempty"`
 	// +kubebuilder:default:=443