From d5832c8f83655646e1986d94c867d6f5f7a2d03f Mon Sep 17 00:00:00 2001 From: "Weber.Yang" Date: Mon, 3 Jul 2023 14:11:54 +0800 Subject: [PATCH] SKS-1479: Add ElfMachineMutation webhhok to set cape version to ElfMachine (#125) --- Makefile | 6 +- PROJECT | 8 ++ api/v1beta1/consts.go | 3 + config/certmanager/certificate.yaml | 39 ++++++++++ config/certmanager/kustomization.yaml | 5 ++ config/certmanager/kustomizeconfig.yaml | 16 ++++ config/default/kustomization.yaml | 60 +++++++-------- config/default/manager_webhook_patch.yaml | 23 ++++++ config/default/webhookcainjection_patch.yaml | 29 ++++++++ config/webhook/kustomization.yaml | 6 ++ config/webhook/kustomizeconfig.yaml | 25 +++++++ config/webhook/manifests.yaml | 25 +++++++ config/webhook/service.yaml | 20 +++++ .../elfmachine_controller_placement_group.go | 13 +++- controllers/elfmachine_controller_test.go | 15 ++++ go.mod | 2 +- hack/version.sh | 5 +- main.go | 27 +++++-- pkg/manager/constants.go | 4 + pkg/util/annotations/helpers.go | 6 ++ pkg/version/consts.go | 31 ++++++++ pkg/version/util.go | 67 +++++++++++++++++ pkg/version/util_test.go | 70 ++++++++++++++++++ {version => pkg/version}/version.go | 30 +++++++- test/fake/types.go | 52 +++++++------ test/helpers/envtest.go | 8 ++ webhooks/elfmachine_webhook_mutation.go | 73 +++++++++++++++++++ webhooks/elfmachine_webhook_mutation_test.go | 17 +++++ 28 files changed, 619 insertions(+), 66 deletions(-) create mode 100644 config/certmanager/certificate.yaml create mode 100644 config/certmanager/kustomization.yaml create mode 100644 config/certmanager/kustomizeconfig.yaml create mode 100644 config/default/manager_webhook_patch.yaml create mode 100644 config/default/webhookcainjection_patch.yaml create mode 100644 config/webhook/kustomization.yaml create mode 100644 config/webhook/kustomizeconfig.yaml create mode 100644 config/webhook/manifests.yaml create mode 100644 config/webhook/service.yaml create mode 100644 pkg/version/consts.go create mode 100644 pkg/version/util.go create mode 100644 pkg/version/util_test.go rename {version => pkg/version}/version.go (69%) create mode 100644 webhooks/elfmachine_webhook_mutation.go create mode 100644 webhooks/elfmachine_webhook_mutation_test.go diff --git a/Makefile b/Makefile index eafc5b66..6f3cca34 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ ARCH ?= amd64 ALL_ARCH = amd64 arm64 # Set build time variables including version details -LDFLAGS := $(shell hack/version.sh) +LDFLAGS := $(shell hack/version.sh $(IMAGE_TAG)) # Common docker variables IMAGE_NAME ?= cape-manager @@ -123,7 +123,7 @@ e2e-templates: kustomize ## Generate e2e cluster templates $(KUSTOMIZE) build $(E2E_TEMPLATE_DIR)/kustomization/conformance > $(E2E_TEMPLATE_DIR)/cluster-template-conformance.yaml test: generate ## Run tests. - source ./hack/fetch_ext_bins.sh; fetch_tools; setup_envs; go test -v ./api/... ./controllers/... ./pkg/... -coverprofile=cover.out + source ./hack/fetch_ext_bins.sh; fetch_tools; setup_envs; go test -v ./api/... ./webhooks/... ./controllers/... ./pkg/... -coverprofile=cover.out .PHONY: e2e-image e2e-image: docker-pull-prerequisites ## Build the e2e manager image. Docker ignores docker.io causing CAPI to fail to load local e2e image. @@ -190,12 +190,14 @@ generate-go: controller-gen ## Runs Go related generate targets go generate ./... $(CONTROLLER_GEN) \ paths=./api/... \ + paths=./webhooks/... \ object:headerFile=./hack/boilerplate.go.txt .PHONY: generate-manifests generate-manifests: controller-gen ## Generate manifests e.g. CRD, RBAC etc. $(CONTROLLER_GEN) \ paths=./api/... \ + paths=./webhooks/... \ crd:crdVersions=v1 \ output:crd:dir=$(CRD_ROOT) \ webhook diff --git a/PROJECT b/PROJECT index 132d2ca9..8263dd52 100644 --- a/PROJECT +++ b/PROJECT @@ -1,3 +1,7 @@ +# Code generated by tool. DO NOT EDIT. +# This file is used to track the info used to scaffold your project +# and allow the plugins properly work. +# More info: https://book.kubebuilder.io/reference/project-config.html domain: cluster.x-k8s.io layout: - go.kubebuilder.io/v3 @@ -22,6 +26,10 @@ resources: kind: ElfMachine path: github.com/smartxworks/cluster-api-provider-elf/api/v1beta1 version: v1beta1 + webhooks: + defaulting: true + validation: true + webhookVersion: v1 - api: crdVersion: v1 namespaced: true diff --git a/api/v1beta1/consts.go b/api/v1beta1/consts.go index c1dff6b0..e65e0a02 100644 --- a/api/v1beta1/consts.go +++ b/api/v1beta1/consts.go @@ -30,6 +30,9 @@ const ( const ( // PlacementGroupNameAnnotation is the annotation identifying the name of placement group. PlacementGroupNameAnnotation = "cape.infrastructure.cluster.x-k8s.io/placement-group-name" + + // CAPEVersionAnnotation is the annotation identifying the version of CAPE that the resource reconciled by. + CAPEVersionAnnotation = "cape.infrastructure.cluster.x-k8s.io/cape-version" ) // Labels. diff --git a/config/certmanager/certificate.yaml b/config/certmanager/certificate.yaml new file mode 100644 index 00000000..55cdbeff --- /dev/null +++ b/config/certmanager/certificate.yaml @@ -0,0 +1,39 @@ +# The following manifests contain a self-signed issuer CR and a certificate CR. +# More document can be found at https://docs.cert-manager.io +# WARNING: Targets CertManager v1.0. Check https://cert-manager.io/docs/installation/upgrading/ for breaking changes. +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/name: issuer + app.kubernetes.io/instance: selfsigned-issuer + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: cluster-api-provider-elf + app.kubernetes.io/part-of: cluster-api-provider-elf + app.kubernetes.io/managed-by: kustomize + name: selfsigned-issuer + namespace: system +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/name: certificate + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: cluster-api-provider-elf + app.kubernetes.io/part-of: cluster-api-provider-elf + app.kubernetes.io/managed-by: kustomize + name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml + namespace: system +spec: + # $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize + dnsNames: + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc + - $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local + issuerRef: + kind: Issuer + name: selfsigned-issuer + secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize diff --git a/config/certmanager/kustomization.yaml b/config/certmanager/kustomization.yaml new file mode 100644 index 00000000..bebea5a5 --- /dev/null +++ b/config/certmanager/kustomization.yaml @@ -0,0 +1,5 @@ +resources: +- certificate.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/certmanager/kustomizeconfig.yaml b/config/certmanager/kustomizeconfig.yaml new file mode 100644 index 00000000..e631f777 --- /dev/null +++ b/config/certmanager/kustomizeconfig.yaml @@ -0,0 +1,16 @@ +# This configuration is for teaching kustomize how to update name ref and var substitution +nameReference: +- kind: Issuer + group: cert-manager.io + fieldSpecs: + - kind: Certificate + group: cert-manager.io + path: spec/issuerRef/name + +varReference: +- kind: Certificate + group: cert-manager.io + path: spec/commonName +- kind: Certificate + group: cert-manager.io + path: spec/dnsNames diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index be906c63..12bdf0ed 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -19,9 +19,9 @@ bases: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml -#- ../webhook +- ../webhook # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. -#- ../certmanager +- ../certmanager # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus @@ -38,39 +38,39 @@ patchesStrategicMerge: # [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in # crd/kustomization.yaml -#- manager_webhook_patch.yaml + - manager_webhook_patch.yaml # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks. # 'CERTMANAGER' needs to be enabled to use ca injection -#- webhookcainjection_patch.yaml + - webhookcainjection_patch.yaml # the following config is for teaching kustomize how to do var substitution vars: # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix. -#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -# fieldref: -# fieldpath: metadata.namespace -#- name: CERTIFICATE_NAME -# objref: -# kind: Certificate -# group: cert-manager.io -# version: v1 -# name: serving-cert # this name should match the one in certificate.yaml -#- name: SERVICE_NAMESPACE # namespace of the service -# objref: -# kind: Service -# version: v1 -# name: webhook-service -# fieldref: -# fieldpath: metadata.namespace -#- name: SERVICE_NAME -# objref: -# kind: Service -# version: v1 -# name: webhook-service +- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml + fieldref: + fieldpath: metadata.namespace +- name: CERTIFICATE_NAME + objref: + kind: Certificate + group: cert-manager.io + version: v1 + name: serving-cert # this name should match the one in certificate.yaml +- name: SERVICE_NAMESPACE # namespace of the service + objref: + kind: Service + version: v1 + name: webhook-service + fieldref: + fieldpath: metadata.namespace +- name: SERVICE_NAME + objref: + kind: Service + version: v1 + name: webhook-service diff --git a/config/default/manager_webhook_patch.yaml b/config/default/manager_webhook_patch.yaml new file mode 100644 index 00000000..738de350 --- /dev/null +++ b/config/default/manager_webhook_patch.yaml @@ -0,0 +1,23 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: controller-manager + namespace: system +spec: + template: + spec: + containers: + - name: manager + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert diff --git a/config/default/webhookcainjection_patch.yaml b/config/default/webhookcainjection_patch.yaml new file mode 100644 index 00000000..e334e162 --- /dev/null +++ b/config/default/webhookcainjection_patch.yaml @@ -0,0 +1,29 @@ +# This patch add annotation to admission webhook config and +# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize. +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/name: mutatingwebhookconfiguration + app.kubernetes.io/instance: mutating-webhook-configuration + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: cluster-api-provider-elf + app.kubernetes.io/part-of: cluster-api-provider-elf + app.kubernetes.io/managed-by: kustomize + name: mutating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) +#! --- +#! apiVersion: admissionregistration.k8s.io/v1 +#! kind: ValidatingWebhookConfiguration +#! metadata: +#! labels: +#! app.kubernetes.io/name: validatingwebhookconfiguration +#! app.kubernetes.io/instance: validating-webhook-configuration +#! app.kubernetes.io/component: webhook +#! app.kubernetes.io/created-by: cluster-api-provider-elf +#! app.kubernetes.io/part-of: cluster-api-provider-elf +#! app.kubernetes.io/managed-by: kustomize +#! name: validating-webhook-configuration +#! annotations: +#! cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME) diff --git a/config/webhook/kustomization.yaml b/config/webhook/kustomization.yaml new file mode 100644 index 00000000..9cf26134 --- /dev/null +++ b/config/webhook/kustomization.yaml @@ -0,0 +1,6 @@ +resources: +- manifests.yaml +- service.yaml + +configurations: +- kustomizeconfig.yaml diff --git a/config/webhook/kustomizeconfig.yaml b/config/webhook/kustomizeconfig.yaml new file mode 100644 index 00000000..310c4817 --- /dev/null +++ b/config/webhook/kustomizeconfig.yaml @@ -0,0 +1,25 @@ +# the following config is for teaching kustomize where to look at when substituting vars. +# It requires kustomize v2.1.0 or newer to work properly. +nameReference: +- kind: Service + version: v1 + fieldSpecs: + - kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/name + #! - kind: ValidatingWebhookConfiguration + #! group: admissionregistration.k8s.io + #! path: webhooks/clientConfig/service/name + +namespace: +- kind: MutatingWebhookConfiguration + group: admissionregistration.k8s.io + path: webhooks/clientConfig/service/namespace + create: true +#! - kind: ValidatingWebhookConfiguration +#! group: admissionregistration.k8s.io +#! path: webhooks/clientConfig/service/namespace +#! create: true + +varReference: +- path: metadata/annotations diff --git a/config/webhook/manifests.yaml b/config/webhook/manifests.yaml new file mode 100644 index 00000000..bbab1a91 --- /dev/null +++ b/config/webhook/manifests.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: mutating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: webhook-service + namespace: system + path: /mutate-infrastructure-cluster-x-k8s-io-v1beta1-elfmachine + failurePolicy: Fail + name: mutation.elfmachine.infrastructure.x-k8s.io + rules: + - apiGroups: + - infrastructure.cluster.x-k8s.io + apiVersions: + - v1beta1 + operations: + - CREATE + resources: + - elfmachines + sideEffects: None diff --git a/config/webhook/service.yaml b/config/webhook/service.yaml new file mode 100644 index 00000000..5c868588 --- /dev/null +++ b/config/webhook/service.yaml @@ -0,0 +1,20 @@ + +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: service + app.kubernetes.io/instance: webhook-service + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: cluster-api-provider-elf + app.kubernetes.io/part-of: cluster-api-provider-elf + app.kubernetes.io/managed-by: kustomize + name: webhook-service + namespace: system +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: controller-manager diff --git a/controllers/elfmachine_controller_placement_group.go b/controllers/elfmachine_controller_placement_group.go index a3994acd..ac80d6da 100644 --- a/controllers/elfmachine_controller_placement_group.go +++ b/controllers/elfmachine_controller_placement_group.go @@ -41,6 +41,7 @@ import ( annotationsutil "github.com/smartxworks/cluster-api-provider-elf/pkg/util/annotations" kcputil "github.com/smartxworks/cluster-api-provider-elf/pkg/util/kcp" machineutil "github.com/smartxworks/cluster-api-provider-elf/pkg/util/machine" + "github.com/smartxworks/cluster-api-provider-elf/pkg/version" ) // reconcilePlacementGroup makes sure that the placement group exist. @@ -175,7 +176,7 @@ func (r *ElfMachineReconciler) preCheckPlacementGroup(ctx *context.MachineContex // KCP is not in scaling down/rolling update. if !(kcputil.IsKCPRollingUpdateFirstMachine(kcp) || kcputil.IsKCPInScalingDown(kcp)) { - ctx.Logger.V(2).Info("The placement group is full, wait for enough available hosts", "placementGroup", *placementGroup.Name, "availableHosts", availableHostSet.UnsortedList(), "usedHosts", usedHostSet.UnsortedList()) + ctx.Logger.V(1).Info("The placement group is full, wait for enough available hosts", "placementGroup", *placementGroup.Name, "availableHosts", availableHostSet.UnsortedList(), "usedHosts", usedHostSet.UnsortedList()) return nil, nil } @@ -230,7 +231,7 @@ func (r *ElfMachineReconciler) preCheckPlacementGroup(ctx *context.MachineContex return pointer.String(hostID), err } - ctx.Logger.V(2).Info("The placement group is full, wait for enough available hosts", "placementGroup", *placementGroup.Name, "availableHosts", availableHostSet.UnsortedList(), "usedHosts", usedHostSet.UnsortedList()) + ctx.Logger.V(1).Info("The placement group is full, wait for enough available hosts", "placementGroup", *placementGroup.Name, "availableHosts", availableHostSet.UnsortedList(), "usedHosts", usedHostSet.UnsortedList()) return nil, nil } @@ -346,6 +347,12 @@ func (r *ElfMachineReconciler) getPlacementGroup(ctx *context.MachineContext, pl // joinPlacementGroup puts the virtual machine into the placement group. func (r *ElfMachineReconciler) joinPlacementGroup(ctx *context.MachineContext, vm *models.VM) (ret bool, reterr error) { + if machineutil.IsControlPlaneMachine(ctx.Machine) && !version.IsCompatibleWithPlacementGroup(ctx.ElfMachine) { + ctx.Logger.V(1).Info(fmt.Sprintf("The capeVersion of ElfMachine is lower than %s, skip adding VM to the placement group", version.CAPEVersion1_2_0), "capeVersion", version.GetCAPEVersion(ctx.ElfMachine)) + + return true, nil + } + defer func() { if reterr != nil { conditions.MarkFalse(ctx.ElfMachine, infrav1.VMProvisionedCondition, infrav1.JoiningPlacementGroupFailedReason, clusterv1.ConditionSeverityWarning, reterr.Error()) @@ -419,7 +426,7 @@ func (r *ElfMachineReconciler) joinPlacementGroup(ctx *context.MachineContext, v } // KCP is scaling out or being created. - ctx.Logger.V(2).Info("The placement group is full, wait for enough available hosts", "placementGroup", *placementGroup.Name, "availableHosts", availableHostSet.UnsortedList(), "usedHosts", usedHostSet.UnsortedList(), "vmRef", ctx.ElfMachine.Status.VMRef, "vmId", *vm.ID) + ctx.Logger.V(1).Info("The placement group is full, wait for enough available hosts", "placementGroup", *placementGroup.Name, "availableHosts", availableHostSet.UnsortedList(), "usedHosts", usedHostSet.UnsortedList(), "vmRef", ctx.ElfMachine.Status.VMRef, "vmId", *vm.ID) return false, nil } diff --git a/controllers/elfmachine_controller_test.go b/controllers/elfmachine_controller_test.go index 67e923f1..5dafa3ec 100644 --- a/controllers/elfmachine_controller_test.go +++ b/controllers/elfmachine_controller_test.go @@ -775,6 +775,21 @@ var _ = Describe("ElfMachineReconciler", func() { machine.Spec.Bootstrap = clusterv1.Bootstrap{DataSecretName: &secret.Name} }) + It("should skip adding VM to the placement group when capeVersion of ElfMachine is lower than v1.2.0", func() { + fake.ToControlPlaneMachine(machine, kcp) + fake.ToControlPlaneMachine(elfMachine, kcp) + delete(elfMachine.Annotations, infrav1.CAPEVersionAnnotation) + ctrlContext := newCtrlContexts(elfCluster, cluster, elfMachine, machine, secret, md) + machineContext := newMachineContext(ctrlContext, elfCluster, cluster, elfMachine, machine, mockVMService) + fake.InitOwnerReferences(ctrlContext, elfCluster, cluster, elfMachine, machine) + + reconciler := &ElfMachineReconciler{ControllerContext: ctrlContext, NewVMService: mockNewVMService} + ok, err := reconciler.joinPlacementGroup(machineContext, nil) + Expect(ok).To(BeTrue()) + Expect(err).To(BeZero()) + Expect(logBuffer.String()).To(ContainSubstring("The capeVersion of ElfMachine is lower than")) + }) + It("should add vm to the placement group", func() { vm := fake.NewTowerVM() vm.EntityAsyncStatus = nil diff --git a/go.mod b/go.mod index 2e6bde0b..a3fa4eb8 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( github.com/BurntSushi/toml v1.0.0 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/Masterminds/goutils v1.1.1 // indirect - github.com/Masterminds/semver/v3 v3.2.0 // indirect + github.com/Masterminds/semver/v3 v3.2.0 github.com/Masterminds/sprig/v3 v3.2.3 // indirect github.com/Microsoft/go-winio v0.5.0 // indirect github.com/alessio/shellescape v1.4.1 // indirect diff --git a/hack/version.sh b/hack/version.sh index ee861a7d..c30e5831 100755 --- a/hack/version.sh +++ b/hack/version.sh @@ -85,10 +85,11 @@ version::ldflags() { local key=${1} local val=${2} ldflags+=( - "-X 'github.com/smartxworks/cluster-api-provider-elf/version.${key}=${val}'" + "-X 'github.com/smartxworks/cluster-api-provider-elf/pkg/version.${key}=${val}'" ) } + add_ldflag "buildVersion" $1 add_ldflag "buildDate" "$(date ${SOURCE_DATE_EPOCH:+"--date=@${SOURCE_DATE_EPOCH}"} -u +'%Y-%m-%dT%H:%M:%SZ')" add_ldflag "gitCommit" "${GIT_COMMIT}" add_ldflag "gitTreeState" "${GIT_TREE_STATE}" @@ -101,4 +102,4 @@ version::ldflags() { echo "${ldflags[*]-}" } -version::ldflags +version::ldflags $1 diff --git a/main.go b/main.go index 047354c7..d3a07647 100644 --- a/main.go +++ b/main.go @@ -24,7 +24,6 @@ import ( "k8s.io/klog/v2" "k8s.io/klog/v2/klogr" - "sigs.k8s.io/controller-runtime/pkg/healthz" ctrllog "sigs.k8s.io/controller-runtime/pkg/log" ctrlmgr "sigs.k8s.io/controller-runtime/pkg/manager" ctrlsig "sigs.k8s.io/controller-runtime/pkg/manager/signals" @@ -33,7 +32,8 @@ import ( "github.com/smartxworks/cluster-api-provider-elf/pkg/config" "github.com/smartxworks/cluster-api-provider-elf/pkg/context" "github.com/smartxworks/cluster-api-provider-elf/pkg/manager" - "github.com/smartxworks/cluster-api-provider-elf/version" + "github.com/smartxworks/cluster-api-provider-elf/pkg/version" + "github.com/smartxworks/cluster-api-provider-elf/webhooks" ) var ( @@ -44,6 +44,8 @@ var ( defaultSyncPeriod = manager.DefaultSyncPeriod defaultLeaderElectionID = manager.DefaultLeaderElectionID + + defaultWebhookPort = manager.DefaultWebhookServiceContainerPort ) func main() { @@ -91,6 +93,11 @@ func main() { "max-concurrent-reconciles", 10, "The maximum number of allowed, concurrent reconciles.") + flag.IntVar( + &managerOpts.Port, + "webhook-port", + defaultWebhookPort, + "Webhook Server port (set to 0 to disable)") flag.StringVar( &managerOpts.HealthProbeBindAddress, "health-addr", @@ -115,6 +122,15 @@ func main() { // Create a function that adds all of the controllers and webhooks to the manager. addToManager := func(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + if os.Getenv("ENABLE_WEBHOOKS") != "false" { + if err := (&webhooks.ElfMachineMutation{ + Client: mgr.GetClient(), + Logger: mgr.GetLogger().WithName("ElfMachineMutation"), + }).SetupWebhookWithManager(mgr); err != nil { + return err + } + } + if err := controllers.AddClusterControllerToManager(ctx, mgr); err != nil { return err } @@ -126,7 +142,7 @@ func main() { return nil } - setupLog.Info("creating controller manager", "version", version.Get().String()) + setupLog.Info("creating controller manager", "capeVersion", version.CAPEVersion(), "version", version.Get().String()) managerOpts.AddToManager = addToManager mgr, err := manager.New(managerOpts) if err != nil { @@ -145,12 +161,11 @@ func main() { } func setupChecks(mgr ctrlmgr.Manager) { - if err := mgr.AddReadyzCheck("ping", healthz.Ping); err != nil { + if err := mgr.AddReadyzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil { setupLog.Error(err, "unable to create ready check") os.Exit(1) } - - if err := mgr.AddHealthzCheck("ping", healthz.Ping); err != nil { + if err := mgr.AddHealthzCheck("webhook", mgr.GetWebhookServer().StartedChecker()); err != nil { setupLog.Error(err, "unable to create health check") os.Exit(1) } diff --git a/pkg/manager/constants.go b/pkg/manager/constants.go index aa275b8b..7d92729a 100644 --- a/pkg/manager/constants.go +++ b/pkg/manager/constants.go @@ -33,4 +33,8 @@ const ( // DefaultLeaderElectionID is the default value for the eponymous manager option. DefaultLeaderElectionID = DefaultPodName + "-runtime" + + // DefaultWebhookServiceContainerPort is the default value for the eponymous + // manager option. + DefaultWebhookServiceContainerPort = 0 ) diff --git a/pkg/util/annotations/helpers.go b/pkg/util/annotations/helpers.go index 2f83e376..ceb61872 100644 --- a/pkg/util/annotations/helpers.go +++ b/pkg/util/annotations/helpers.go @@ -18,6 +18,7 @@ package annotations import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/cluster-api/util/annotations" infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" ) @@ -41,3 +42,8 @@ func GetPlacementGroupName(o metav1.Object) string { return annotations[infrav1.PlacementGroupNameAnnotation] } + +// AddAnnotations sets the desired annotations on the object and returns true if the annotations have changed. +func AddAnnotations(o metav1.Object, desired map[string]string) bool { + return annotations.AddAnnotations(o, desired) +} diff --git a/pkg/version/consts.go b/pkg/version/consts.go new file mode 100644 index 00000000..ed1f1bad --- /dev/null +++ b/pkg/version/consts.go @@ -0,0 +1,31 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package version + +const ( + // CAPEVersion1_1_0 defines version v1.1.0. + CAPEVersion1_1_0 = "v1.1.0" + + // CAPEVersion1_2_0 defines version v1.2.0. + CAPEVersion1_2_0 = "v1.2.0" + + // CAPEVersionLatest defines version latest. + CAPEVersionLatest = "latest" + + // CAPEVersionDefault defines current CAPE version. + CAPEVersionDefault = CAPEVersion1_2_0 +) diff --git a/pkg/version/util.go b/pkg/version/util.go new file mode 100644 index 00000000..fbd5447f --- /dev/null +++ b/pkg/version/util.go @@ -0,0 +1,67 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package version + +import ( + "github.com/Masterminds/semver/v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" + annotationsutil "github.com/smartxworks/cluster-api-provider-elf/pkg/util/annotations" +) + +// GetCAPEVersion returns the CAPE version of the object. +func GetCAPEVersion(o metav1.Object) string { + annotations := o.GetAnnotations() + if annotations == nil { + return "" + } + + return annotations[infrav1.CAPEVersionAnnotation] +} + +// SetCAPEVersion sets the CAPE version for the object. +func SetCAPEVersion(o metav1.Object, version string) { + if !annotationsutil.HasAnnotation(o, infrav1.CAPEVersionAnnotation) { + annotationsutil.AddAnnotations(o, map[string]string{infrav1.CAPEVersionAnnotation: version}) + } +} + +// SetCurrentCAPEVersion sets the latest CAPE version for the object. +func SetCurrentCAPEVersion(o metav1.Object) { + SetCAPEVersion(o, CAPEVersion()) +} + +// IsCompatibleWithPlacementGroup returns whether the current object can use a placement group. +func IsCompatibleWithPlacementGroup(o metav1.Object) bool { + capeVersion := GetCAPEVersion(o) + if (capeVersion == CAPEVersionLatest) || + (IsSemanticVersion(capeVersion) && capeVersion >= CAPEVersion1_2_0) { + return true + } + + return false +} + +// IsSemanticVersion returns whether the version is an valid Semantic Version. +func IsSemanticVersion(version string) bool { + if _, err := semver.NewVersion(version); err != nil { + return false + } + + return true +} diff --git a/pkg/version/util_test.go b/pkg/version/util_test.go new file mode 100644 index 00000000..e287c68b --- /dev/null +++ b/pkg/version/util_test.go @@ -0,0 +1,70 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package version + +import ( + "testing" + + "github.com/onsi/gomega" + + infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" + "github.com/smartxworks/cluster-api-provider-elf/test/fake" +) + +func TestIsCompatibleWithPlacementGroup(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + t.Run("", func(t *testing.T) { + elfMachine := newElfMachineWithoutCAPEVersion() + g.Expect(IsCompatibleWithPlacementGroup(elfMachine)).To(gomega.BeFalse()) + + elfMachine = newElfMachineWithoutCAPEVersion() + SetCAPEVersion(elfMachine, "") + g.Expect(IsCompatibleWithPlacementGroup(elfMachine)).To(gomega.BeFalse()) + + elfMachine = newElfMachineWithoutCAPEVersion() + SetCAPEVersion(elfMachine, "a") + g.Expect(IsCompatibleWithPlacementGroup(elfMachine)).To(gomega.BeFalse()) + + elfMachine = newElfMachineWithoutCAPEVersion() + SetCAPEVersion(elfMachine, CAPEVersion1_1_0) + g.Expect(IsCompatibleWithPlacementGroup(elfMachine)).To(gomega.BeFalse()) + + elfMachine = newElfMachineWithoutCAPEVersion() + SetCAPEVersion(elfMachine, "v1.1.9") + g.Expect(IsCompatibleWithPlacementGroup(elfMachine)).To(gomega.BeFalse()) + + elfMachine = newElfMachineWithoutCAPEVersion() + SetCurrentCAPEVersion(elfMachine) + g.Expect(IsCompatibleWithPlacementGroup(elfMachine)).To(gomega.BeTrue()) + + elfMachine = newElfMachineWithoutCAPEVersion() + SetCAPEVersion(elfMachine, CAPEVersion1_2_0+"-alpha.0") + g.Expect(IsCompatibleWithPlacementGroup(elfMachine)).To(gomega.BeTrue()) + + elfMachine = newElfMachineWithoutCAPEVersion() + SetCAPEVersion(elfMachine, CAPEVersionLatest) + g.Expect(IsCompatibleWithPlacementGroup(elfMachine)).To(gomega.BeTrue()) + }) +} + +func newElfMachineWithoutCAPEVersion() *infrav1.ElfMachine { + elfMachine := fake.NewElfMachine(nil) + delete(elfMachine.Annotations, infrav1.CAPEVersionAnnotation) + + return elfMachine +} diff --git a/version/version.go b/pkg/version/version.go similarity index 69% rename from version/version.go rename to pkg/version/version.go index 322cbd1a..76078d4c 100644 --- a/version/version.go +++ b/pkg/version/version.go @@ -22,6 +22,7 @@ import ( ) var ( + buildVersion string // TAG_NAME var or git branch tag name gitMajor string // major version, always numeric gitMinor string // minor version, numeric possibly followed by "+" gitVersion string // semantic version, derived by build scripts @@ -30,7 +31,27 @@ var ( buildDate string // build date in ISO8601 format, output of $(date -u +'%Y-%m-%dT%H:%M:%SZ') ) +var ( + // The full CAPE version. + capeVersion string +) + +func init() { + // buildVersion will be set at compiling phase. + // When executing 'go run main.go' in local env, it will be empty. So set it to "latest" same as building against main branch. + if buildVersion == "" { + buildVersion = CAPEVersionLatest + } + + if IsSemanticVersion(buildVersion) { + capeVersion = buildVersion + } else { + capeVersion = CAPEVersionDefault + } +} + type Info struct { + BuildVersion string `json:"buildVersion,omitempty"` Major string `json:"major,omitempty"` Minor string `json:"minor,omitempty"` GitVersion string `json:"gitVersion,omitempty"` @@ -44,6 +65,7 @@ type Info struct { func Get() Info { return Info{ + BuildVersion: buildVersion, Major: gitMajor, Minor: gitMinor, GitVersion: gitVersion, @@ -58,5 +80,11 @@ func Get() Info { // String returns info as a human-friendly version string. func (info Info) String() string { - return info.GitVersion + return fmt.Sprintf("Version: %s, BuildDate: %s, GitVersion: %s, GitCommit: %s, GoVersion: %s", + info.BuildVersion, info.BuildDate, info.GitVersion, info.GitCommit, info.GoVersion) +} + +// CAPEVersion returns the full CAPE version, e.g. v1.1.0 or v1.1.0-rc.1. +func CAPEVersion() string { + return capeVersion } diff --git a/test/fake/types.go b/test/fake/types.go index d89f8158..ff69cea2 100644 --- a/test/fake/types.go +++ b/test/fake/types.go @@ -77,27 +77,7 @@ func NewClusterObjects() (*infrav1.ElfCluster, *clusterv1.Cluster) { } func NewMachineObjects(elfCluster *infrav1.ElfCluster, cluster *clusterv1.Cluster) (*infrav1.ElfMachine, *clusterv1.Machine) { - elfMachine := &infrav1.ElfMachine{ - ObjectMeta: metav1.ObjectMeta{ - Name: names.SimpleNameGenerator.GenerateName("elfmachine-"), - Namespace: Namespace, - Labels: map[string]string{ - clusterv1.ClusterNameLabel: elfCluster.Name, - }, - CreationTimestamp: metav1.Now(), - }, - Spec: infrav1.ElfMachineSpec{ - HA: true, - NumCPUs: 1, - NumCoresPerSocket: 1, - MemoryMiB: 1, - Network: infrav1.NetworkSpec{ - Devices: []infrav1.NetworkDeviceSpec{ - {}, - }, - }, - }, - } + elfMachine := NewElfMachine(elfCluster) machine := &clusterv1.Machine{ ObjectMeta: metav1.ObjectMeta{ @@ -122,6 +102,36 @@ func NewMachineObjects(elfCluster *infrav1.ElfCluster, cluster *clusterv1.Cluste return elfMachine, machine } +func NewElfMachine(elfCluster *infrav1.ElfCluster) *infrav1.ElfMachine { + elfMachine := &infrav1.ElfMachine{ + ObjectMeta: metav1.ObjectMeta{ + Name: names.SimpleNameGenerator.GenerateName("elfmachine-"), + Namespace: Namespace, + CreationTimestamp: metav1.Now(), + Annotations: map[string]string{infrav1.CAPEVersionAnnotation: "latest"}, + }, + Spec: infrav1.ElfMachineSpec{ + HA: true, + NumCPUs: 1, + NumCoresPerSocket: 1, + MemoryMiB: 1, + Network: infrav1.NetworkSpec{ + Devices: []infrav1.NetworkDeviceSpec{ + {}, + }, + }, + }, + } + + if elfCluster != nil { + elfMachine.Labels = map[string]string{ + clusterv1.ClusterNameLabel: elfCluster.Name, + } + } + + return elfMachine +} + func NewClusterAndMachineObjects() (*infrav1.ElfCluster, *clusterv1.Cluster, *infrav1.ElfMachine, *clusterv1.Machine, *corev1.Secret) { elfCluster, cluster := NewClusterObjects() elfMachine, machine := NewMachineObjects(elfCluster, cluster) diff --git a/test/helpers/envtest.go b/test/helpers/envtest.go index 9e61b713..3d1fce7a 100644 --- a/test/helpers/envtest.go +++ b/test/helpers/envtest.go @@ -57,6 +57,7 @@ import ( infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" "github.com/smartxworks/cluster-api-provider-elf/pkg/context" "github.com/smartxworks/cluster-api-provider-elf/pkg/manager" + "github.com/smartxworks/cluster-api-provider-elf/webhooks" ) func init() { @@ -143,6 +144,13 @@ func NewTestEnvironment() *TestEnvironment { KubeConfig: env.Config, } managerOpts.AddToManager = func(ctx *context.ControllerManagerContext, mgr ctrlmgr.Manager) error { + if err := (&webhooks.ElfMachineMutation{ + Client: mgr.GetClient(), + Logger: mgr.GetLogger().WithName("ElfMachineMutation"), + }).SetupWebhookWithManager(mgr); err != nil { + return err + } + return nil } diff --git a/webhooks/elfmachine_webhook_mutation.go b/webhooks/elfmachine_webhook_mutation.go new file mode 100644 index 00000000..0c30a099 --- /dev/null +++ b/webhooks/elfmachine_webhook_mutation.go @@ -0,0 +1,73 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhooks + +import ( + goctx "context" + "encoding/json" + "net/http" + + "github.com/go-logr/logr" + admissionv1 "k8s.io/api/admission/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/webhook" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + infrav1 "github.com/smartxworks/cluster-api-provider-elf/api/v1beta1" + "github.com/smartxworks/cluster-api-provider-elf/pkg/version" +) + +func (m *ElfMachineMutation) SetupWebhookWithManager(mgr ctrl.Manager) error { + hookServer := mgr.GetWebhookServer() + hookServer.Register("/mutate-infrastructure-cluster-x-k8s-io-v1beta1-elfmachine", &webhook.Admission{Handler: m}) + return ctrl.NewWebhookManagedBy(mgr). + For(&infrav1.ElfMachine{}). + Complete() +} + +//+kubebuilder:object:generate=false +//+kubebuilder:webhook:verbs=create,path=/mutate-infrastructure-cluster-x-k8s-io-v1beta1-elfmachine,mutating=true,failurePolicy=fail,sideEffects=None,groups=infrastructure.cluster.x-k8s.io,resources=elfmachines,versions=v1beta1,name=mutation.elfmachine.infrastructure.x-k8s.io,admissionReviewVersions=v1 + +type ElfMachineMutation struct { + client.Client + decoder *admission.Decoder + logr.Logger +} + +func (m *ElfMachineMutation) Handle(ctx goctx.Context, request admission.Request) admission.Response { + var elfMachine infrav1.ElfMachine + if err := m.decoder.Decode(request, &elfMachine); err != nil { + return admission.Errored(http.StatusBadRequest, err) + } + + if request.Operation == admissionv1.Create { + version.SetCurrentCAPEVersion(&elfMachine) + } + + if marshaledElfMachine, err := json.Marshal(elfMachine); err != nil { + return admission.Errored(http.StatusInternalServerError, err) + } else { + return admission.PatchResponseFromRaw(request.Object.Raw, marshaledElfMachine) + } +} + +// InjectDecoder injects the decoder. +func (m *ElfMachineMutation) InjectDecoder(d *admission.Decoder) error { + m.decoder = d + return nil +} diff --git a/webhooks/elfmachine_webhook_mutation_test.go b/webhooks/elfmachine_webhook_mutation_test.go new file mode 100644 index 00000000..d3898357 --- /dev/null +++ b/webhooks/elfmachine_webhook_mutation_test.go @@ -0,0 +1,17 @@ +/* +Copyright 2023. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhooks