From 0ad170900685838ba423d34db64faed81d4884a6 Mon Sep 17 00:00:00 2001 From: Avishay Traeger Date: Wed, 27 Apr 2022 22:28:43 +0300 Subject: [PATCH] MGMT-9272 Automatic agent classification labels (#3721) * MGMT-9272: Define AgentClassification CRD This CRD defines the API for users to classify Agents by providing a query that is run on the Agent's inventory along with a corresponding label. * MGMT-9272: AgentClassification and AgentLabel controllers The AgentLabel controller updates Agents' labels according to defined AgentClassification CRs in the same namespace. The AgentClassification controller maintain counts for each AgentClassification of how many Agents matched and how many had errors when trying to match. It also has a finalizer such that the AgentClassification won't be deleted as long as there are Agents matching it. * MGMT-9272: AgentClassification admission hooks This ensures that: 1. Specified labels are valid and do not change 2. The specified query can be parsed * MGMT-9272: AgentClassification documentation * MGMT-9272: go.mod and go.sum * MGMT-9272: Agent labels subsystem test --- api/v1beta1/agentclassification_types.go | 83 ++++++ api/v1beta1/zz_generated.deepcopy.go | 96 +++++++ cmd/main.go | 10 + cmd/webadmission/main.go | 1 + ...all.openshift.io_agentclassifications.yaml | 107 ++++++++ config/crd/kustomization.yaml | 1 + config/crd/resources.yaml | 106 +++++++ config/rbac/role.yaml | 26 ++ ...all.openshift.io_agentclassifications.yaml | 105 +++++++ ...ervice-operator.clusterserviceversion.yaml | 29 ++ .../automatic-agent-classification-labels.md | 6 +- docs/hive-integration/agent-labels.md | 44 ++- .../crds/agentClassification.yaml | 9 + go.mod | 4 +- go.sum | 8 +- .../controllers/agent_controller.go | 3 + .../agentclassification_controller.go | 188 +++++++++++++ .../agentclassification_controller_test.go | 143 ++++++++++ .../controllers/agentlabel_controller.go | 203 ++++++++++++++ .../controllers/agentlabel_controller_test.go | 133 +++++++++ .../agent_classification_admission_hook.go | 238 ++++++++++++++++ ...gent_classification_admission_hook_test.go | 259 ++++++++++++++++++ subsystem/cluster_test.go | 2 +- subsystem/kubeapi_test.go | 93 ++++++- 24 files changed, 1880 insertions(+), 17 deletions(-) create mode 100644 api/v1beta1/agentclassification_types.go create mode 100644 config/crd/bases/agent-install.openshift.io_agentclassifications.yaml create mode 100644 deploy/olm-catalog/manifests/agent-install.openshift.io_agentclassifications.yaml create mode 100644 docs/hive-integration/crds/agentClassification.yaml create mode 100644 internal/controller/controllers/agentclassification_controller.go create mode 100644 internal/controller/controllers/agentclassification_controller_test.go create mode 100644 internal/controller/controllers/agentlabel_controller.go create mode 100644 internal/controller/controllers/agentlabel_controller_test.go create mode 100644 pkg/validating-webhooks/agentinstall/v1beta1/agent_classification_admission_hook.go create mode 100644 pkg/validating-webhooks/agentinstall/v1beta1/agent_classification_admission_hook_test.go diff --git a/api/v1beta1/agentclassification_types.go b/api/v1beta1/agentclassification_types.go new file mode 100644 index 00000000000..b183a8a166e --- /dev/null +++ b/api/v1beta1/agentclassification_types.go @@ -0,0 +1,83 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1 + +import ( + conditionsv1 "github.com/openshift/custom-resource-status/conditions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + QueryErrorsCondition conditionsv1.ConditionType = "QueryErrors" + QueryNoErrorsReason string = "NoQueryErrors" + QueryHasErrorsReason string = "HasQueryErrors" +) + +// AgentClassificationSpec defines the desired state of AgentClassification +type AgentClassificationSpec struct { + // LabelKey specifies the label key to apply to matched Agents + // + // +immutable + LabelKey string `json:"labelKey"` + + // LabelValue specifies the label value to apply to matched Agents + // + // +immutable + LabelValue string `json:"labelValue"` + + // Query is in gojq format (https://github.com/itchyny/gojq#difference-to-jq) + // and will be invoked on each Agent's inventory. The query should return a + // boolean. The operator will apply the label to any Agent for which "true" + // is returned. + Query string `json:"query"` +} + +// AgentClassificationStatus defines the observed state of AgentClassification +type AgentClassificationStatus struct { + // MatchedCount shows how many Agents currently match the classification + MatchedCount int `json:"matchedCount,omitempty"` + + // ErrorCount shows how many Agents encountered errors when matching the classification + ErrorCount int `json:"errorCount,omitempty"` + + Conditions []conditionsv1.Condition `json:"conditions,omitempty"` +} + +//+kubebuilder:object:root=true +//+kubebuilder:subresource:status + +// AgentClassification is the Schema for the AgentClassifications API +type AgentClassification struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec AgentClassificationSpec `json:"spec,omitempty"` + Status AgentClassificationStatus `json:"status,omitempty"` +} + +//+kubebuilder:object:root=true + +// AgentClassificationList contains a list of AgentClassification +type AgentClassificationList struct { + metav1.TypeMeta `json:",inline"` + metav1.ListMeta `json:"metadata,omitempty"` + Items []AgentClassification `json:"items"` +} + +func init() { + SchemeBuilder.Register(&AgentClassification{}, &AgentClassificationList{}) +} diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 57162d3143f..d93a93a7bb4 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -56,6 +56,102 @@ func (in *Agent) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AgentClassification) DeepCopyInto(out *AgentClassification) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) + out.Spec = in.Spec + in.Status.DeepCopyInto(&out.Status) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentClassification. +func (in *AgentClassification) DeepCopy() *AgentClassification { + if in == nil { + return nil + } + out := new(AgentClassification) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AgentClassification) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AgentClassificationList) DeepCopyInto(out *AgentClassificationList) { + *out = *in + out.TypeMeta = in.TypeMeta + in.ListMeta.DeepCopyInto(&out.ListMeta) + if in.Items != nil { + in, out := &in.Items, &out.Items + *out = make([]AgentClassification, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentClassificationList. +func (in *AgentClassificationList) DeepCopy() *AgentClassificationList { + if in == nil { + return nil + } + out := new(AgentClassificationList) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *AgentClassificationList) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AgentClassificationSpec) DeepCopyInto(out *AgentClassificationSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentClassificationSpec. +func (in *AgentClassificationSpec) DeepCopy() *AgentClassificationSpec { + if in == nil { + return nil + } + out := new(AgentClassificationSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AgentClassificationStatus) DeepCopyInto(out *AgentClassificationStatus) { + *out = *in + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AgentClassificationStatus. +func (in *AgentClassificationStatus) DeepCopy() *AgentClassificationStatus { + if in == nil { + return nil + } + out := new(AgentClassificationStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AgentList) DeepCopyInto(out *AgentList) { *out = *in diff --git a/cmd/main.go b/cmd/main.go index ef8cd409b2f..bac17c21822 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -543,6 +543,16 @@ func main() { CRDEventsHandler: crdEventsHandler, }).SetupWithManager(ctrlMgr), "unable to create controller AgentClusterInstall") + failOnError((&controllers.AgentClassificationReconciler{ + Client: ctrlMgr.GetClient(), + Log: log, + }).SetupWithManager(ctrlMgr), "unable to create controller AgentClassification") + + failOnError((&controllers.AgentLabelReconciler{ + Client: ctrlMgr.GetClient(), + Log: log, + }).SetupWithManager(ctrlMgr), "unable to create controller AgentLabel") + log.Infof("Starting controllers") failOnError(ctrlMgr.Start(ctrl.SetupSignalHandler()), "failed to run manager") } diff --git a/cmd/webadmission/main.go b/cmd/webadmission/main.go index 629ce8d302e..a0e2f407102 100644 --- a/cmd/webadmission/main.go +++ b/cmd/webadmission/main.go @@ -21,6 +21,7 @@ func main() { hiveextvalidatingwebhooks.NewAgentClusterInstallValidatingAdmissionHook(decoder), agentinstallvalidatingwebhooks.NewInfraEnvValidatingAdmissionHook(decoder), agentinstallvalidatingwebhooks.NewAgentValidatingAdmissionHook(decoder), + agentinstallvalidatingwebhooks.NewAgentClassificationValidatingAdmissionHook(decoder), ) } diff --git a/config/crd/bases/agent-install.openshift.io_agentclassifications.yaml b/config/crd/bases/agent-install.openshift.io_agentclassifications.yaml new file mode 100644 index 00000000000..3dc9e9df8b2 --- /dev/null +++ b/config/crd/bases/agent-install.openshift.io_agentclassifications.yaml @@ -0,0 +1,107 @@ + +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.2 + creationTimestamp: null + name: agentclassifications.agent-install.openshift.io +spec: + group: agent-install.openshift.io + names: + kind: AgentClassification + listKind: AgentClassificationList + plural: agentclassifications + singular: agentclassification + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: AgentClassification is the Schema for the AgentClassifications + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AgentClassificationSpec defines the desired state of AgentClassification + properties: + labelKey: + description: LabelKey specifies the label key to apply to matched + Agents + type: string + labelValue: + description: LabelValue specifies the label value to apply to matched + Agents + type: string + query: + description: Query is in gojq format (https://github.com/itchyny/gojq#difference-to-jq) + and will be invoked on each Agent's inventory. The query should + return a boolean. The operator will apply the label to any Agent + for which "true" is returned. + type: string + required: + - labelKey + - labelValue + - query + type: object + status: + description: AgentClassificationStatus defines the observed state of AgentClassification + properties: + conditions: + items: + description: Condition represents the state of the operator's reconciliation + functionality. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + description: ConditionType is the state of the operator's reconciliation + functionality. + type: string + required: + - status + - type + type: object + type: array + errorCount: + description: ErrorCount shows how many Agents encountered errors when + matching the classification + type: integer + matchedCount: + description: MatchedCount shows how many Agents currently match the + classification + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/config/crd/kustomization.yaml b/config/crd/kustomization.yaml index 767289996ab..50ee1296623 100644 --- a/config/crd/kustomization.yaml +++ b/config/crd/kustomization.yaml @@ -6,6 +6,7 @@ resources: - bases/agent-install.openshift.io_infraenvs.yaml - bases/agent-install.openshift.io_agents.yaml - bases/agent-install.openshift.io_nmstateconfigs.yaml +- bases/agent-install.openshift.io_agentclassifications.yaml - bases/extensions.hive.openshift.io_agentclusterinstalls.yaml # +kubebuilder:scaffold:crdkustomizeresource diff --git a/config/crd/resources.yaml b/config/crd/resources.yaml index 619832089a8..0814c39a93d 100644 --- a/config/crd/resources.yaml +++ b/config/crd/resources.yaml @@ -1,5 +1,111 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.2 + creationTimestamp: null + name: agentclassifications.agent-install.openshift.io +spec: + group: agent-install.openshift.io + names: + kind: AgentClassification + listKind: AgentClassificationList + plural: agentclassifications + singular: agentclassification + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: AgentClassification is the Schema for the AgentClassifications + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AgentClassificationSpec defines the desired state of AgentClassification + properties: + labelKey: + description: LabelKey specifies the label key to apply to matched + Agents + type: string + labelValue: + description: LabelValue specifies the label value to apply to matched + Agents + type: string + query: + description: Query is in gojq format (https://github.com/itchyny/gojq#difference-to-jq) + and will be invoked on each Agent's inventory. The query should + return a boolean. The operator will apply the label to any Agent + for which "true" is returned. + type: string + required: + - labelKey + - labelValue + - query + type: object + status: + description: AgentClassificationStatus defines the observed state of AgentClassification + properties: + conditions: + items: + description: Condition represents the state of the operator's reconciliation + functionality. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + description: ConditionType is the state of the operator's reconciliation + functionality. + type: string + required: + - status + - type + type: object + type: array + errorCount: + description: ErrorCount shows how many Agents encountered errors when + matching the classification + type: integer + matchedCount: + description: MatchedCount shows how many Agents currently match the + classification + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition metadata: annotations: controller-gen.kubebuilder.io/version: v0.6.2 diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 04c3acfb0ad..b22382f6381 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -63,6 +63,32 @@ rules: - patch - update - watch +- apiGroups: + - agent-install.openshift.io + resources: + - agentclassifications + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - agent-install.openshift.io + resources: + - agentclassifications/finalizers + verbs: + - update +- apiGroups: + - agent-install.openshift.io + resources: + - agentclassifications/status + verbs: + - get + - patch + - update - apiGroups: - agent-install.openshift.io resources: diff --git a/deploy/olm-catalog/manifests/agent-install.openshift.io_agentclassifications.yaml b/deploy/olm-catalog/manifests/agent-install.openshift.io_agentclassifications.yaml new file mode 100644 index 00000000000..66789948bc0 --- /dev/null +++ b/deploy/olm-catalog/manifests/agent-install.openshift.io_agentclassifications.yaml @@ -0,0 +1,105 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.6.2 + creationTimestamp: null + name: agentclassifications.agent-install.openshift.io +spec: + group: agent-install.openshift.io + names: + kind: AgentClassification + listKind: AgentClassificationList + plural: agentclassifications + singular: agentclassification + scope: Namespaced + versions: + - name: v1beta1 + schema: + openAPIV3Schema: + description: AgentClassification is the Schema for the AgentClassifications + API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: AgentClassificationSpec defines the desired state of AgentClassification + properties: + labelKey: + description: LabelKey specifies the label key to apply to matched + Agents + type: string + labelValue: + description: LabelValue specifies the label value to apply to matched + Agents + type: string + query: + description: Query is in gojq format (https://github.com/itchyny/gojq#difference-to-jq) + and will be invoked on each Agent's inventory. The query should + return a boolean. The operator will apply the label to any Agent + for which "true" is returned. + type: string + required: + - labelKey + - labelValue + - query + type: object + status: + description: AgentClassificationStatus defines the observed state of AgentClassification + properties: + conditions: + items: + description: Condition represents the state of the operator's reconciliation + functionality. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + description: ConditionType is the state of the operator's reconciliation + functionality. + type: string + required: + - status + - type + type: object + type: array + errorCount: + description: ErrorCount shows how many Agents encountered errors when + matching the classification + type: integer + matchedCount: + description: MatchedCount shows how many Agents currently match the + classification + type: integer + type: object + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] diff --git a/deploy/olm-catalog/manifests/assisted-service-operator.clusterserviceversion.yaml b/deploy/olm-catalog/manifests/assisted-service-operator.clusterserviceversion.yaml index 3aa67b2f268..3f07096cd31 100644 --- a/deploy/olm-catalog/manifests/assisted-service-operator.clusterserviceversion.yaml +++ b/deploy/olm-catalog/manifests/assisted-service-operator.clusterserviceversion.yaml @@ -108,6 +108,9 @@ spec: apiservicedefinitions: {} customresourcedefinitions: owned: + - kind: AgentClassification + name: agentclassifications.agent-install.openshift.io + version: v1beta1 - kind: AgentClusterInstall name: agentclusterinstalls.extensions.hive.openshift.io version: v1beta1 @@ -299,6 +302,32 @@ spec: - patch - update - watch + - apiGroups: + - agent-install.openshift.io + resources: + - agentclassifications + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - agent-install.openshift.io + resources: + - agentclassifications/finalizers + verbs: + - update + - apiGroups: + - agent-install.openshift.io + resources: + - agentclassifications/status + verbs: + - get + - patch + - update - apiGroups: - agent-install.openshift.io resources: diff --git a/docs/enhancements/automatic-agent-classification-labels.md b/docs/enhancements/automatic-agent-classification-labels.md index a172b1b7480..f67e310c3a3 100644 --- a/docs/enhancements/automatic-agent-classification-labels.md +++ b/docs/enhancements/automatic-agent-classification-labels.md @@ -89,7 +89,7 @@ type AgentClassificationStatus struct { We will base the expression definition on the [gojq](https://github.com/itchyny/gojq) library, which supports jq queries in Go. The query will be run on each Agent's inventory. If the query returns true, then the specified label will be applied. For example, -The query for label "size:medium" might be: ".cpu.count == 2 and .memory.PhysicalBytes >= 4294967296 and .memory.PhysicalBytes < 8589934592" +The query for label "size:medium" might be: ".cpu.count == 2 and .memory.physicalBytes >= 4294967296 and .memory.physicalBytes < 8589934592" The query for label "storage:large" might be: "[.disks[] | select(.sizeBytes > 1073741824000)] | length > 5" ### Operator changes @@ -97,13 +97,13 @@ The query for label "storage:large" might be: "[.disks[] | select(.sizeBytes > 1 A new agent-label-controller will reconcile Agent CRs and update labels on Agents as necessary. It will be triggered by updates to both Agents and AgentClassifications via watch mapping. The controller lists the AgentClassifications in the Agent's namespace and for each: * If the AgentClassifications is being deleted or if the query returns false, deletes the label from the Agent if it exists * If the query returns true, set the label if not already set (the label key will be prefixed with inventoryclassification.agent-install.openshift.io/) -* If the query fails to run, set the label key, but the value to "QUERY ERROR: \" +* If the query fails to run, set the label key, but the value to "QUERYERROR-\" * An annotation inventoryclassification.agent-install.openshift.io/updatedat will be set with the current timestamp to help debugging A new agent-classification-controller will reconcile AgentClassification CRs. It will be triggered by updates to both AgentClassifications and Agents via watch mapping. The controller first: * Tries to compile the query, and if it fails, sets QueryValidCondition to false with a proper message. * Sets a finalizer to ensure the AgentClassification is deleted only after no Agents have its label set. -The controller then list the Agents in the AgentClassification's namespace and counts how many Agents have the label key/value (Status.MatchedCount) or how many have the key with the value set to "QUERY ERROR" (Status.ErrorCount). +The controller then list the Agents in the AgentClassification's namespace and counts how many Agents have the label key/value (Status.MatchedCount) or how many have the key with the value set to "QUERYERROR" (Status.ErrorCount). If Status.ErrorCount is not zero, then QueryErrorsCondition is set. If the AgentClassification is being deleted and Status.MatchedCount and Status.ErrorCount are both zero, then it is deleted - otherwise it will check again in the next reconcile. diff --git a/docs/hive-integration/agent-labels.md b/docs/hive-integration/agent-labels.md index e66cf2cc79a..3bb69ce2955 100644 --- a/docs/hive-integration/agent-labels.md +++ b/docs/hive-integration/agent-labels.md @@ -10,9 +10,41 @@ An annotation on the Agent CR indicates a version for the labels, so clients can Labels marked as boolean will have either the string "true" or "false". ### v0.1 -* feature.agent-install.openshift.io/storage-hasnonrotationaldisk (boolean): Indicates if the Agent has at least one SSD -* feature.agent-install.openshift.io/cpu-architecture (string): The CPU architecture (e.g., x86_64, arm64) -* feature.agent-install.openshift.io/cpu-virtenabled (boolean): Indicates if the CPU has the virtualization flag (VMX or SVM) -* feature.agent-install.openshift.io/host-manufacturer (string): The host's manufacturer -* feature.agent-install.openshift.io/host-productname (string): The host's product name -* feature.agent-install.openshift.io/host-isvirtual (boolean): Indicates if the host is a virtual machine \ No newline at end of file +* inventory.agent-install.openshift.io/storage-hasnonrotationaldisk (boolean): Indicates if the Agent has at least one SSD +* inventory.agent-install.openshift.io/cpu-architecture (string): The CPU architecture (e.g., x86_64, arm64) +* inventory.agent-install.openshift.io/cpu-virtenabled (boolean): Indicates if the CPU has the virtualization flag (VMX or SVM) +* inventory.agent-install.openshift.io/host-manufacturer (string): The host's manufacturer +* inventory.agent-install.openshift.io/host-productname (string): The host's product name +* inventory.agent-install.openshift.io/host-isvirtual (boolean): Indicates if the host is a virtual machine + +# Agent Classification labels + +The AgentClassification CRD defines the API for users to classify Agents by providing a query that is run on the Agent's inventory along with a corresponding label. +The query format is defined by the [gojq](https://github.com/itchyny/gojq) library, which supports jq queries in Go. +Any Agent in the same namespace as the AgentClassification whose inventory causes the specified query to evaluate to `true` will have the specified label applied (prefixed by "agentclassification.agent-install.openshift.io/"). + +Some examples include: + +``` +spec: + labelKey: size + labelValue: medium + query: ".cpu.count == 2 and .memory.physicalBytes >= 4294967296 and .memory.physicalBytes < 8589934592" +``` + +``` +spec: + labelKey: storage + labelValue: large + query: "[.disks[] | select(.sizeBytes > 1073741824000)] | length > 5" +``` + +The AgentClassification CRD has the following information in its Status: +* MatchedCount: shows how many Agents currently match the classification +* ErrorCount: shows how many Agents encountered errors when matching the classification +* Conditions: + * QueryErrors: true if there were errors when processing the query + +Notes: +1. The labelKey and labelValue properties are immutable. +1. If an AgentClassification is deleted, the specified label will first be removed from all Agents. \ No newline at end of file diff --git a/docs/hive-integration/crds/agentClassification.yaml b/docs/hive-integration/crds/agentClassification.yaml new file mode 100644 index 00000000000..7eb5b0370a3 --- /dev/null +++ b/docs/hive-integration/crds/agentClassification.yaml @@ -0,0 +1,9 @@ +apiVersion: agent-install.openshift.io/v1beta1 +kind: AgentClassification +metadata: + name: sizexl + namespace: agents +spec: + labelKey: size + labelValue: xlarge + query: ".cpu.count == 4 and .memory.physicalBytes >= 17179869184 and .memory.physicalBytes < 34359738368" \ No newline at end of file diff --git a/go.mod b/go.mod index e3281a79548..7f27cf9d896 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-version v1.4.0 github.com/iancoleman/strcase v0.2.0 + github.com/itchyny/gojq v0.12.7 github.com/jinzhu/copier v0.3.5 github.com/kelseyhightower/envconfig v1.4.0 github.com/kennygrant/sanitize v1.2.4 @@ -67,7 +68,7 @@ require ( go.elastic.co/apm/module/apmlogrus v1.15.0 golang.org/x/crypto v0.0.0-20220214200702-86341886e292 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20220209214540-3681064d5158 + golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 gopkg.in/ini.v1 v1.66.4 gopkg.in/square/go-jose.v2 v2.6.0 gopkg.in/yaml.v2 v2.4.0 @@ -138,6 +139,7 @@ require ( github.com/hashicorp/errwrap v1.0.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/itchyny/timefmt-go v0.1.3 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect github.com/jackc/pgconn v1.12.0 // indirect github.com/jackc/pgio v1.0.0 // indirect diff --git a/go.sum b/go.sum index 694406d9e8f..5179d45865d 100644 --- a/go.sum +++ b/go.sum @@ -932,6 +932,10 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/itchyny/gojq v0.12.7 h1:hYPTpeWfrJ1OT+2j6cvBScbhl0TkdwGM4bc66onUSOQ= +github.com/itchyny/gojq v0.12.7/go.mod h1:ZdvNHVlzPgUf8pgjnuDTmGfHA/21KoutQUJ3An/xNuw= +github.com/itchyny/timefmt-go v0.1.3 h1:7M3LGVDsqcd0VZH2U+x393obrzZisp7C0uEe921iRkU= +github.com/itchyny/timefmt-go v0.1.3/go.mod h1:0osSSCQSASBJMsIZnhAaF1C2fCBTJZXrnj37mG8/c+A= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/jackc/chunkreader v1.0.0 h1:4s39bBR8ByfqH+DKm8rQA3E1LHZWB9XWcrz8fqaZbe0= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= @@ -1145,6 +1149,7 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= @@ -2074,8 +2079,9 @@ golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 h1:nhht2DYV/Sn3qOayu8lM+cU1ii9sTLUeBQwQQfUHtrs= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/internal/controller/controllers/agent_controller.go b/internal/controller/controllers/agent_controller.go index 7959b4bc4a5..96cd658e4b0 100644 --- a/internal/controller/controllers/agent_controller.go +++ b/internal/controller/controllers/agent_controller.go @@ -1040,6 +1040,9 @@ func setAgentAnnotation(log logrus.FieldLogger, agent *aiv1beta1.Agent, key stri func setAgentLabel(log logrus.FieldLogger, agent *aiv1beta1.Agent, key string, value string) bool { labels := agent.GetLabels() + if labels == nil { + labels = make(map[string]string) + } // Label values can only have alphanumeric characters, '-', '_' or '.' re := regexp.MustCompile("[^-A-Za-z0-9_.]+") diff --git a/internal/controller/controllers/agentclassification_controller.go b/internal/controller/controllers/agentclassification_controller.go new file mode 100644 index 00000000000..4f6b9cc528e --- /dev/null +++ b/internal/controller/controllers/agentclassification_controller.go @@ -0,0 +1,188 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "fmt" + "strings" + + aiv1beta1 "github.com/openshift/assisted-service/api/v1beta1" + logutil "github.com/openshift/assisted-service/pkg/log" + conditionsv1 "github.com/openshift/custom-resource-status/conditions/v1" + "github.com/sirupsen/logrus" + "github.com/thoas/go-funk" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +const ( + AgentClassificationFinalizer = "agentclassification." + aiv1beta1.Group + ClassificationLabelPrefix = "agentclassification." + aiv1beta1.Group + "/" +) + +// AgentClassificationReconciler reconciles a AgentClassification object +type AgentClassificationReconciler struct { + client.Client + Log logrus.FieldLogger +} + +//+kubebuilder:rbac:groups=agent-install.openshift.io,resources=agentclassifications,verbs=get;list;watch;create;update;patch;delete +//+kubebuilder:rbac:groups=agent-install.openshift.io,resources=agentclassifications/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=agent-install.openshift.io,resources=agentclassifications/finalizers,verbs=update +//+kubebuilder:rbac:groups=agent-install.openshift.io,resources=agents,verbs=get;list;watch + +func (r *AgentClassificationReconciler) Reconcile(origCtx context.Context, req ctrl.Request) (ctrl.Result, error) { + ctx := addRequestIdIfNeeded(origCtx) + log := r.Log.WithFields( + logrus.Fields{ + "agent_classification": req.Name, + "agent_classification_namespace": req.Namespace, + }) + + defer func() { + log.Info("AgentClassification Reconcile ended") + }() + + log.Info("AgentClassification Reconcile started") + + classification := &aiv1beta1.AgentClassification{} + if err := r.Get(ctx, req.NamespacedName, classification); err != nil { + log.WithError(err).Errorf("Failed to get AgentClassification %s", req.NamespacedName) + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // Add a finalizer to newly created objects. + if classification.DeletionTimestamp.IsZero() && !funk.ContainsString(classification.GetFinalizers(), AgentClassificationFinalizer) { + controllerutil.AddFinalizer(classification, AgentClassificationFinalizer) + if err := r.Update(ctx, classification); err != nil { + log.WithError(err).Errorf("failed to add finalizer %s to resource %s %s", AgentClassificationFinalizer, classification.Name, classification.Namespace) + return ctrl.Result{}, err + } + } + + matchedCount := 0 + errorCount := 0 + + agents := aiv1beta1.AgentList{} + opts := &client.ListOptions{ + Namespace: classification.Namespace, + } + if err := r.List(ctx, &agents, opts); err != nil { + return ctrl.Result{}, err + } + matchedCount, errorCount = countAgentsByClassification(log, &agents, classification) + + setErrorCountCondition(classification, errorCount) + classification.Status.MatchedCount = matchedCount + classification.Status.ErrorCount = errorCount + + if !classification.DeletionTimestamp.IsZero() { + if matchedCount > 0 { + log.Info("waiting to delete") + } else { + controllerutil.RemoveFinalizer(classification, AgentClassificationFinalizer) + if err := r.Update(ctx, classification); err != nil { + log.WithError(err).Errorf("failed to remove finalizer %s from resource %s %s", AgentClassificationFinalizer, classification.Name, classification.Namespace) + return ctrl.Result{}, err + } + return ctrl.Result{}, nil + } + } + + if err := r.Status().Update(ctx, classification); err != nil { + log.WithError(err).Error("failed to update classification status") + return ctrl.Result{}, err + } + + return ctrl.Result{}, nil +} + +func countAgentsByClassification(log logrus.FieldLogger, agents *aiv1beta1.AgentList, classification *aiv1beta1.AgentClassification) (matchedCount, errorCount int) { + for _, agent := range agents.Items { + labels := agent.GetLabels() + if labels == nil { + continue + } + if _, ok := labels[ClassificationLabelPrefix+classification.Spec.LabelKey]; !ok { + continue + } + if labels[ClassificationLabelPrefix+classification.Spec.LabelKey] == classification.Spec.LabelValue { + matchedCount++ + } else if strings.HasPrefix(labels[ClassificationLabelPrefix+classification.Spec.LabelKey], "QUERYERROR") { + errorCount++ + } + } + + return +} + +func setErrorCountCondition(classification *aiv1beta1.AgentClassification, errorCount int) { + if errorCount != 0 { + conditionsv1.SetStatusConditionNoHeartbeat(&classification.Status.Conditions, conditionsv1.Condition{ + Type: aiv1beta1.QueryErrorsCondition, + Status: corev1.ConditionTrue, + Reason: aiv1beta1.QueryHasErrorsReason, + Message: fmt.Sprintf("%d Agents failed to apply the classification", errorCount), + }) + } else { + conditionsv1.SetStatusConditionNoHeartbeat(&classification.Status.Conditions, conditionsv1.Condition{ + Type: aiv1beta1.QueryErrorsCondition, + Status: corev1.ConditionFalse, + Reason: aiv1beta1.QueryNoErrorsReason, + Message: "No Agents failed to apply the classification", + }) + } +} + +func (r *AgentClassificationReconciler) SetupWithManager(mgr ctrl.Manager) error { + mapAgentToAgentClassification := func(agent client.Object) []reconcile.Request { + log := logutil.FromContext(context.Background(), r.Log).WithFields( + logrus.Fields{ + "agent": agent.GetName(), + "agent_namespace": agent.GetNamespace(), + }) + acList := &aiv1beta1.AgentClassificationList{} + opts := &client.ListOptions{ + Namespace: agent.GetNamespace(), + } + if err := r.List(context.Background(), acList, opts); err != nil { + log.Debugf("failed to list agent classifications") + return []reconcile.Request{} + } + + reply := make([]reconcile.Request, 0, len(acList.Items)) + for _, classification := range acList.Items { + reply = append(reply, reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: classification.Namespace, + Name: classification.Name, + }}) + } + return reply + } + + return ctrl.NewControllerManagedBy(mgr). + For(&aiv1beta1.AgentClassification{}). + Watches(&source.Kind{Type: &aiv1beta1.Agent{}}, handler.EnqueueRequestsFromMapFunc(mapAgentToAgentClassification)). + Complete(r) +} diff --git a/internal/controller/controllers/agentclassification_controller_test.go b/internal/controller/controllers/agentclassification_controller_test.go new file mode 100644 index 00000000000..3f1e81e17a0 --- /dev/null +++ b/internal/controller/controllers/agentclassification_controller_test.go @@ -0,0 +1,143 @@ +package controllers + +import ( + "context" + + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/openshift/assisted-service/api/v1beta1" + "github.com/openshift/assisted-service/internal/common" + conditionsv1 "github.com/openshift/custom-resource-status/conditions/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func newAgentClassification(name, namespace string, spec v1beta1.AgentClassificationSpec, withFinalizer bool) *v1beta1.AgentClassification { + ac := &v1beta1.AgentClassification{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: spec, + } + if withFinalizer { + ac.ObjectMeta.Finalizers = []string{AgentFinalizerName} // adding finalizer to avoid reconciling twice in the unit tests + } + return ac +} + +func newAgentWithLabel(name, namespace string, key, value string) *v1beta1.Agent { + return &v1beta1.Agent{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: map[string]string{ClassificationLabelPrefix + key: value}, + }, + Spec: v1beta1.AgentSpec{Approved: true}, + Status: v1beta1.AgentStatus{}, + } +} + +func newAgentClassificationRequest(agentClassification *v1beta1.AgentClassification) ctrl.Request { + namespacedName := types.NamespacedName{ + Namespace: agentClassification.Namespace, + Name: agentClassification.Name, + } + return ctrl.Request{NamespacedName: namespacedName} +} + +var _ = Describe("AgentClassification reconcile", func() { + var ( + c client.Client + ir *AgentClassificationReconciler + mockCtrl *gomock.Controller + ctx = context.Background() + defaultClassificationName = "medium-size" + defaultClassificationSpec v1beta1.AgentClassificationSpec + ) + + BeforeEach(func() { + c = fakeclient.NewClientBuilder().Build() + mockCtrl = gomock.NewController(GinkgoT()) + ir = &AgentClassificationReconciler{ + Client: c, + Log: common.GetTestLog(), + } + defaultClassificationSpec = v1beta1.AgentClassificationSpec{ + LabelKey: "size", + LabelValue: "medium", + Query: ".cpu.count == 2 and .memory.physicalBytes >= 4294967296 and .memory.physicalBytes < 8589934592", + } + }) + + AfterEach(func() { + mockCtrl.Finish() + }) + + getTestClassification := func() *v1beta1.AgentClassification { + classification := &v1beta1.AgentClassification{} + Expect(c.Get(ctx, + types.NamespacedName{ + Namespace: testNamespace, + Name: defaultClassificationName, + }, + classification)).To(BeNil()) + return classification + } + + reconcileClassification := func(classification *v1beta1.AgentClassification) { + result, err := ir.Reconcile(ctx, newAgentClassificationRequest(classification)) + Expect(err).To(BeNil()) + Expect(result).To(Equal(ctrl.Result{})) + } + + It("AgentClassification add finalizer", func() { + classification := newAgentClassification(defaultClassificationName, testNamespace, defaultClassificationSpec, false) + Expect(c.Create(ctx, classification)).ShouldNot(HaveOccurred()) + + reconcileClassification(classification) + classification = getTestClassification() + Expect(classification.GetFinalizers()).To(ContainElement(AgentClassificationFinalizer)) + }) + + It("AgentClassification basic flow", func() { + classification := newAgentClassification(defaultClassificationName, testNamespace, defaultClassificationSpec, true) + Expect(c.Create(ctx, classification)).ShouldNot(HaveOccurred()) + + agent1 := newAgentWithLabel("agent1", testNamespace, defaultClassificationSpec.LabelKey, defaultClassificationSpec.LabelValue) + Expect(c.Create(ctx, agent1)).ShouldNot(HaveOccurred()) + agent2 := newAgentWithLabel("agent2", testNamespace, "differentkey", "differentvalue") + Expect(c.Create(ctx, agent2)).ShouldNot(HaveOccurred()) + agent3 := newAgentWithLabel("agent3", testNamespace, defaultClassificationSpec.LabelKey, queryErrorValue("foo")) + Expect(c.Create(ctx, agent3)).ShouldNot(HaveOccurred()) + + reconcileClassification(classification) + + // Expect 1 match (agent1) and 1 error (agent3) + classification = getTestClassification() + Expect(classification.Status.MatchedCount).To(Equal(1)) + Expect(classification.Status.ErrorCount).To(Equal(1)) + Expect(conditionsv1.FindStatusCondition(classification.Status.Conditions, v1beta1.QueryErrorsCondition).Reason).To(Equal(v1beta1.QueryHasErrorsReason)) + + // Deletion should not actually occur because of the finalizer + Expect(c.Delete(ctx, classification)).ShouldNot(HaveOccurred()) + + // Delete agent3 and then expect 1 match and no errors (agent1) + Expect(c.Delete(ctx, agent3)).ShouldNot(HaveOccurred()) + reconcileClassification(classification) + classification = getTestClassification() + Expect(classification.Status.MatchedCount).To(Equal(1)) + Expect(classification.Status.ErrorCount).To(Equal(0)) + Expect(conditionsv1.FindStatusCondition(classification.Status.Conditions, v1beta1.QueryErrorsCondition).Reason).To(Equal(v1beta1.QueryNoErrorsReason)) + + // Delete agent1 and then expect the finalizer to be removed and the classification deleted + Expect(c.Delete(ctx, agent1)).ShouldNot(HaveOccurred()) + reconcileClassification(classification) + classification = getTestClassification() + Expect(classification.GetFinalizers()).ToNot(ContainElement(AgentClassificationFinalizer)) + }) +}) diff --git a/internal/controller/controllers/agentlabel_controller.go b/internal/controller/controllers/agentlabel_controller.go new file mode 100644 index 00000000000..a5a3100ad08 --- /dev/null +++ b/internal/controller/controllers/agentlabel_controller.go @@ -0,0 +1,203 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controllers + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/itchyny/gojq" + aiv1beta1 "github.com/openshift/assisted-service/api/v1beta1" + logutil "github.com/openshift/assisted-service/pkg/log" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" +) + +type AgentLabelReconciler struct { + client.Client + Log logrus.FieldLogger +} + +//+kubebuilder:rbac:groups=agent-install.openshift.io,resources=agents,verbs=get;list;watch;update +//+kubebuilder:rbac:groups=agent-install.openshift.io,resources=agents/status,verbs=get +//+kubebuilder:rbac:groups=agent-install.openshift.io,resources=agents,verbs=get;list;watch +//+kubebuilder:rbac:groups=agent-install.openshift.io,resources=agentclassifications,verbs=get;list;watch;create;update;patch;delete + +func (r *AgentLabelReconciler) Reconcile(origCtx context.Context, req ctrl.Request) (ctrl.Result, error) { + ctx := addRequestIdIfNeeded(origCtx) + log := r.Log.WithFields( + logrus.Fields{ + "agent_label": req.Name, + "agent_label_namespace": req.Namespace, + }) + + defer func() { + log.Info("AgentLabel Reconcile ended") + }() + + log.Info("AgentLabel Reconcile started") + + agent := &aiv1beta1.Agent{} + if err := r.Get(ctx, req.NamespacedName, agent); err != nil { + log.WithError(err).Errorf("Failed to get AgentClassification %s", req.NamespacedName) + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + // Get the inventory into interfaces by way of json marshal/unmarshal + var inventoryInterface interface{} + jsonInventory, _ := json.Marshal(agent.Status.Inventory) + _ = json.Unmarshal(jsonInventory, &inventoryInterface) + + classifications := aiv1beta1.AgentClassificationList{} + opts := &client.ListOptions{ + Namespace: agent.Namespace, + } + err := r.List(ctx, &classifications, opts) + if err != nil { + return ctrl.Result{}, err + } + + changed := false + for _, classification := range classifications.Items { + if !classification.DeletionTimestamp.IsZero() { + log.Infof("classification %s is being deleted", classification.Name) + changed = deleteAgentLabel(log, agent, ClassificationLabelPrefix+classification.Spec.LabelKey, classification.Spec.LabelValue) || changed + continue + } + + query, err := gojq.Parse(classification.Spec.Query) + if err != nil { + // Should not happen - validated via webhook + log.Errorf("Failed to parse query: %s\n", query) + changed = setAgentLabel(log, agent, ClassificationLabelPrefix+classification.Spec.LabelKey, queryErrorValue(classification.Spec.LabelValue)) || changed + continue + } + + matched, err := checkMatch(log, query, inventoryInterface) + if err != nil { + changed = setAgentLabel(log, agent, ClassificationLabelPrefix+classification.Spec.LabelKey, queryErrorValue(classification.Spec.LabelValue)) || changed + } else if !matched { + changed = deleteAgentLabel(log, agent, ClassificationLabelPrefix+classification.Spec.LabelKey, classification.Spec.LabelValue) || changed + } else { + changed = setAgentLabel(log, agent, ClassificationLabelPrefix+classification.Spec.LabelKey, classification.Spec.LabelValue) || changed + } + } + + if changed { + if err := r.Update(ctx, agent); err != nil { + log.WithError(err).Error("failed to update agent") + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, nil +} + +func queryErrorValue(originalValue string) string { + return fmt.Sprintf("QUERYERROR-%s", originalValue) +} + +func checkMatch(log *logrus.Entry, query *gojq.Query, inventoryInterface interface{}) (bool, error) { + iter := query.Run(inventoryInterface) + values := []interface{}{} + for { + v, ok := iter.Next() + if !ok { + break + } + if err, ok := v.(error); ok { + return false, err + } + values = append(values, v) + } + if len(values) == 0 { + return false, errors.New("Expected boolean, found no values") + } + if len(values) > 1 { + return false, errors.New("Expected boolean, found multiple values") + } + value := values[0] + if res, ok := value.(bool); ok { + if res { + return true, nil + } + } + return false, nil +} + +func deleteAgentLabel(log *logrus.Entry, agent *aiv1beta1.Agent, labelKey, labelValue string) bool { + labels := agent.GetLabels() + + if labels == nil { + return false + } + + if _, ok := labels[labelKey]; !ok { + return false + } + + // If the label has a different value, then don't delete + if val, ok := labels[labelKey]; ok { + if val != labelValue && val != queryErrorValue(labelValue) { + return false + } + } + + delete(labels, labelKey) + agent.SetLabels(labels) + log.Infof("Deleted label %s from agent %s/%s", labelKey, agent.Namespace, agent.Name) + return true +} + +func (r *AgentLabelReconciler) SetupWithManager(mgr ctrl.Manager) error { + mapAgentClassificationToAgent := func(classification client.Object) []reconcile.Request { + log := logutil.FromContext(context.Background(), r.Log).WithFields( + logrus.Fields{ + "classification": classification.GetName(), + "classification_namespace": classification.GetNamespace(), + }) + agentList := &aiv1beta1.AgentList{} + opts := &client.ListOptions{ + Namespace: classification.GetNamespace(), + } + if err := r.List(context.Background(), agentList, opts); err != nil { + log.Debugf("failed to list agents") + return []reconcile.Request{} + } + + reply := make([]reconcile.Request, 0, len(agentList.Items)) + for _, agent := range agentList.Items { + reply = append(reply, reconcile.Request{NamespacedName: types.NamespacedName{ + Namespace: agent.Namespace, + Name: agent.Name, + }}) + } + return reply + } + + return ctrl.NewControllerManagedBy(mgr). + For(&aiv1beta1.Agent{}). + Watches(&source.Kind{Type: &aiv1beta1.AgentClassification{}}, handler.EnqueueRequestsFromMapFunc(mapAgentClassificationToAgent)). + Complete(r) +} diff --git a/internal/controller/controllers/agentlabel_controller_test.go b/internal/controller/controllers/agentlabel_controller_test.go new file mode 100644 index 00000000000..8c3abb1711b --- /dev/null +++ b/internal/controller/controllers/agentlabel_controller_test.go @@ -0,0 +1,133 @@ +package controllers + +import ( + "context" + + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/openshift/assisted-service/api/v1beta1" + "github.com/openshift/assisted-service/internal/common" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func newAgentWithInventory(name, namespace string, cpu, ram int64) *v1beta1.Agent { + return &v1beta1.Agent{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: v1beta1.AgentSpec{Approved: true}, + Status: v1beta1.AgentStatus{ + Inventory: v1beta1.HostInventory{ + Hostname: name, + Cpu: v1beta1.HostCPU{Count: cpu}, + Memory: v1beta1.HostMemory{PhysicalBytes: ram}, + }, + }, + } +} + +func newAgentRequest(agent *v1beta1.Agent) ctrl.Request { + namespacedName := types.NamespacedName{ + Namespace: agent.Namespace, + Name: agent.Name, + } + return ctrl.Request{NamespacedName: namespacedName} +} + +var _ = Describe("AgentLabel reconcile", func() { + var ( + c client.Client + ir *AgentLabelReconciler + mockCtrl *gomock.Controller + ctx = context.Background() + agentName = "agent" + ) + + BeforeEach(func() { + c = fakeclient.NewClientBuilder().Build() + mockCtrl = gomock.NewController(GinkgoT()) + ir = &AgentLabelReconciler{ + Client: c, + Log: common.GetTestLog(), + } + + }) + + AfterEach(func() { + mockCtrl.Finish() + }) + + getTestAgent := func() *v1beta1.Agent { + classification := &v1beta1.Agent{} + Expect(c.Get(ctx, + types.NamespacedName{ + Namespace: testNamespace, + Name: agentName, + }, + classification)).To(BeNil()) + return classification + } + + reconcileAgent := func(agent *v1beta1.Agent) { + result, err := ir.Reconcile(ctx, newAgentRequest(agent)) + Expect(err).To(BeNil()) + Expect(result).To(Equal(ctrl.Result{})) + } + + It("AgentLabel basic flow", func() { + mediumQuery := ".cpu.count == 2 and .memory.physicalBytes >= 4294967296 and .memory.physicalBytes < 8589934592" + xlargeQuery := ".cpu.count == 4 and .memory.physicalBytes >= 17179869184 and .memory.physicalBytes < 34359738368" + classificationSpecMedium := v1beta1.AgentClassificationSpec{ + LabelKey: "size", + LabelValue: "medium", + Query: mediumQuery, + } + classificationMedium := newAgentClassification(classificationSpecMedium.LabelValue, testNamespace, classificationSpecMedium, true) + Expect(c.Create(ctx, classificationMedium)).ShouldNot(HaveOccurred()) + + classificationSpecXlarge := v1beta1.AgentClassificationSpec{ + LabelKey: "size", + LabelValue: "xlarge", + Query: xlargeQuery, + } + classificationXlarge := newAgentClassification(classificationSpecXlarge.LabelValue, testNamespace, classificationSpecXlarge, true) + Expect(c.Create(ctx, classificationXlarge)).ShouldNot(HaveOccurred()) + + classificationSpecError := v1beta1.AgentClassificationSpec{ + LabelKey: "error", + LabelValue: "error", + Query: ".cpu.count & .memory.physicalBytes", + } + classificationError := newAgentClassification(classificationSpecError.LabelValue, testNamespace, classificationSpecError, true) + Expect(c.Create(ctx, classificationError)).ShouldNot(HaveOccurred()) + + agent := newAgentWithInventory(agentName, testNamespace, 2, 4294967296) + Expect(c.Create(ctx, agent)).ShouldNot(HaveOccurred()) + + // For the first reconcile, we expect the "size=medium" label and a "size=QUERYERROR..." label + reconcileAgent(agent) + agent = getTestAgent() + Expect(len(agent.GetLabels())).To(Equal(2)) + Expect(agent.GetLabels()[ClassificationLabelPrefix+"size"]).To(Equal("medium")) + Expect(agent.GetLabels()[ClassificationLabelPrefix+"error"]).To(Equal(queryErrorValue("error"))) + + // Delete the error classification and swap the queries of medium and xlarge, so now we should have + // one label - xlarge + Expect(c.Delete(ctx, classificationError)).ShouldNot(HaveOccurred()) + classificationMedium.Spec.Query = xlargeQuery + Expect(c.Update(ctx, classificationMedium)).ShouldNot(HaveOccurred()) + classificationXlarge.Spec.Query = mediumQuery + Expect(c.Update(ctx, classificationXlarge)).ShouldNot(HaveOccurred()) + + reconcileAgent(agent) + agent = getTestAgent() + Expect(len(agent.GetLabels())).To(Equal(1)) + Expect(agent.GetLabels()[ClassificationLabelPrefix+"size"]).To(Equal("xlarge")) + }) +}) diff --git a/pkg/validating-webhooks/agentinstall/v1beta1/agent_classification_admission_hook.go b/pkg/validating-webhooks/agentinstall/v1beta1/agent_classification_admission_hook.go new file mode 100644 index 00000000000..c3173fe9033 --- /dev/null +++ b/pkg/validating-webhooks/agentinstall/v1beta1/agent_classification_admission_hook.go @@ -0,0 +1,238 @@ +package v1beta1 + +import ( + "net/http" + "strings" + + "github.com/itchyny/gojq" + "github.com/openshift/assisted-service/api/v1beta1" + log "github.com/sirupsen/logrus" + admissionv1 "k8s.io/api/admission/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/validation" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/validation/field" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" +) + +const ( + agentClassificationResource = "agentclassifications" + agentClassificationAdmissionGroup = "admission.agentinstall.openshift.io" + agentClassificationAdmissionVersion = "v1" + ClassificationLabelPrefix = "agentclassification." + v1beta1.Group + "/" +) + +// AgentClassificationValidatingAdmissionHook is a struct that is used to reference what code should be run by the generic-admission-server. +type AgentClassificationValidatingAdmissionHook struct { + decoder *admission.Decoder +} + +// NewAgentClassificationValidatingAdmissionHook constructs a new AgentClassificationValidatingAdmissionHook +func NewAgentClassificationValidatingAdmissionHook(decoder *admission.Decoder) *AgentClassificationValidatingAdmissionHook { + return &AgentClassificationValidatingAdmissionHook{decoder: decoder} +} + +// ValidatingResource is called by generic-admission-server on startup to register the returned REST resource through which the +// webhook is accessed by the kube apiserver. +// For example, generic-admission-server uses the data below to register the webhook on the REST resource "/apis/admission.agentinstall.openshift.io/v1/agentclassificationvalidators". +// When the kube apiserver calls this registered REST resource, the generic-admission-server calls the Validate() method below. +func (a *AgentClassificationValidatingAdmissionHook) ValidatingResource() (plural schema.GroupVersionResource, singular string) { + log.WithFields(log.Fields{ + "group": agentClassificationAdmissionGroup, + "version": agentClassificationAdmissionVersion, + "resource": "agentclassificationvalidator", + }).Info("Registering validation REST resource") + // NOTE: This GVR is meant to be different than the AgentClassification CRD GVR which has group "agent-install.openshift.io". + return schema.GroupVersionResource{ + Group: agentClassificationAdmissionGroup, + Version: agentClassificationAdmissionVersion, + Resource: "agentclassificationvalidators", + }, + "agentclassificationvalidator" +} + +// Initialize is called by generic-admission-server on startup to setup any special initialization that your webhook needs. +func (a *AgentClassificationValidatingAdmissionHook) Initialize(kubeClientConfig *rest.Config, stopCh <-chan struct{}) error { + log.WithFields(log.Fields{ + "group": agentClassificationAdmissionGroup, + "version": agentClassificationAdmissionVersion, + "resource": "agentclassificationvalidator", + }).Info("Initializing validation REST resource") + return nil // No initialization needed right now. +} + +// Validate is called by generic-admission-server when the registered REST resource above is called with an admission request. +// Usually it's the kube apiserver that is making the admission validation request. +func (a *AgentClassificationValidatingAdmissionHook) Validate(admissionSpec *admissionv1.AdmissionRequest) *admissionv1.AdmissionResponse { + contextLogger := log.WithFields(log.Fields{ + "operation": admissionSpec.Operation, + "group": admissionSpec.Resource.Group, + "version": admissionSpec.Resource.Version, + "resource": admissionSpec.Resource.Resource, + "method": "Validate", + }) + + if !a.shouldValidate(admissionSpec) { + contextLogger.Info("Skipping validation for request") + // The request object isn't something that this validator should validate. + // Therefore, we say that it's allowed. + return &admissionv1.AdmissionResponse{ + Allowed: true, + } + } + + contextLogger.Info("Validating request") + + if admissionSpec.Operation == admissionv1.Create { + return a.validateCreate(admissionSpec) + } + + if admissionSpec.Operation == admissionv1.Update { + return a.validateUpdate(admissionSpec) + } + + // We're only validating updates at this time, so all other operations are explicitly allowed. + contextLogger.Info("Successful validation") + return &admissionv1.AdmissionResponse{ + Allowed: true, + } +} + +// shouldValidate explicitly checks if the request should be validated. For example, this webhook may have accidentally been registered to check +// the validity of some other type of object with a different GVR. +func (a *AgentClassificationValidatingAdmissionHook) shouldValidate(admissionSpec *admissionv1.AdmissionRequest) bool { + contextLogger := log.WithFields(log.Fields{ + "operation": admissionSpec.Operation, + "group": admissionSpec.Resource.Group, + "version": admissionSpec.Resource.Version, + "resource": admissionSpec.Resource.Resource, + "method": "shouldValidate", + }) + + if admissionSpec.Resource.Group != v1beta1.Group { + contextLogger.Debug("Returning False, not our group") + return false + } + + if admissionSpec.Resource.Version != v1beta1.Version { + contextLogger.Debug("Returning False, it's our group, but not the right version") + return false + } + + if admissionSpec.Resource.Resource != agentClassificationResource { + contextLogger.Debug("Returning False, it's our group and version, but not the right resource") + return false + } + + // If we get here, then we're supposed to validate the object. + contextLogger.Debug("Returning True, passed all prerequisites.") + return true +} + +// validateCreate specifically validates create operations for AgentClassification objects. +func (a *AgentClassificationValidatingAdmissionHook) validateCreate(admissionSpec *admissionv1.AdmissionRequest) *admissionv1.AdmissionResponse { + contextLogger := log.WithFields(log.Fields{ + "operation": admissionSpec.Operation, + "group": admissionSpec.Resource.Group, + "version": admissionSpec.Resource.Version, + "resource": admissionSpec.Resource.Resource, + "method": "validateCreate", + }) + + newObject := &v1beta1.AgentClassification{} + if err := a.decoder.DecodeRaw(admissionSpec.Object, newObject); err != nil { + contextLogger.Errorf("Failed unmarshaling Object: %v", err.Error()) + return &admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Status: metav1.StatusFailure, Code: http.StatusBadRequest, Reason: metav1.StatusReasonBadRequest, + Message: err.Error(), + }, + } + } + + // Validate the specified label key and value + f := field.NewPath("spec") + errs := validation.ValidateLabels(map[string]string{ClassificationLabelPrefix + newObject.Spec.LabelKey: newObject.Spec.LabelValue}, f) + if strings.HasPrefix(newObject.Spec.LabelValue, "QUERYERROR") { + errs = append(errs, field.Invalid(f, newObject.Spec.LabelValue, "label must not start with QUERYERROR as this is reserved")) + } + + // Validate that we can parse the specified query + _, err := gojq.Parse(newObject.Spec.Query) + if err != nil { + errs = append(errs, field.Invalid(f, newObject.Spec.Query, err.Error())) + } + + if len(errs) > 0 { + contextLogger.Infof("Validation failed: %s", errs.ToAggregate().Error()) + return &admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Status: metav1.StatusFailure, Code: http.StatusBadRequest, Reason: metav1.StatusReasonBadRequest, + Message: errs.ToAggregate().Error(), + }, + } + } + + contextLogger.Info("Successful validation") + return &admissionv1.AdmissionResponse{ + Allowed: true, + } +} + +// validateUpdate specifically validates update operations for AgentClassification objects. +func (a *AgentClassificationValidatingAdmissionHook) validateUpdate(admissionSpec *admissionv1.AdmissionRequest) *admissionv1.AdmissionResponse { + contextLogger := log.WithFields(log.Fields{ + "operation": admissionSpec.Operation, + "group": admissionSpec.Resource.Group, + "version": admissionSpec.Resource.Version, + "resource": admissionSpec.Resource.Resource, + "method": "validateUpdate", + }) + + newObject := &v1beta1.AgentClassification{} + if err := a.decoder.DecodeRaw(admissionSpec.Object, newObject); err != nil { + contextLogger.Errorf("Failed unmarshaling Object: %v", err.Error()) + return &admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Status: metav1.StatusFailure, Code: http.StatusBadRequest, Reason: metav1.StatusReasonBadRequest, + Message: err.Error(), + }, + } + } + + // Add the new data to the contextLogger + contextLogger.Data["object.Name"] = newObject.Name + + oldObject := &v1beta1.AgentClassification{} + if err := a.decoder.DecodeRaw(admissionSpec.OldObject, oldObject); err != nil { + contextLogger.Errorf("Failed unmarshaling OldObject: %v", err.Error()) + return &admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Status: metav1.StatusFailure, Code: http.StatusBadRequest, Reason: metav1.StatusReasonBadRequest, + Message: err.Error(), + }, + } + } + + // Validate that the label key and value haven't changed + if (oldObject.Spec.LabelKey != newObject.Spec.LabelKey) || (oldObject.Spec.LabelValue != newObject.Spec.LabelValue) { + return &admissionv1.AdmissionResponse{ + Allowed: false, + Result: &metav1.Status{ + Status: metav1.StatusFailure, Code: http.StatusBadRequest, Reason: metav1.StatusReasonBadRequest, + Message: "Label modified: the specified label may not be modified after creation", + }, + } + } + + // If we get here, then all checks passed, so the object is valid. + contextLogger.Info("Successful validation") + return &admissionv1.AdmissionResponse{ + Allowed: true, + } +} diff --git a/pkg/validating-webhooks/agentinstall/v1beta1/agent_classification_admission_hook_test.go b/pkg/validating-webhooks/agentinstall/v1beta1/agent_classification_admission_hook_test.go new file mode 100644 index 00000000000..491ae9ef3ee --- /dev/null +++ b/pkg/validating-webhooks/agentinstall/v1beta1/agent_classification_admission_hook_test.go @@ -0,0 +1,259 @@ +package v1beta1 + +import ( + "encoding/json" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + v1beta1 "github.com/openshift/assisted-service/api/v1beta1" + apiserver "github.com/openshift/generic-admission-server/pkg/apiserver" + admissionv1 "k8s.io/api/admission/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" +) + +var _ = Describe("agent classification web hook init", func() { + It("ValidatingResource", func() { + data := NewAgentClassificationValidatingAdmissionHook(createDecoder()) + expectedPlural := schema.GroupVersionResource{ + Group: "admission.agentinstall.openshift.io", + Version: "v1", + Resource: "agentclassificationvalidators", + } + expectedSingular := "agentclassificationvalidator" + + plural, singular := data.ValidatingResource() + Expect(plural).To(Equal(expectedPlural)) + Expect(singular).To(Equal(expectedSingular)) + + }) + + It("Initialize", func() { + data := NewAgentClassificationValidatingAdmissionHook(createDecoder()) + err := data.Initialize(nil, nil) + Expect(err).To(BeNil()) + }) + + It("Check implements interface ", func() { + var hook interface{} = NewAgentClassificationValidatingAdmissionHook(createDecoder()) + _, ok := hook.(apiserver.ValidatingAdmissionHookV1) + Expect(ok).To(BeTrue()) + }) +}) + +var _ = Describe("agent classification web validate", func() { + var ( + validKey = "size" + invalidKey = "s!ze" + keyWithPrefix = "a/size" + validValue = "medium" + invalidValue = "med!um" + validQuery = ".cpu.count == 2 and .memory.physicalBytes >= 4294967296 and .memory.physicalBytes < 8589934592" + invalidQuery = ".cpu.count == 2 and" + ) + cases := []struct { + name string + newSpec v1beta1.AgentClassificationSpec + newObjectRaw []byte + oldSpec v1beta1.AgentClassificationSpec + oldObjectRaw []byte + operation admissionv1.Operation + expectedAllowed bool + gvr *metav1.GroupVersionResource + }{ + { + name: "Test doesn't validate with right version and resource, but wrong group", + gvr: &metav1.GroupVersionResource{ + Group: "not the right group", + Version: "v1beta1", + Resource: "agentclassifications", + }, + expectedAllowed: true, + }, + { + name: "Test doesn't validate with right group and resource, wrong version", + gvr: &metav1.GroupVersionResource{ + Group: "agent-install.openshift.io", + Version: "not the right version", + Resource: "agentclassifications", + }, + expectedAllowed: true, + }, + { + name: "Test doesn't validate with right group and version, wrong resource", + gvr: &metav1.GroupVersionResource{ + Group: "agent-install.openshift.io", + Version: "v1beta1", + Resource: "not the right resource", + }, + expectedAllowed: true, + }, + { + name: "Test AgentClassification Spec is valid on create", + newSpec: v1beta1.AgentClassificationSpec{ + LabelKey: validKey, + LabelValue: validValue, + Query: validQuery, + }, + oldSpec: v1beta1.AgentClassificationSpec{}, + operation: admissionv1.Create, + expectedAllowed: true, + }, + { + name: "Test AgentClassification label key is invalid on create", + newSpec: v1beta1.AgentClassificationSpec{ + LabelKey: invalidKey, + LabelValue: validValue, + Query: validQuery, + }, + oldSpec: v1beta1.AgentClassificationSpec{}, + operation: admissionv1.Create, + expectedAllowed: false, + }, + { + name: "Test AgentClassification label key has a prefix on create", + newSpec: v1beta1.AgentClassificationSpec{ + LabelKey: keyWithPrefix, + LabelValue: validValue, + Query: validQuery, + }, + oldSpec: v1beta1.AgentClassificationSpec{}, + operation: admissionv1.Create, + expectedAllowed: false, + }, + { + name: "Test AgentClassification label value is invalid on create", + newSpec: v1beta1.AgentClassificationSpec{ + LabelKey: validKey, + LabelValue: invalidValue, + Query: validQuery, + }, + oldSpec: v1beta1.AgentClassificationSpec{}, + operation: admissionv1.Create, + expectedAllowed: false, + }, + { + name: "Test AgentClassification label value is query error on create", + newSpec: v1beta1.AgentClassificationSpec{ + LabelKey: validKey, + LabelValue: "QUERYERROR-foo", + Query: validQuery, + }, + oldSpec: v1beta1.AgentClassificationSpec{}, + operation: admissionv1.Create, + expectedAllowed: false, + }, + { + name: "Test AgentClassification query is invalid on create", + newSpec: v1beta1.AgentClassificationSpec{ + LabelKey: validKey, + LabelValue: validValue, + Query: invalidQuery, + }, + oldSpec: v1beta1.AgentClassificationSpec{}, + operation: admissionv1.Create, + expectedAllowed: false, + }, + { + name: "Test AgentClassification everything is invalid on create", + newSpec: v1beta1.AgentClassificationSpec{ + LabelKey: invalidKey, + LabelValue: invalidValue, + Query: invalidQuery, + }, + oldSpec: v1beta1.AgentClassificationSpec{}, + operation: admissionv1.Create, + expectedAllowed: false, + }, + { + name: "Test AgentClassification label key is changed on update", + newSpec: v1beta1.AgentClassificationSpec{ + LabelKey: "newkey", + LabelValue: validValue, + Query: validQuery, + }, + oldSpec: v1beta1.AgentClassificationSpec{ + LabelKey: validKey, + LabelValue: validValue, + Query: validQuery, + }, + operation: admissionv1.Update, + expectedAllowed: false, + }, + { + name: "Test AgentClassification label value is changed on update", + newSpec: v1beta1.AgentClassificationSpec{ + LabelKey: validKey, + LabelValue: "newvalue", + Query: validQuery, + }, + oldSpec: v1beta1.AgentClassificationSpec{ + LabelKey: validKey, + LabelValue: validValue, + Query: validQuery, + }, + operation: admissionv1.Update, + expectedAllowed: false, + }, + { + name: "Test AgentClassification query is changed on update", + newSpec: v1beta1.AgentClassificationSpec{ + LabelKey: validKey, + LabelValue: validValue, + Query: validQuery, + }, + oldSpec: v1beta1.AgentClassificationSpec{ + LabelKey: validKey, + LabelValue: validValue, + Query: ".cpu.count == 2", + }, + operation: admissionv1.Update, + expectedAllowed: true, + }, + } + + for i := range cases { + tc := cases[i] + It(tc.name, func() { + data := NewAgentClassificationValidatingAdmissionHook(createDecoder()) + newObject := &v1beta1.AgentClassification{ + Spec: tc.newSpec, + } + oldObject := &v1beta1.AgentClassification{ + Spec: tc.oldSpec, + } + + if tc.newObjectRaw == nil { + tc.newObjectRaw, _ = json.Marshal(newObject) + } + + if tc.oldObjectRaw == nil { + tc.oldObjectRaw, _ = json.Marshal(oldObject) + } + + if tc.gvr == nil { + tc.gvr = &metav1.GroupVersionResource{ + Group: "agent-install.openshift.io", + Version: "v1beta1", + Resource: "agentclassifications", + } + } + + request := &admissionv1.AdmissionRequest{ + Operation: tc.operation, + Resource: *tc.gvr, + Object: runtime.RawExtension{ + Raw: tc.newObjectRaw, + }, + OldObject: runtime.RawExtension{ + Raw: tc.oldObjectRaw, + }, + } + + response := data.Validate(request) + Expect(response.Allowed).To(Equal(tc.expectedAllowed)) + }) + } + +}) diff --git a/subsystem/cluster_test.go b/subsystem/cluster_test.go index e902857d974..2a3137bc175 100644 --- a/subsystem/cluster_test.go +++ b/subsystem/cluster_test.go @@ -93,7 +93,7 @@ var ( } validHwInfo = &models.Inventory{ - CPU: &models.CPU{Count: 16}, + CPU: &models.CPU{Count: 16, Architecture: "x86_64"}, Memory: &models.Memory{PhysicalBytes: int64(32 * units.GiB), UsableBytes: int64(32 * units.GiB)}, Disks: []*models.Disk{&loop0, &sdb}, Interfaces: []*models.Interface{ diff --git a/subsystem/kubeapi_test.go b/subsystem/kubeapi_test.go index 881527fe638..88ddf1c05f2 100644 --- a/subsystem/kubeapi_test.go +++ b/subsystem/kubeapi_test.go @@ -10,9 +10,11 @@ import ( "net/http" "os" "reflect" + "strconv" "strings" "time" + "github.com/alecthomas/units" "github.com/go-openapi/strfmt" "github.com/go-openapi/swag" "github.com/google/uuid" @@ -625,6 +627,7 @@ func printCRs(ctx context.Context, client k8sclient.Client) { infraEnvList v1beta1.InfraEnvList bareMetalHostList bmhv1alpha1.BareMetalHostList nmStateConfigList v1beta1.NMStateConfigList + classificationList v1beta1.AgentClassificationList clusterImageSetList hivev1.ClusterImageSetList clusterDeploymentList hivev1.ClusterDeploymentList ) @@ -647,6 +650,9 @@ func printCRs(ctx context.Context, client k8sclient.Client) { multiErr = multierror.Append(multiErr, client.List(ctx, &nmStateConfigList, k8sclient.InNamespace(Options.Namespace))) multiErr = multierror.Append(multiErr, GinkgoResourceLogger("NMStateConfig", nmStateConfigList)) + multiErr = multierror.Append(multiErr, client.List(ctx, &classificationList, k8sclient.InNamespace(Options.Namespace))) + multiErr = multierror.Append(multiErr, GinkgoResourceLogger("AgentClassification", classificationList)) + multiErr = multierror.Append(multiErr, client.List(ctx, &bareMetalHostList, k8sclient.InNamespace(Options.Namespace))) multiErr = multierror.Append(multiErr, GinkgoResourceLogger("BareMetalHost", bareMetalHostList)) @@ -667,7 +673,9 @@ func cleanUpCRs(ctx context.Context, client k8sclient.Client) { Eventually(func() error { return client.DeleteAllOf(ctx, &v1beta1.NMStateConfig{}, k8sclient.InNamespace(Options.Namespace)) }, "1m", "2s").Should(BeNil()) - + Eventually(func() error { + return client.DeleteAllOf(ctx, &v1beta1.AgentClassification{}, k8sclient.InNamespace(Options.Namespace)) + }, "1m", "2s").Should(BeNil()) Eventually(func() error { return client.DeleteAllOf(ctx, &bmhv1alpha1.BareMetalHost{}, k8sclient.InNamespace(Options.Namespace)) }, "1m", "2s").Should(BeNil()) @@ -747,6 +755,14 @@ func verifyCleanUP(ctx context.Context, client k8sclient.Client) { return len(agentList.Items) }, "2m", "2s").Should(Equal(0)) + By("Verify AgentClassification Cleanup") + Eventually(func() int { + classificationList := &v1beta1.AgentClassificationList{} + err := client.List(ctx, classificationList, k8sclient.InNamespace(Options.Namespace)) + Expect(err).To(BeNil()) + return len(classificationList.Items) + }, "2m", "2s").Should(Equal(0)) + By("Verify BareMetalHost Cleanup") Eventually(func() int { bareMetalHostList := &bmhv1alpha1.BareMetalHostList{} @@ -1173,7 +1189,7 @@ var _ = Describe("[kube-api]cluster installation", func() { By("Verify ISO URL is populated") Eventually(func() string { return getInfraEnvCRD(ctx, kubeClient, infraEnvKubeName).Status.ISODownloadURL - }, "15s", "5s").Should(Not(BeEmpty())) + }, "15s", "1s").Should(Not(BeEmpty())) By("Verify infraEnv has no reference to CD") infraEnvCr := getInfraEnvCRD(ctx, kubeClient, infraEnvKubeName) @@ -1191,7 +1207,7 @@ var _ = Describe("[kube-api]cluster installation", func() { hwInfo.Interfaces[0].IPV4Addresses = []string{defaultCIDRv4} generateHWPostStepReply(ctx, host, hwInfo, "hostname1") - By("Verify agent and host are not bind") + By("Verify agent and host are not bound") h, err := common.GetHostFromDB(db, infraEnv.ID.String(), host.ID.String()) Expect(err).To(BeNil()) Expect(h.ClusterID).To(BeNil()) @@ -1202,11 +1218,78 @@ var _ = Describe("[kube-api]cluster installation", func() { Eventually(func() bool { agent := getAgentCRD(ctx, kubeClient, key) return agent.Spec.ClusterDeploymentName == nil - }, "30s", "10s").Should(BeTrue()) + }, "30s", "1s").Should(BeTrue()) checkAgentCondition(ctx, host.ID.String(), v1beta1.BoundCondition, v1beta1.UnboundReason) }) + It("Agent labels", func() { + By("Deploy InfraEnv") + infraEnvSpec.ClusterRef = nil + deployInfraEnvCRD(ctx, kubeClient, infraNsName.Name, infraEnvSpec) + + infraEnvKubeName := types.NamespacedName{ + Namespace: Options.Namespace, + Name: infraNsName.Name, + } + + By("Verify ISO URL is populated") + Eventually(func() string { + return getInfraEnvCRD(ctx, kubeClient, infraEnvKubeName).Status.ISODownloadURL + }, "15s", "1s").Should(Not(BeEmpty())) + + By("Register Agent to InfraEnv") + infraEnvKey := types.NamespacedName{ + Namespace: Options.Namespace, + Name: infraNsName.Name, + } + infraEnv := getInfraEnvFromDBByKubeKey(ctx, db, infraEnvKey, waitForReconcileTimeout) + configureLocalAgentClient(infraEnv.ID.String()) + host := ®isterHost(*infraEnv.ID).Host + hwInfo := validHwInfo + hwInfo.Interfaces[0].IPV4Addresses = []string{defaultCIDRv4} + generateHWPostStepReply(ctx, host, hwInfo, "hostname1") + + By("Verify agent inventory labels") + key := types.NamespacedName{ + Namespace: Options.Namespace, + Name: host.ID.String(), + } + var agentLabels map[string]string + Eventually(func() bool { + agent := getAgentCRD(ctx, kubeClient, key) + agentLabels = agent.GetLabels() + _, ok := agentLabels["inventory.agent-install.openshift.io/storage-hasnonrotationaldisk"] + return ok + }, "30s", "1s").Should(BeTrue()) + Expect(agentLabels["inventory.agent-install.openshift.io/storage-hasnonrotationaldisk"]).To(Equal("true")) + Expect(agentLabels["inventory.agent-install.openshift.io/cpu-architecture"]).To(Equal("x86_64")) + Expect(agentLabels["inventory.agent-install.openshift.io/cpu-virtenabled"]).To(Equal("false")) + Expect(agentLabels["inventory.agent-install.openshift.io/host-manufacturer"]).To(Equal(validHwInfo.SystemVendor.Manufacturer)) + Expect(agentLabels["inventory.agent-install.openshift.io/host-productname"]).To(Equal(validHwInfo.SystemVendor.ProductName)) + Expect(agentLabels["inventory.agent-install.openshift.io/host-isvirtual"]).To(Equal(strconv.FormatBool(validHwInfo.SystemVendor.Virtual))) + + By("Verify agent classification labels") + classificationXXL := v1beta1.AgentClassification{ + ObjectMeta: metav1.ObjectMeta{Name: "xxl", Namespace: Options.Namespace}, + Spec: v1beta1.AgentClassificationSpec{ + LabelKey: "size", + LabelValue: "xxl", + Query: fmt.Sprintf(".cpu.count == 16 and .memory.physicalBytes >= %d and .memory.physicalBytes < %d", int64(31*units.GiB), int64(33*units.GiB)), + }, + } + err := kubeClient.Create(ctx, &classificationXXL) + Expect(err).To(BeNil()) + + Eventually(func() bool { + agent := getAgentCRD(ctx, kubeClient, key) + agentLabels = agent.GetLabels() + _, ok := agentLabels["agentclassification.agent-install.openshift.io/size"] + return ok + }, "30s", "1s").Should(BeTrue()) + Expect(agentLabels["agentclassification.agent-install.openshift.io/size"]).To(Equal("xxl")) + }) + It("[kube-cpu-arch]create infra-env with arm64 cpu arch", func() { infraEnvSpec.ClusterRef = nil infraEnvSpec.CpuArchitecture = "arm64" @@ -1218,7 +1301,7 @@ var _ = Describe("[kube-api]cluster installation", func() { } Eventually(func() string { return getInfraEnvCRD(ctx, kubeClient, infraEnvKey).Status.ISODownloadURL - }, "15s", "5s").Should(Not(BeEmpty())) + }, "15s", "1s").Should(Not(BeEmpty())) infraEnv := getInfraEnvFromDBByKubeKey(ctx, db, infraEnvKey, waitForReconcileTimeout) Expect(infraEnv.CPUArchitecture).To(Equal("arm64"))