From 2675af9855f740fbdd252be83e3a8e98da093cf7 Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Wed, 20 Nov 2024 11:31:19 +0100 Subject: [PATCH 1/2] Reconcile ipset when osnetconfig changes When e.g. a network definition changes, the IPsets have to reconcile to update its information. This allows changing network definition, but needs to be done carefully and tests in a pre-prod env as it is an invasive change. The process looks like this: * update the OpenStackNetConfig to do the network change * validate the IPSet for the role and the OpenStackNetConfig status reflects the new IPs for the nodes ``` 172.18.3.20/24 ``` * Make sure you have a parameter file to allow update the network config for the role: ``` network-environment.yaml: | parameter_defaults: ControllerNetworkConfigUpdate: True ComputeNetworkConfigUpdate: True ComputeLeaf1NetworkConfigUpdate: True ``` * use the OpenStackConfigGenerator resource to create a new configversion * when config generator process finished, depending on the number of changed, the new created openstackconfigversion will show the diff (gets trunkated if it is too big) ``` diff: | diff --git a/source-templates/environments/deployed-network-environment.yaml b/source-templates/environments/deployed-network-environment.yaml index 168a9242caa93630d17ade3d254604b967a3a3f0..d08508054b2fc8e01d8f6499b08fba25fbe84860 100644 --- a/source-templates/environments/deployed-network-environment.yaml +++ b/source-templates/environments/deployed-network-environment.yaml @@ -99,2 +98,2 @@ storage_subnet: - cidr: '172.18.3.0/24' - gateway_ip: '172.18.3.1' + cidr: '172.18.0.0/24' + gateway_ip: '172.18.0.1' @@ -208 +207 @@ storage: - - 172.18.3.0/24 + - 172.18.0.0/24 ... ``` * use the OpenStackDeploy resource to apply the generated configversion to the overcloud Notes: * from checking the overcloud node, the /etc/os-net-config/config.yaml and /etc/sysconfig/network-scripts/ifcfg-enp5s0 are correct. When checking the running interface configuration we still see the old IP address 172.18.0.10. Reboot the node/if-down|if-up would fix it. This is not an OSPdO issue. 6: enp5s0: mtu 9000 qdisc mq state UP group default qlen 1000 link/ether 02:c8:20:00:00:04 brd ff:ff:ff:ff:ff:ff inet 172.18.0.10/32 brd 172.18.0.255 scope global enp5s0 valid_lft forever preferred_lft forever inet 172.18.3.20/24 scope global enp5s0 valid_lft forever preferred_lft forever inet 172.18.3.10/32 scope global enp5s0 valid_lft forever preferred_lft forever inet6 fe80::c8:20ff:fe00:4/64 scope link valid_lft forever preferred_lft forever * General on changing networks. when e.g. removing a network from the openstackclient will still have a reservation on that network resolved: https://bugzilla.redhat.com/show_bug.cgi?id=2325902 (cherry picked from commit 961c0ed11a53a38b7e5f6b38f7a534953b8e156b) --- controllers/openstackipset_controller.go | 42 +++++++++++++++++++++++- main.go | 3 +- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/controllers/openstackipset_controller.go b/controllers/openstackipset_controller.go index d1505f72..43421408 100644 --- a/controllers/openstackipset_controller.go +++ b/controllers/openstackipset_controller.go @@ -31,8 +31,10 @@ import ( "k8s.io/utils/diff" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" "github.com/go-logr/logr" metal3v1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1" @@ -321,9 +323,47 @@ func (r *OpenStackIPSetReconciler) getNormalizedStatus(status *ospdirectorv1beta } // SetupWithManager sets up the controller with the Manager. -func (r *OpenStackIPSetReconciler) SetupWithManager(mgr ctrl.Manager) error { +func (r *OpenStackIPSetReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + Log := r.GetLogger() + ipsetFN := handler.EnqueueRequestsFromMapFunc(func(o client.Object) []reconcile.Request { + result := []reconcile.Request{} + + // For each NetConfig event get the list of all + // OpenStackIPSet to trigger reconcile for the one which has the label on it + // matching the NetConfig (there can only be one) in the same namespace + ipSetList := &ospdirectorv1beta1.OpenStackIPSetList{} + + listOpts := []client.ListOption{ + client.InNamespace(o.GetNamespace()), + client.MatchingLabels{ + shared.OpenStackNetConfigReconcileLabel: o.GetName(), + }, + } + if err := r.Client.List(ctx, ipSetList, listOpts...); err != nil { + Log.Error(err, "Unable to retrieve OpenStackIPSetList") + return nil + } + + // For each OpenStackIPSet instance create a reconcile request + for _, i := range ipSetList.Items { + name := client.ObjectKey{ + Namespace: i.Namespace, + Name: i.Name, + } + result = append(result, reconcile.Request{NamespacedName: name}) + } + if len(result) > 0 { + Log.Info("Reconcile request for:", "result", result) + + return result + } + return nil + }) + return ctrl.NewControllerManagedBy(mgr). For(&ospdirectorv1beta1.OpenStackIPSet{}). + Watches(&source.Kind{Type: &ospdirectorv1beta1.OpenStackNetConfig{}}, + ipsetFN). Complete(r) } diff --git a/main.go b/main.go index 984aad0f..9fbe2d0c 100644 --- a/main.go +++ b/main.go @@ -17,6 +17,7 @@ limitations under the License. package main import ( + "context" "flag" "fmt" "os" @@ -316,7 +317,7 @@ func main() { Kclient: kclient, Log: ctrl.Log.WithName("controllers").WithName("OpenStackIPSet"), Scheme: mgr.GetScheme(), - }).SetupWithManager(mgr); err != nil { + }).SetupWithManager(context.Background(), mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "OpenStackIPSet") os.Exit(1) } From f3c399d82f6f1117747864f9665aa873eac02259 Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Wed, 20 Nov 2024 11:41:45 +0100 Subject: [PATCH 2/2] Restart openstackclient pod if network attachment list changes currently if a new network gets added to the openstackclient network list, the ipset and pod annotation gets updated, but the interface is not configured. Therefore this change makes sure that the openstackclient pod gets restarted when the network list changes related https://bugzilla.redhat.com/show_bug.cgi?id=2325902 Signed-off-by: Martin Schuppert (cherry picked from commit 6b7e8be795179c7fd469849b0600611afd7a4ad8) --- controllers/openstackclient_controller.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/controllers/openstackclient_controller.go b/controllers/openstackclient_controller.go index b1841608..ed2512dc 100644 --- a/controllers/openstackclient_controller.go +++ b/controllers/openstackclient_controller.go @@ -549,7 +549,16 @@ func (r *OpenStackClientReconciler) podCreateOrUpdate( isPodUpdate := !pod.ObjectMeta.CreationTimestamp.IsZero() + val, ok := pod.ObjectMeta.Annotations["k8s.v1.cni.cncf.io/networks"] + if ok && val != annotation { + return k8s_errors.NewForbidden( + schema.GroupResource{Group: "", Resource: "pods"}, // Specify the group and resource type + pod.Name, + errors.New("Restart Pod required to get new network attachment configured"), + ) + } pod.ObjectMeta.Annotations["k8s.v1.cni.cncf.io/networks"] = annotation + for k, v := range common.GetLabels(instance, openstackclient.AppLabel, map[string]string{}) { pod.Labels[k] = v }