From bfcf4ddc8493373bd6fe4ca865a112063a474322 Mon Sep 17 00:00:00 2001 From: Shay Rozen Date: Sun, 5 Jan 2025 20:55:43 +0200 Subject: [PATCH] Multus ipv6 on vsphere Signed-off-by: Shay Rozen --- ...multus_public_cluster_vsan_3m_3w_ipv6.yaml | 25 ++++++++++++++ ...e_upi_enable_multus_ipv6_single_stack.yaml | 13 +++++++ ocs_ci/deployment/deployment.py | 34 +++++++++++++------ ocs_ci/framework/conf/default_config.yaml | 2 ++ ocs_ci/helpers/helpers.py | 15 ++++++-- ocs_ci/ocs/constants.py | 12 +++++++ ocs_ci/ocs/resources/storage_cluster.py | 13 +++++-- .../multus-cluster-net-ipv6.yaml | 18 ++++++++++ .../multus-public-net-ipv6.yaml | 19 +++++++++++ ...ode_network_configuration_policy_ipv6.yaml | 31 +++++++++++++++++ 10 files changed, 168 insertions(+), 14 deletions(-) create mode 100644 conf/deployment/vsphere/upi_1az_rhcos_multus_public_cluster_vsan_3m_3w_ipv6.yaml create mode 100644 conf/ocsci/vsphere_upi_enable_multus_ipv6_single_stack.yaml create mode 100644 ocs_ci/templates/ocs-deployment/multus-cluster-net-ipv6.yaml create mode 100644 ocs_ci/templates/ocs-deployment/multus-public-net-ipv6.yaml create mode 100644 ocs_ci/templates/ocs-deployment/node_network_configuration_policy_ipv6.yaml diff --git a/conf/deployment/vsphere/upi_1az_rhcos_multus_public_cluster_vsan_3m_3w_ipv6.yaml b/conf/deployment/vsphere/upi_1az_rhcos_multus_public_cluster_vsan_3m_3w_ipv6.yaml new file mode 100644 index 00000000000..26f365a028c --- /dev/null +++ b/conf/deployment/vsphere/upi_1az_rhcos_multus_public_cluster_vsan_3m_3w_ipv6.yaml @@ -0,0 +1,25 @@ +--- +# This config is suppose to work on most of DCs we have. +DEPLOYMENT: + allow_lower_instance_requirements: false + ipv6: true +ENV_DATA: + platform: "vsphere" + deployment_type: "upi" + worker_replicas: 3 + master_replicas: 3 + worker_num_cpus: "16" + master_num_cpus: "4" + master_memory: "16384" + compute_memory: "65536" + fio_storageutilization_min_mbps: 10.0 + is_multus_enabled: true + multus_public_net_interface: "ens224" + multus_cluster_net_interface: "ens224" + multus_create_public_net: true + multus_create_cluster_net: true + multus_public_net_namespace: "default" + +REPORTING: + polarion: + deployment_id: "OCS-6317" diff --git a/conf/ocsci/vsphere_upi_enable_multus_ipv6_single_stack.yaml b/conf/ocsci/vsphere_upi_enable_multus_ipv6_single_stack.yaml new file mode 100644 index 00000000000..10208c39735 --- /dev/null +++ b/conf/ocsci/vsphere_upi_enable_multus_ipv6_single_stack.yaml @@ -0,0 +1,13 @@ +# This configuration file is to enable vsphere upi ipv6 single stack +DEPLOYMENT: + ipv6: true +ENV_DATA: + is_multus_enabled: true + multus_public_net_interface: "ens224" + multus_cluster_net_interface: "ens224" + multus_create_public_net: true + multus_create_cluster_net: true + multus_public_net_namespace: "default" +REPORTING: + polarion: + deployment_id: "OCS-6137" diff --git a/ocs_ci/deployment/deployment.py b/ocs_ci/deployment/deployment.py index d9dd0ec7759..3ad4ba0f6d6 100644 --- a/ocs_ci/deployment/deployment.py +++ b/ocs_ci/deployment/deployment.py @@ -1144,8 +1144,11 @@ def deploy_ocs_via_operator(self, image=None): node_obj.exec_oc_debug_cmd(node=node, cmd_list=[ip_link_cmd]) if create_public_net: + nad_to_load = constants.MULTUS_PUBLIC_NET_YAML logger.info("Creating Multus public network") - public_net_data = templating.load_yaml(constants.MULTUS_PUBLIC_NET_YAML) + if config.DEPLOYMENT.get("ipv6"): + nad_to_load = constants.MULTUS_PUBLIC_NET_IPV6_YAML + public_net_data = templating.load_yaml(nad_to_load) public_net_data["metadata"]["name"] = config.ENV_DATA.get( "multus_public_net_name" ) @@ -1157,9 +1160,14 @@ def deploy_ocs_via_operator(self, image=None): public_net_config_dict["master"] = config.ENV_DATA.get( "multus_public_net_interface" ) - public_net_config_dict["ipam"]["range"] = config.ENV_DATA.get( - "multus_public_net_range" - ) + if not config.DEPLOYMENT.get("ipv6"): + public_net_config_dict["ipam"]["range"] = config.ENV_DATA.get( + "multus_public_net_range" + ) + else: + public_net_config_dict["ipam"]["range"] = config.ENV_DATA.get( + "multus_public_ipv6_net_range" + ) public_net_config_dict["type"] = config.ENV_DATA.get( "multus_public_net_type" ) @@ -1175,6 +1183,10 @@ def deploy_ocs_via_operator(self, image=None): if create_cluster_net: logger.info("Creating Multus cluster network") + if config.DEPLOYMENT.get("ipv6"): + constants.MULTUS_CLUSTER_NET_YAML = ( + constants.MULTUS_CLUSTER_NET_IPV6_YAML + ) cluster_net_data = templating.load_yaml( constants.MULTUS_CLUSTER_NET_YAML ) @@ -1189,12 +1201,14 @@ def deploy_ocs_via_operator(self, image=None): cluster_net_config_dict["master"] = config.ENV_DATA.get( "multus_cluster_net_interface" ) - cluster_net_config_dict["ipam"]["range"] = config.ENV_DATA.get( - "multus_cluster_net_range" - ) - cluster_net_config_dict["type"] = config.ENV_DATA.get( - "multus_cluster_net_type" - ) + if not config.DEPLOYMENT.get("ipv6"): + cluster_net_config_dict["ipam"]["range"] = config.ENV_DATA.get( + "multus_cluster_net_range" + ) + else: + cluster_net_config_dict["ipam"]["range"] = config.ENV_DATA.get( + "multus_cluster_ipv6_net_range" + ) cluster_net_config_dict["mode"] = config.ENV_DATA.get( "multus_cluster_net_mode" ) diff --git a/ocs_ci/framework/conf/default_config.yaml b/ocs_ci/framework/conf/default_config.yaml index 4db39aa2863..82a9ae26142 100644 --- a/ocs_ci/framework/conf/default_config.yaml +++ b/ocs_ci/framework/conf/default_config.yaml @@ -264,6 +264,7 @@ ENV_DATA: multus_public_net_namespace: "openshift-storage" multus_public_net_interface: "br-ex" multus_public_net_range: "192.168.20.0/24" + multus_public_ipv6_net_range: "fd01:db8:1:1::/96" multus_public_net_type: "macvlan" multus_public_net_mode: "bridge" multus_create_cluster_net: true @@ -271,6 +272,7 @@ ENV_DATA: multus_cluster_net_namespace: "openshift-storage" multus_cluster_net_interface: "br-ex" multus_cluster_net_range: "192.168.30.0/24" + multus_cluster_ipv6_net_range: "fd01:db8:2:1::/96" multus_cluster_net_type: "macvlan" multus_cluster_net_mode: "bridge" multus_destination_route: "192.168.252.0/24" diff --git a/ocs_ci/helpers/helpers.py b/ocs_ci/helpers/helpers.py index b0fad161afd..54e0941df78 100644 --- a/ocs_ci/helpers/helpers.py +++ b/ocs_ci/helpers/helpers.py @@ -4870,6 +4870,8 @@ def add_route_public_nad(): ] nad_config_dict_string = json.dumps(nad_config_dict) logger.info("Creating Multus public network") + if config.DEPLOYMENT.get("ipv6"): + constants.MULTUS_PUBLIC_NET_YAML = constants.MULTUS_PUBLIC_NET_IPV6_YAML public_net_data = templating.load_yaml(constants.MULTUS_PUBLIC_NET_YAML) public_net_data["metadata"]["name"] = config.ENV_DATA.get("multus_public_net_name") public_net_data["metadata"]["namespace"] = config.ENV_DATA.get( @@ -4959,6 +4961,15 @@ def configure_node_network_configuration_policy_on_all_worker_nodes(): # This function require changes for compact mode logger.info("Configure NodeNetworkConfigurationPolicy on all worker nodes") worker_node_names = get_worker_nodes() + ip_version = "ipv4" + if ( + config.DEPLOYMENT.get("ipv6") + and config.ENV_DATA.get("platform") == constants.VSPHERE_PLATFORM + ): + constants.NODE_NETWORK_CONFIGURATION_POLICY = ( + constants.NODE_NETWORK_CONFIGURATION_POLICY_IPV6 + ) + ip_version = "ipv6" interface_num = 0 for worker_node_name in worker_node_names: node_network_configuration_policy = templating.load_yaml( @@ -5001,13 +5012,13 @@ def configure_node_network_configuration_policy_on_all_worker_nodes(): ] = f"ceph-public-net-shim-{worker_node_name}" shim_default_ip = node_network_configuration_policy["spec"]["desiredState"][ "interfaces" - ][0]["ipv4"]["address"][0]["ip"] + ][0][ip_version]["address"][0]["ip"] shim_ip = str(ipaddress.ip_address(shim_default_ip) + interface_num) interface_num += 1 node_network_configuration_policy["spec"]["desiredState"]["interfaces"][0][ - "ipv4" + ip_version ]["address"][0]["ip"] = shim_ip node_network_configuration_policy["spec"]["desiredState"]["interfaces"][0][ diff --git a/ocs_ci/ocs/constants.py b/ocs_ci/ocs/constants.py index 97c62d4db6e..02ba43e07b1 100644 --- a/ocs_ci/ocs/constants.py +++ b/ocs_ci/ocs/constants.py @@ -1047,6 +1047,18 @@ NODE_NETWORK_CONFIGURATION_POLICY = os.path.join( TEMPLATE_DEPLOYMENT_DIR, "node_network_configuration_policy.yaml" ) + +MULTUS_PUBLIC_NET_IPV6_YAML = os.path.join( + TEMPLATE_DEPLOYMENT_DIR, "multus-public-net-ipv6.yaml" +) +MULTUS_CLUSTER_NET_IPV6_YAML = os.path.join( + TEMPLATE_DEPLOYMENT_DIR, "multus-cluster-net-ipv6.yaml" +) +NODE_NETWORK_CONFIGURATION_POLICY_IPV6 = os.path.join( + TEMPLATE_DEPLOYMENT_DIR, "node_network_configuration_policy_ipv6.yaml" +) + + NETWORK_ATTACHEMENT_DEFINITION = "network-attachment-definitions.k8s.cni.cncf.io" VSPHERE_MULTUS_INTERFACE = "ens224" diff --git a/ocs_ci/ocs/resources/storage_cluster.py b/ocs_ci/ocs/resources/storage_cluster.py index c18da0b946a..0fdfe755b2f 100644 --- a/ocs_ci/ocs/resources/storage_cluster.py +++ b/ocs_ci/ocs/resources/storage_cluster.py @@ -1983,8 +1983,17 @@ def verify_multus_network(): ) mds_map = ceph_fs_dump_data["filesystems"][0]["mdsmap"] for _, gid_data in mds_map["info"].items(): - ip = gid_data["addr"].split(":")[0] - range = config.ENV_DATA["multus_public_net_range"] + if not config.DEPLOYMENT.get("ipv6"): + ip = gid_data["addr"].split(":")[0] + range = config.ENV_DATA["multus_public_net_range"] + + else: + gid_dt_ip = gid_data["addr"] + pattern = r"^\[([\da-f:]+)\]" + match = re.search(pattern, gid_dt_ip, re.IGNORECASE) + ip = match.group(1) + range = config.ENV_DATA["multus_public_ipv6_net_range"] + assert ipaddress.ip_address(ip) in ipaddress.ip_network(range) log.info("Verifying StorageCluster multus network data") diff --git a/ocs_ci/templates/ocs-deployment/multus-cluster-net-ipv6.yaml b/ocs_ci/templates/ocs-deployment/multus-cluster-net-ipv6.yaml new file mode 100644 index 00000000000..b7a1e685e32 --- /dev/null +++ b/ocs_ci/templates/ocs-deployment/multus-cluster-net-ipv6.yaml @@ -0,0 +1,18 @@ +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: cluster-net + namespace: openshift-storage + labels: {} + annotations: {} +spec: + config: '{ + "cniVersion": "0.3.1", + "type": "macvlan", + "master": "br-ex", + "mode": "bridge", + "ipam": { + "type": "whereabouts", + "range": "fd01:db8:2:1::/96" + } + }' diff --git a/ocs_ci/templates/ocs-deployment/multus-public-net-ipv6.yaml b/ocs_ci/templates/ocs-deployment/multus-public-net-ipv6.yaml new file mode 100644 index 00000000000..f2b5de7deec --- /dev/null +++ b/ocs_ci/templates/ocs-deployment/multus-public-net-ipv6.yaml @@ -0,0 +1,19 @@ +apiVersion: "k8s.cni.cncf.io/v1" +kind: NetworkAttachmentDefinition +metadata: + name: public-net + namespace: openshift-storage + labels: {} + annotations: {} +spec: + config: '{ + "cniVersion": "0.3.1", + "type": "macvlan", + "master": "br-ex", + "mode": "bridge", + "ipam": { + "type": "whereabouts", + "range": "fd01:db8:1:1::/96", + "routes": [{"dst": "fd01:db8:3:1::/96"}] + } + }' diff --git a/ocs_ci/templates/ocs-deployment/node_network_configuration_policy_ipv6.yaml b/ocs_ci/templates/ocs-deployment/node_network_configuration_policy_ipv6.yaml new file mode 100644 index 00000000000..f41ba994732 --- /dev/null +++ b/ocs_ci/templates/ocs-deployment/node_network_configuration_policy_ipv6.yaml @@ -0,0 +1,31 @@ +apiVersion: nmstate.io/v1 +kind: NodeNetworkConfigurationPolicy +metadata: + name: ceph-public-net-shim-worker-node + namespace: openshift-storage +spec: + nodeSelector: + node-role.kubernetes.io/worker: "" + kubernetes.io/hostname: worker-node + desiredState: + interfaces: + - name: odf-pub-shim + description: Shim interface used to connect host to OpenShift Data Foundation public Multus network + type: mac-vlan + state: up + mac-vlan: + base-iface: enp1s0f1 + mode: bridge + promiscuous: true + ipv6: + enabled: true + dhcp: false + address: + - ip: fd01:db8:3:1::1 # STATIC IP FOR worker node + prefix-length: 96 + routes: + config: + - destination: fd01:db8:1:1::/96 + next-hop-interface: odf-pub-shim + - destination: fd01:db8:2:1::/96 + next-hop-interface: odf-pub-shim