tripleo muilti-cell: conditionally use ext net

Move the role generation and external networks removal snippet from ceph.sh to the main tripleo.sh script, and make it controled by the TRIPLEO_NETWORKING flag. Also add the HCI role to the main roles file, so that other roles defined there may still be used in HCI Ceph topology. Add attach_external_net j2 var based on the TRIPLEO_NETWORKING value. Multi-node adoption CI jobs deploy tripleo isolnet w/o external network historically, so it would use TRIPLEO_NETWORKING=false. For local setups, one might want to use it. Add granular tripleo netconfig control flags In local multi-cell setups, we use tripleo netconfig for cell stacks. Extend it to the main stack as well. In today multi-node CI we keep it disabled, so this addition doesn't change the status quo. Add more granular EDPM flags to control attaching external network, configuring (tripleo) network, and managing (undercloud) default route. This are currently only used with tripleo dev setups, but may be extended to edpm dev envs if/as needed. Align the config download templates to use them consistently Signed-off-by: Bohdan Dobrelia <[email protected]>
openstack-k8s-operators · Nov 5, 2024 · f7a954b · f7a954b
1 parent 449712d
commit f7a954b
Show file tree

Hide file tree

Showing 8 changed files with 114 additions and 27 deletions.
diff --git a/devsetup/Makefile b/devsetup/Makefile
@@ -73,6 +73,8 @@ EDPM_COMPUTE_CEPH_ENABLED ?= true
 EDPM_COMPUTE_CELLS ?= 1
 EDPM_COMPUTE_CEPH_NOVA ?= true
 EDPM_CONFIGURE_NETWORKING ?= true
+EDPM_ATTACH_EXTNET ?= true
+EDPM_CONFIGURE_DEFAULT_ROUTE ?= true
 EDPM_COMPUTE_SRIOV_ENABLED ?= true
 EDPM_COMPUTE_DHCP_AGENT_ENABLED ?= true
 EDPM_TOTAL_NETWORKERS ?= 1
@@ -482,6 +484,8 @@ tripleo_deploy: export REGISTRY_PWD ?= ${RH_REGISTRY_PWD}
 tripleo_deploy: export COMPUTE_CEPH_ENABLED=${EDPM_COMPUTE_CEPH_ENABLED}
 tripleo_deploy: export COMPUTE_CEPH_NOVA=${EDPM_COMPUTE_CEPH_NOVA}
 tripleo_deploy: export TRIPLEO_NETWORKING=${EDPM_CONFIGURE_NETWORKING}
+tripleo_deploy: export TRIPLEO_ATTACH_EXTNET=${EDPM_ATTACH_EXTNET}
+tripleo_deploy: export TRIPLEO_UNDERCLOUD_CONFIGURE_DEFAULT_ROUTE=${EDPM_CONFIGURE_DEFAULT_ROUTE}
 tripleo_deploy:
 	$(eval $(call vars))
 	scripts/tripleo.sh

diff --git a/devsetup/scripts/tripleo.sh b/devsetup/scripts/tripleo.sh
@@ -32,6 +32,8 @@ REPO_SETUP_CMDS=${REPO_SETUP_CMDS:-"${MY_TMP_DIR}/standalone_repos"}
 CMDS_FILE=${CMDS_FILE:-"${MY_TMP_DIR}/standalone_cmds"}
 SKIP_TRIPLEO_REPOS=${SKIP_TRIPLEO_REPOS:="false"}
 TRIPLEO_NETWORKING=${TRIPLEO_NETWORKING:-true}
+TRIPLEO_ATTACH_EXTNET=${TRIPLEO_ATTACH_EXTNET:=true}
+TRIPLEO_UNDERCLOUD_CONFIGURE_DEFAULT_ROUTE=${TRIPLEO_UNDERCLOUD_CONFIGURE_DEFAULT_ROUTE:=true}
 MANILA_ENABLED=${MANILA_ENABLED:-true}
 OCTAVIA_ENABLED=${OCTAVIA_ENABLED:-false}
 TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
@@ -97,6 +99,9 @@ export OCTAVIA_ENABLED=${OCTAVIA_ENABLED}
 export TELEMETRY_ENABLED=${TELEMETRY_ENABLED:-true}
 export TLSE_ENABLED=${TLSE_ENABLED:-false}
 export CLOUD_DOMAIN=${CLOUD_DOMAIN:-localdomain}
+export TRIPLEO_NETWORKING=${TRIPLEO_NETWORKING:-true}
+export TRIPLEO_ATTACH_EXTNET=${TRIPLEO_ATTACH_EXTNET:=true}
+export TRIPLEO_UNDERCLOUD_CONFIGURE_DEFAULT_ROUTE=${TRIPLEO_UNDERCLOUD_CONFIGURE_DEFAULT_ROUTE:-true}
 
 set +x
 if [ ! -f \$HOME/containers-prepare-parameters.yaml ]; then
@@ -164,7 +169,9 @@ ip_address_suffix: ${IP_ADRESS_SUFFIX}
 interface_mtu: ${INTERFACE_MTU:-1500}
 ntp_server: ${NTP_SERVER}
 gateway_ip: ${GATEWAY}
-manage_default_route: ${TRIPLEO_NETWORKING}
+manage_undercloud_default_route: ${TRIPLEO_UNDERCLOUD_CONFIGURE_DEFAULT_ROUTE}
+attach_external_net: ${TRIPLEO_ATTACH_EXTNET}
+tripleo_networking: ${TRIPLEO_NETWORKING}
 dns_server: ${PRIMARY_RESOLV_CONF_ENTRY}
 user_home: /home/zuul
 cloud_domain: ${CLOUD_DOMAIN}

diff --git a/devsetup/tripleo/ceph.sh b/devsetup/tripleo/ceph.sh
@@ -39,14 +39,6 @@ data_devices:
     - /dev/ceph_vg1/ceph_lv1
 EOF
 
-# create roles file
-openstack overcloud roles generate Controller ComputeHCI > roles.yaml
-
-# disable external gateway for controller nodes
-sed -i "s/default_route_networks: \['External'\]/default_route_networks: \['ControlPlane'\]/" roles.yaml
-sed -i "/External:/d" roles.yaml
-sed -i "/subnet: external_subnet/d" roles.yaml
-
 # NOTE: TripleO has the hardcoded --yes-i-know option that is not valid anymore
 # in RHCS 7. TripleO does not receive any new patch both upstream and downstream
 # (it is a retired project), hence the only option we have is to patch the

diff --git a/devsetup/tripleo/config-download-multistack.j2 b/devsetup/tripleo/config-download-multistack.j2
@@ -7,22 +7,25 @@ resource_registry:
   OS::TripleO::Compute::Ports::StorageMgmtPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage_mgmt.yaml
   OS::TripleO::Compute::Ports::StoragePort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage.yaml
   OS::TripleO::Compute::Ports::TenantPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_tenant.yaml
-  OS::TripleO::Compute::Ports::ExternalPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_external.yaml
   OS::TripleO::Controller::Ports::InternalApiPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_internal_api.yaml
   OS::TripleO::Controller::Ports::StorageMgmtPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage_mgmt.yaml
   OS::TripleO::Controller::Ports::StoragePort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage.yaml
   OS::TripleO::Controller::Ports::TenantPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_tenant.yaml
-  OS::TripleO::Controller::Ports::ExternalPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_external.yaml
   OS::TripleO::CellController::Ports::InternalApiPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_internal_api.yaml
   OS::TripleO::CellController::Ports::StorageMgmtPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage_mgmt.yaml
   OS::TripleO::CellController::Ports::StoragePort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage.yaml
   OS::TripleO::CellController::Ports::TenantPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_tenant.yaml
-  OS::TripleO::CellController::Ports::ExternalPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_external.yaml
   OS::TripleO::CellControllerCompute::Ports::InternalApiPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_internal_api.yaml
   OS::TripleO::CellControllerCompute::Ports::StorageMgmtPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage_mgmt.yaml
   OS::TripleO::CellControllerCompute::Ports::StoragePort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage.yaml
   OS::TripleO::CellControllerCompute::Ports::TenantPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_tenant.yaml
+  {% if attach_external_net|bool %}
+  OS::TripleO::Compute::Ports::ExternalPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_external.yaml
+  OS::TripleO::Controller::Ports::ExternalPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_external.yaml
+  OS::TripleO::CellController::Ports::ExternalPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_external.yaml
   OS::TripleO::CellControllerCompute::Ports::ExternalPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_external.yaml
+  {% endif %}
+
 parameter_defaults:
   DeployedServerPortMap:
     controller-0-ctlplane:
@@ -83,10 +86,13 @@ parameter_defaults:
         ip_address: 172.20.0.103
         ip_address_uri: 172.20.0.103
         ip_subnet: 172.20.0.0/24
+      {% if attach_external_net|bool %}
       external:
         ip_address: 172.21.0.103
         ip_address_uri: 172.21.0.103
         ip_subnet: 172.21.0.0/24
+      {% endif %}
+
     compute-0: # cell1-controller-0 (cell1, ind0)
       ctlplane:
         ip_address: 192.168.122.106
@@ -108,10 +114,13 @@ parameter_defaults:
         ip_address: 172.20.0.106
         ip_address_uri: 172.20.0.106
         ip_subnet: 172.20.0.0/24
+      {% if attach_external_net|bool %}
       external:
         ip_address: 172.21.0.106
         ip_address_uri: 172.21.0.106
         ip_subnet: 172.21.0.0/24
+      {% endif %}
+
     compute-1: # cell1-compute-0 (cell1, ind1)
       ctlplane:
         ip_address: 192.168.122.107
@@ -133,10 +142,13 @@ parameter_defaults:
         ip_address: 172.20.0.107
         ip_address_uri: 172.20.0.107
         ip_subnet: 172.20.0.0/24
+      {% if attach_external_net|bool %}
       external:
         ip_address: 172.21.0.107
         ip_address_uri: 172.21.0.107
         ip_subnet: 172.21.0.0/24
+      {% endif %}
+
     compute-2: # cell2-controller-compute-0 AIO (cell2, ind0)
       ctlplane:
         ip_address: 192.168.122.109
@@ -158,10 +170,13 @@ parameter_defaults:
         ip_address: 172.20.0.109
         ip_address_uri: 172.20.0.109
         ip_subnet: 172.20.0.0/24
+      {% if attach_external_net|bool %}
       external:
         ip_address: 172.21.0.109
         ip_address_uri: 172.21.0.109
         ip_subnet: 172.21.0.0/24
+      {% endif %}
+
   CtlplaneNetworkAttributes:
     network:
       dns_domain: localdomain
@@ -174,9 +189,13 @@ parameter_defaults:
         cidr: 192.168.122.0/24
         dns_nameservers: [{{ dns_server }}, 192.168.122.1]
         gateway_ip: {{ gateway_ip }}
+        {% if manage_undercloud_default_route|bool %}
         host_routes:
           - ip_netmask: 0.0.0.0/0
             next_hop: 192.168.122.1
             default: true
+        {% else %}
+        host_routes: []
+        {% endif %}
         ip_version: 4
         name: ctlplane-subnet
diff --git a/devsetup/tripleo/config-download.yaml.j2 b/devsetup/tripleo/config-download.yaml.j2
@@ -4,13 +4,18 @@ resource_registry:
   OS::TripleO::OVNMacAddressNetwork: OS::Heat::None
   OS::TripleO::OVNMacAddressPort: OS::Heat::None
   OS::TripleO::Compute::Ports::InternalApiPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_internal_api.yaml
+  OS::TripleO::Compute::Ports::StorageMgmtPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage_mgmt.yaml
   OS::TripleO::Compute::Ports::StoragePort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage.yaml
   OS::TripleO::Compute::Ports::TenantPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_tenant.yaml
-  OS::TripleO::Controller::Ports::ExternalPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_external.yaml
   OS::TripleO::Controller::Ports::InternalApiPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_internal_api.yaml
   OS::TripleO::Controller::Ports::StorageMgmtPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage_mgmt.yaml
   OS::TripleO::Controller::Ports::StoragePort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage.yaml
   OS::TripleO::Controller::Ports::TenantPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_tenant.yaml
+  {% if attach_external_net|bool %}
+  OS::TripleO::Compute::Ports::ExternalPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_external.yaml
+  OS::TripleO::Controller::Ports::ExternalPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_external.yaml
+  {% endif %}
+
 parameter_defaults:
   DeployedServerPortMap:
     controller-0-ctlplane:
@@ -84,6 +89,13 @@ parameter_defaults:
         ip_address: 172.19.0.103
         ip_address_uri: 172.19.0.103
         ip_subnet: 172.19.0.0/24
+      {% if attach_external_net|bool %}
+      external:
+        ip_address: 172.21.0.103
+        ip_address_uri: 172.21.0.103
+        ip_subnet: 172.21.0.0/24
+      {% endif %}
+
     controller-1:
       ctlplane:
         ip_address: 192.168.122.104
@@ -105,6 +117,13 @@ parameter_defaults:
         ip_address: 172.19.0.104
         ip_address_uri: 172.19.0.104
         ip_subnet: 172.19.0.0/24
+      {% if attach_external_net|bool %}
+      external:
+        ip_address: 172.21.0.104
+        ip_address_uri: 172.21.0.104
+        ip_subnet: 172.21.0.0/24
+      {% endif %}
+
     controller-2:
       ctlplane:
         ip_address: 192.168.122.105
@@ -126,6 +145,13 @@ parameter_defaults:
         ip_address: 172.19.0.105
         ip_address_uri: 172.19.0.105
         ip_subnet: 172.19.0.0/24
+      {% if attach_external_net|bool %}
+      external:
+        ip_address: 172.21.0.105
+        ip_address_uri: 172.21.0.105
+        ip_subnet: 172.21.0.0/24
+      {% endif %}
+
     compute-0:
       ctlplane:
         ip_address: 192.168.122.106
@@ -147,6 +173,13 @@ parameter_defaults:
         ip_address: 172.19.0.106
         ip_address_uri: 172.19.0.106
         ip_subnet: 172.19.0.0/24
+      {% if attach_external_net|bool %}
+      external:
+        ip_address: 172.21.0.106
+        ip_address_uri: 172.21.0.106
+        ip_subnet: 172.21.0.0/24
+      {% endif %}
+
     compute-1:
       ctlplane:
         ip_address: 192.168.122.107
@@ -168,6 +201,13 @@ parameter_defaults:
         ip_address: 172.19.0.107
         ip_address_uri: 172.19.0.107
         ip_subnet: 172.19.0.0/24
+      {% if attach_external_net|bool %}
+      external:
+        ip_address: 172.21.0.107
+        ip_address_uri: 172.21.0.107
+        ip_subnet: 172.21.0.0/24
+      {% endif %}
+
     compute-2:
       ctlplane:
         ip_address: 192.168.122.108
@@ -189,6 +229,13 @@ parameter_defaults:
         ip_address: 172.19.0.108
         ip_address_uri: 172.19.0.108
         ip_subnet: 172.19.0.0/24
+      {% if attach_external_net|bool %}
+      external:
+        ip_address: 172.21.0.108
+        ip_address_uri: 172.21.0.108
+        ip_subnet: 172.21.0.0/24
+      {% endif %}
+
   CtlplaneNetworkAttributes:
     network:
       dns_domain: {{ cloud_domain }}
@@ -199,8 +246,15 @@ parameter_defaults:
     subnets:
       ctlplane-subnet:
         cidr: 192.168.122.0/24
-        dns_nameservers: 192.168.122.10
-        gateway_ip: 192.168.122.10
+        dns_nameservers: [{{ dns_server }}, 192.168.122.1]
+        gateway_ip: {{ gateway_ip }}
+        {% if manage_undercloud_default_route|bool %}
+        host_routes:
+          - ip_netmask: 0.0.0.0/0
+            next_hop: 192.168.122.1
+            default: true
+        {% else %}
         host_routes: []
+        {% endif %}
         ip_version: 4
         name: ctlplane-subnet
diff --git a/devsetup/tripleo/net_config.j2 b/devsetup/tripleo/net_config.j2
@@ -14,7 +14,7 @@ network_config:
   - ip_netmask: {{ ctlplane_ip }}/{{ ctlplane_cidr }}
   - ip_netmask: {{ ctlplane_ip }}/32
   - ip_netmask: {{ ctlplane_vip }}/32
-  {% if manage_default_route|bool %}
+  {% if manage_undercloud_default_route|bool %}
   routes:
     - ip_netmask: 0.0.0.0/0
       next_hop: 192.168.122.1

diff --git a/devsetup/tripleo/overcloud_services.yaml.j2 b/devsetup/tripleo/overcloud_services.yaml.j2
@@ -51,15 +51,15 @@ parameter_defaults:
   CloudNameStorageManagement: overcloud.storagemgmt.{{ cloud_domain }}
   CloudNameCtlplane: overcloud.ctlplane.{{ cloud_domain }}
   CloudDomain: {{ cloud_domain }}
-  NetworkConfigWithAnsible: false
-  ControllerNetworkConfigUpdate: false
-  ComputeNetworkConfigUpdate: false
-  BlockStorageNetworkConfigUpdate: false
-  ObjectStorageNetworkConfigUpdate: false
-  CephStorageNetworkConfigUpdate: false
-  NetworkerNetworkConfigUpdate: false
-  CellControllerNetworkConfigUpdate: false
-  CellControllerComputeNetworkConfigUpdate: false
+  NetworkConfigWithAnsible: {{ tripleo_networking }}
+  ControllerNetworkConfigUpdate: {{ tripleo_networking }}
+  ComputeNetworkConfigUpdate: {{ tripleo_networking }}
+  BlockStorageNetworkConfigUpdate: {{ tripleo_networking }}
+  ObjectStorageNetworkConfigUpdate: {{ tripleo_networking }}
+  CephStorageNetworkConfigUpdate: {{ tripleo_networking }}
+  NetworkerNetworkConfigUpdate: {{ tripleo_networking }}
+  CellControllerNetworkConfigUpdate: {{ tripleo_networking }}
+  CellControllerComputeNetworkConfigUpdate: {{ tripleo_networking }}
 
   ControllerNetworkConfigTemplate: templates/single_nic_vlans/single_nic_vlans.j2
   ComputeNetworkConfigTemplate: templates/single_nic_vlans/single_nic_vlans.j2

diff --git a/devsetup/tripleo/tripleo_install.sh b/devsetup/tripleo/tripleo_install.sh
@@ -128,7 +128,7 @@ for config_download in ${cdfiles[@]}; do
         stg_mgmt_line="OS::TripleO::ComputeHCI::Ports::StorageMgmtPort: /usr/share/openstack-tripleo-heat-templates/network/ports/deployed_storage_mgmt.yaml"
         sed -i "s#$stg_line#$stg_line\r  $stg_mgmt_line\r#" $config_download
         # correct RoleCount var in overcloud_services
-        sed -i "s/ComputeCount/ComputeHCICount/" overcloud_services.yaml
+        sed -i -r "s/\bComputeCount(.*)$/ComputeHCICount\1\n  ComputeCount: 0/g" overcloud_services.yaml
     fi
     # Remove any quotes e.g. "np10002"-ctlplane -> np10002-ctlplane
     sed -i 's/\"//g' "$config_download"
@@ -157,10 +157,21 @@ fi
 # defaults for non-ceph case
 CEPH_OVERCLOUD_ARGS=""
 ROLES_FILE="/home/zuul/overcloud_roles.yaml"
+if [ "$EDPM_COMPUTE_CEPH_ENABLED" = "true" ] ; then
+    # create roles file
+    cp $ROLES_FILE roles.yaml
+    openstack overcloud roles generate ComputeHCI >> roles.yaml
+    ROLES_FILE=roles.yaml
+fi
+if [ "$TRIPLEO_ATTACH_EXTNET" != "true" ] ; then
+    # disable external gateway for controller nodes
+    sed -i "s/default_route_networks: \['External'\]/default_route_networks: \['ControlPlane'\]/" $ROLES_FILE
+    sed -i "/External:/d" $ROLES_FILE
+    sed -i "/subnet: external_subnet/d" $ROLES_FILE
+fi
 if [ "$EDPM_COMPUTE_CEPH_ENABLED" = "true" ] ; then
     CEPH_OVERCLOUD_ARGS="${CEPH_ARGS}"
     [[ "$MANILA_ENABLED" == "true" ]] && CEPH_OVERCLOUD_ARGS+=' -e /usr/share/openstack-tripleo-heat-templates/environments/cephadm/ceph-mds.yaml'
-    ROLES_FILE="/home/zuul/roles.yaml"
     /tmp/ceph.sh
 fi