From a7f1cb4855bb43395b342ecdb4edbe2d6888b0a9 Mon Sep 17 00:00:00 2001 From: Arnau Verdaguer Date: Mon, 29 Apr 2024 12:53:00 +0200 Subject: [PATCH] Adding support for adoption scenario 1 Dataplane adoption docs supports doing the adoption while wallaby environment uses different subnets than next generation. This commit enables Makefiles in order to prepare the environment with different subnets between environments. Related: OSPRH-5881 --- Makefile | 29 ++++ devsetup/Makefile | 31 ++++ devsetup/scripts/standalone.sh | 9 + devsetup/scripts/tripleo.sh | 11 ++ devsetup/standalone/net_config.j2 | 38 ++++- devsetup/tripleo/net_config.j2 | 48 +++++- .../overcloud_net_config_scenario_a.j2 | 86 ++++++++++ scripts/gen-metallb-config.sh | 6 +- scripts/gen-netatt.sh | 65 ++++---- scripts/gen-nncp.sh | 156 +++++++++++------- 10 files changed, 367 insertions(+), 112 deletions(-) create mode 100644 devsetup/tripleo/overcloud_net_config_scenario_a.j2 diff --git a/Makefile b/Makefile index cd7d15cb0..f371d62c2 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,16 @@ CLEANUP_DIR_CMD ?= rm -Rf # be also cloned from OPENSTACK_REPO/OPENSTACK_BRANCH. CHECKOUT_FROM_OPENSTACK_REF ?= false +NETWORK_INTERNALAPI_ADDRESS_PREFIX ?= 172.17.0 +NETWORK_STORAGE_ADDRESS_PREFIX ?= 172.18.0 +NETWORK_TENANT_ADDRESS_PREFIX ?= 172.19.0 +NETWORK_STORAGEMGMT_ADDRESS_PREFIX ?= 172.20.0 +NETWORK_DESIGNATE_ADDRESS_PREFIX ?= 172.28.0 +INTERNALAPI_HOST_ROUTES ?= +STORAGE_HOST_ROUTES ?= +TENANT_HOST_ROUTES ?= +STORAGEMGMT_HOST_ROUTES ?= + # network isolation NETWORK_ISOLATION ?= true NETWORK_ISOLATION_USE_DEFAULT_NETWORK ?= true @@ -2230,6 +2240,15 @@ nncp_with_retries: ## Deploy NNCP with retries .PHONY: nncp nncp: export INTERFACE=${NNCP_INTERFACE} nncp: export BRIDGE_NAME=${NNCP_BRIDGE} +nncp: export INTERNALAPI_PREFIX=${NETWORK_INTERNALAPI_ADDRESS_PREFIX} +nncp: export NNCP_INTERNALAPI_HOST_ROUTES=${INTERNALAPI_HOST_ROUTES} +nncp: export STORAGE_PREFIX=${NETWORK_STORAGE_ADDRESS_PREFIX} +nncp: export NNCP_STORAGE_HOST_ROUTES=${STORAGE_HOST_ROUTES} +nncp: export STORAGEMGMT_PREFIX=${NETWORK_STORAGEMGMT_ADDRESS_PREFIX} +nncp: export NNCP_STORAGEMGMT_HOST_ROUTES=${STORAGEMGMT_HOST_ROUTES} +nncp: export TENANT_PREFIX=${NETWORK_TENANT_ADDRESS_PREFIX} +nncp: export NNCP_TENANT_HOST_ROUTES=${TENANT_HOST_ROUTES} +nncp: export DESIGNATE_PREFIX=${NETWORK_DESIGNATE_ADDRESS_PREFIX} ifeq ($(NETWORK_BGP), true) nncp: export BGP=enabled nncp: export INTERFACE_BGP_1=${NNCP_BGP_1_INTERFACE} @@ -2295,6 +2314,11 @@ netattach: export IPV6_ENABLED=true netattach: export CTLPLANE_IPV6_ADDRESS_PREFIX=${NNCP_CTLPLANE_IPV6_ADDRESS_PREFIX} netattach: export CTLPLANE_IPV6_ADDRESS_SUFFIX=${NNCP_CTLPLANE_IPV6_ADDRESS_SUFFIX} endif +netattach: export INTERNALAPI_PREFIX=${NETWORK_INTERNALAPI_ADDRESS_PREFIX} +netattach: export STORAGE_PREFIX=${NETWORK_STORAGE_ADDRESS_PREFIX} +netattach: export STORAGEMGMT_PREFIX=${NETWORK_STORAGEMGMT_ADDRESS_PREFIX} +netattach: export TENANT_PREFIX=${NETWORK_TENANT_ADDRESS_PREFIX} +netattach: export DESIGNATE_PREFIX=${NETWORK_DESIGNATE_ADDRESS_PREFIX} netattach: export VLAN_START=${NETWORK_VLAN_START} netattach: export VLAN_STEP=${NETWORK_VLAN_STEP} netattach: export CTLPLANE_IP_ADDRESS_PREFIX=${NNCP_CTLPLANE_IP_ADDRESS_PREFIX} @@ -2344,6 +2368,11 @@ endif metallb_config: export NAMESPACE=metallb-system metallb_config: export CTLPLANE_METALLB_POOL=${METALLB_POOL} metallb_config: export CTLPLANE_METALLB_IPV6_POOL=${METALLB_IPV6_POOL} +metallb_config: export INTERNALAPI_PREFIX=${NETWORK_INTERNALAPI_ADDRESS_PREFIX} +metallb_config: export STORAGE_PREFIX=${NETWORK_STORAGE_ADDRESS_PREFIX} +metallb_config: export STORAGEMGMT_PREFIX=${NETWORK_STORAGEMGMT_ADDRESS_PREFIX} +metallb_config: export TENANT_PREFIX=${NETWORK_TENANT_ADDRESS_PREFIX} +metallb_config: export DESIGNATE_PREFIX=${NETWORK_DESIGNATE_ADDRESS_PREFIX} ifeq ($(NETWORK_ISOLATION_IPV4), true) metallb_config: export IPV4_ENABLED=true endif diff --git a/devsetup/Makefile b/devsetup/Makefile index 0a784c3e3..e1112ab29 100644 --- a/devsetup/Makefile +++ b/devsetup/Makefile @@ -11,6 +11,17 @@ CRC_HTTPS_PROXY ?= OCP_NETWORK_NAME ?=crc NETWORK_ISOLATION_USE_DEFAULT_NETWORK ?= true +ADOPTED_INTERNALAPI_NETWORK ?= 172.17.1.0/24 +ADOPTED_STORAGE_NETWORK ?= 172.18.1.0/24 +ADOPTED_EXTERNAL_NETWORK ?= 172.21.1.0/24 +ADOPTED_STORAGEMGMT_NETWORK ?= 172.20.1.0/24 +ADOPTED_TENANT_NETWORK ?= 172.9.1.0/24 +STANDALONE_INTERNALAPI_NET_PREFIX ?= 172.17.0 +STANDALONE_STORAGE_NET_PREFIX ?= 172.18.0 +STANDALONE_EXTERNAL_NET_PREFFIX ?= 172.21.0 +STANDALONE_STORAGEMGMT_NET_PREFIX ?= 172.20.0 +STANDALONE_TENANT_NET_PREFIX ?= 172.19.0 + NETWORK_ISOLATION_MAC ?= 52:54:00:11:11:10 ifeq ($(NETWORK_ISOLATION_USE_DEFAULT_NETWORK), true) NETWORK_ISOLATION_NET_NAME ?= default @@ -463,6 +474,16 @@ tripleo_deploy: export REGISTRY_PWD ?= ${RH_REGISTRY_PWD} tripleo_deploy: export COMPUTE_CEPH_ENABLED=${EDPM_COMPUTE_CEPH_ENABLED} tripleo_deploy: export COMPUTE_CEPH_NOVA=${EDPM_COMPUTE_CEPH_NOVA} tripleo_deploy: export TRIPLEO_NETWORKING=${EDPM_CONFIGURE_NETWORKING} +tripleo_deploy: export NETWORK_INTERNAL_PREFIX = ${STANDALONE_INTERNALAPI_NET_PREFIX} +tripleo_deploy: export NETWORK_STORAGE_PREFIX = ${STANDALONE_STORAGE_NET_PREFIX} +tripleo_deploy: export NETWORK_STORAGE_MGMT_PREFIX = ${STANDALONE_STORAGEMGMT_NET_PREFIX} +tripleo_deploy: export NETWORK_TENANT_PREFIX = ${STANDALONE_TENANT_NET_PREFIX} +tripleo_deploy: export NETWORK_EXTERNAL_PREFFIX = ${STANDALONE_EXTERNAL_NET_PREFFIX} +tripleo_deploy: export ADOPTED_INTERNAL_NET = ${ADOPTED_INTERNALAPI_NETWORK} +tripleo_deploy: export ADOPTED_STORAGE_NET = ${ADOPTED_STORAGE_NETWORK} +tripleo_deploy: export ADOPTED_STORAGE_MGMT_NET = ${ADOPTED_STORAGEMGMT_NETWORK} +tripleo_deploy: export ADOPTED_TENANT_NET = ${ADOPTED_TENANT_NETWORK} +tripleo_deploy: export ADOPTED_EXTERNAL_NET = ${ADOPTED_EXTERNAL_NETWORK} tripleo_deploy: $(eval $(call vars)) scripts/tripleo.sh @@ -480,6 +501,16 @@ standalone_deploy: export COMPUTE_DHCP_AGENT_ENABLED=${EDPM_COMPUTE_DHCP_AGENT_E standalone_deploy: export STANDALONE=true standalone_deploy: export INTERFACE_MTU=${NETWORK_MTU} standalone_deploy: export EDPM_COMPUTE_NETWORK = ${NETWORK_ISOLATION_NET_NAME} +standalone_deploy: export NETWORK_INTERNAL_PREFIX = ${STANDALONE_INTERNALAPI_NET_PREFIX} +standalone_deploy: export NETWORK_STORAGE_PREFIX = ${STANDALONE_STORAGE_NET_PREFIX} +standalone_deploy: export NETWORK_STORAGE_MGMT_PREFIX = ${STANDALONE_STORAGEMGMT_NET_PREFIX} +standalone_deploy: export NETWORK_TENANT_PREFIX = ${STANDALONE_TENANT_NET_PREFIX} +standalone_deploy: export NETWORK_EXTERNAL_PREFFIX = ${STANDALONE_EXTERNAL_NET_PREFFIX} +standalone_deploy: export ADOPTED_INTERNAL_NET = ${ADOPTED_INTERNALAPI_NETWORK} +standalone_deploy: export ADOPTED_STORAGE_NET = ${ADOPTED_STORAGE_NETWORK} +standalone_deploy: export ADOPTED_STORAGE_MGMT_NET = ${ADOPTED_STORAGEMGMT_NETWORK} +standalone_deploy: export ADOPTED_TENANT_NET = ${ADOPTED_TENANT_NETWORK} +standalone_deploy: export ADOPTED_EXTERNAL_NET = ${ADOPTED_EXTERNAL_NETWORK} standalone_deploy: $(eval $(call vars)) scripts/standalone.sh ${EDPM_COMPUTE_SUFFIX} ${STANDALONE_COMPUTE_DRIVER} '${EDPM_COMPUTE_ADDITIONAL_NETWORKS}' '${EDPM_COMPUTE_ADDITIONAL_HOST_ROUTES}' diff --git a/devsetup/scripts/standalone.sh b/devsetup/scripts/standalone.sh index 5de5e1d56..90ffde6af 100755 --- a/devsetup/scripts/standalone.sh +++ b/devsetup/scripts/standalone.sh @@ -214,6 +214,15 @@ dns_server: ${PRIMARY_RESOLV_CONF_ENTRY} compute_driver: ${COMPUTE_DRIVER} sriov_agent: ${EDPM_COMPUTE_SRIOV_ENABLED} dhcp_agent: ${EDPM_COMPUTE_DHCP_AGENT_ENABLED} +# Adoption related stuff +adopted_internal_net: ${ADOPTED_INTERNAL_NET} +adopted_storage_net: ${ADOPTED_STORAGE_NET} +adopted_storage_mgmt_net: ${ADOPTED_STORAGE_MGMT_NET} +adopted_tenant_net: ${ADOPTED_TENANT_NET} +standalone_internal_prefix: ${NETWORK_INTERNAL_PREFIX} +standalone_storage_prefix: ${NETWORK_STORAGE_PREFIX} +standalone_storage_mgmt_prefix: ${NETWORK_STORAGE_MGMT_PREFIX} +standalone_tenant_prefix: ${NETWORK_TENANT_PREFIX} EOF jinja2_render standalone/network_data.j2 "${J2_VARS_FILE}" > ${MY_TMP_DIR}/network_data.yaml diff --git a/devsetup/scripts/tripleo.sh b/devsetup/scripts/tripleo.sh index 490f7eddf..a73859051 100755 --- a/devsetup/scripts/tripleo.sh +++ b/devsetup/scripts/tripleo.sh @@ -165,6 +165,17 @@ manage_default_route: ${TRIPLEO_NETWORKING} dns_server: ${PRIMARY_RESOLV_CONF_ENTRY} user_home: /home/zuul cloud_domain: ${CLOUD_DOMAIN} +# Adoption related stuff +adopted_internal_net: ${ADOPTED_INTERNAL_NET} +adopted_storage_net: ${ADOPTED_STORAGE_NET} +adopted_storage_mgmt_net: ${ADOPTED_STORAGE_MGMT_NET} +adopted_tenant_net: ${ADOPTED_TENANT_NET} +adopted_external_net: ${ADOPTED_EXTERNAL_NET} +standalone_internal_prefix: ${NETWORK_INTERNAL_PREFIX} +standalone_storage_prefix: ${NETWORK_STORAGE_PREFIX} +standalone_storage_mgmt_prefix: ${NETWORK_STORAGE_MGMT_PREFIX} +standalone_tenant_prefix: ${NETWORK_TENANT_PREFIX} +standalone_external_prefix: ${NETWORK_EXTERNAL_PREFIX} EOF jinja2_render ${SCRIPTPATH}/../tripleo/undercloud.conf.j2 "${J2_VARS_FILE}" > ${MY_TMP_DIR}/undercloud.conf diff --git a/devsetup/standalone/net_config.j2 b/devsetup/standalone/net_config.j2 index 4062b6b4b..f62999295 100644 --- a/devsetup/standalone/net_config.j2 +++ b/devsetup/standalone/net_config.j2 @@ -42,32 +42,56 @@ network_config: mtu: {{ interface_mtu }} vlan_id: 20 addresses: - - ip_netmask: 172.17.0.{{ ip_address_suffix }}/24 - - ip_netmask: 172.17.0.2/32 + - ip_netmask: {{ standalone_internal_prefix | default("172.17.0", true) }}.{{ ip_address_suffix }}/24 + - ip_netmask: {{ standalone_internal_prefix | default("172.17.0", true) }}.2/32 + {% if adopted_internal_net|default %} + routes: + - ip_netmask: {{ adopted_internal_net }} + next_hop: {{ standalone_internal_prefix | default("172.17.0", true) }}.{{ ip_address_suffix }} + {% else %} routes: [] + {% endif %} # storage - type: vlan mtu: {{ interface_mtu }} vlan_id: 21 addresses: - - ip_netmask: 172.18.0.{{ ip_address_suffix }}/24 - - ip_netmask: 172.18.0.2/32 + - ip_netmask: {{ standalone_storage_prefix | default("172.18.0", true) }}.{{ ip_address_suffix }}/24 + - ip_netmask: {{ standalone_storage_prefix | default("172.18.0", true) }}.2/32 + {% if adopted_storage_net|default %} + routes: + - ip_netmask: {{ adopted_storage_net }} + next_hop: {{ standalone_storage_prefix | default("172.18.0", true) }}.{{ ip_address_suffix }} + {% else %} routes: [] + {% endif %} # storage_mgmt - type: vlan mtu: {{ interface_mtu }} vlan_id: 23 addresses: - - ip_netmask: 172.20.0.{{ ip_address_suffix }}/24 - - ip_netmask: 172.20.0.2/32 + - ip_netmask: {{ standalone_storage_mgmt_prefix | default("172.20.0", true) }}.{{ ip_address_suffix }}/24 + - ip_netmask: {{ standalone_storage_mgmt_prefix | default("172.20.0", true) }}.2/32 + {% if adopted_storage_mgmt_net|default %} + routes: + - ip_netmask: {{ adopted_storage_mgmt_net }} + next_hop: {{ standalone_storage_mgmt_prefix | default("172.20.0", true) }}.{{ ip_address_suffix }} + {% else %} routes: [] + {% endif %} # tenant - type: vlan mtu: {{ interface_mtu }} vlan_id: 22 addresses: - - ip_netmask: 172.19.0.{{ ip_address_suffix }}/24 + - ip_netmask: {{ standalone_tenant_prefix | default("172.19.0", true) }}.{{ ip_address_suffix }}/24 + {% if adopted_tenant_net|default %} + routes: + - ip_netmask: {{ adopted_tenant_net }} + next_hop: {{ standalone_tenant_prefix | default("172.19.0", true) }}.{{ ip_address_suffix }} + {% else %} routes: [] + {% endif %} {%- for network in additional_networks if network.standalone_config %} {%- set net = network.standalone_config %} - type: {{ net.type }} diff --git a/devsetup/tripleo/net_config.j2 b/devsetup/tripleo/net_config.j2 index 62c7be182..ea26e168e 100644 --- a/devsetup/tripleo/net_config.j2 +++ b/devsetup/tripleo/net_config.j2 @@ -33,37 +33,67 @@ network_config: mtu: {{ interface_mtu }} vlan_id: 44 addresses: - - ip_netmask: 172.21.0.{{ ip_address_suffix }}/24 - - ip_netmask: 172.21.0.2/32 + - ip_netmask: {{ standalone_external_prefix | default("172.21.0", true)}}.{{ ip_address_suffix }}/24 + - ip_netmask: {{ standalone_external_prefix | default("172.21.0", true)}}.2/32 + {% if adopted_external_net|default %} + routes: + - ip_netmask: {{ adopted_external_net }} + next_hop: {{ standalone_external_prefix | default("172.21.0", true) }}.{{ip_address_suffix}} + {% else %} routes: [] + {% endif %} # internal - type: vlan mtu: {{ interface_mtu }} vlan_id: 20 addresses: - - ip_netmask: 172.17.0.{{ ip_address_suffix }}/24 - - ip_netmask: 172.17.0.2/32 + - ip_netmask: {{ standalone_internal_prefix | default("172.17.0", true)}}.{{ ip_address_suffix }}/24 + - ip_netmask: {{ standalone_internal_prefix | default("172.17.0", true)}}.2/32 + {% if adopted_internal_net|default %} + routes: + - ip_netmask: {{ adopted_internal_net }} + next_hop: {{ standalone_internal_prefix | default("172.17.0", true) }}.{{ip_address_suffix}} + {% else %} routes: [] + {% endif %} # storage - type: vlan mtu: {{ interface_mtu }} vlan_id: 21 addresses: - - ip_netmask: 172.18.0.{{ ip_address_suffix }}/24 - - ip_netmask: 172.18.0.2/32 + - ip_netmask: {{ standalone_storage_prefix | default("172.18.0", true) }}.{{ ip_address_suffix }}/24 + - ip_netmask: {{ standalone_storage_prefix | default("172.18.0", true) }}.2/32 + {% if adopted_storage_net|default %} + routes: + - ip_netmask: {{ adopted_storage_net }} + next_hop: {{ standalone_storage_prefix | default("172.18.0", true) }}.{{ ip_address_suffix }} + {% else %} routes: [] + {% endif %} # storage_mgmt - type: vlan mtu: {{ interface_mtu }} vlan_id: 23 addresses: - - ip_netmask: 172.20.0.{{ ip_address_suffix }}/24 - - ip_netmask: 172.20.0.2/32 + - ip_netmask: {{ standalone_storage_mgmt_prefix | default("172.20.0", true) }}.{{ ip_address_suffix }}/24 + - ip_netmask: {{ standalone_storage_mgmt_prefix | default("172.20.0", true) }}.2/32 + {% if adopted_storage_mgmt_net|default %} + routes: + - ip_netmask: {{ adopted_storage_mgmt_net }} + next_hop: {{ standalone_storage_mgmt_prefix | default("172.20.0", true) }}.{{ ip_address_suffix }} + {% else %} routes: [] + {% endif %} # tenant - type: vlan mtu: {{ interface_mtu }} vlan_id: 22 addresses: - - ip_netmask: 172.19.0.{{ ip_address_suffix }}/24 + - ip_netmask: {{ standalone_tenant_prefix | default("172.19.0", true) }}.{{ ip_address_suffix }}/24 + {% if adopted_tenant_net|default %} + routes: + - ip_netmask: {{ adopted_tenant_net }} + next_hop: {{ standalone_tenant_prefix | default("172.19.0", true) }}.{{ ip_address_suffix }} + {% else %} routes: [] + {% endif %} diff --git a/devsetup/tripleo/overcloud_net_config_scenario_a.j2 b/devsetup/tripleo/overcloud_net_config_scenario_a.j2 new file mode 100644 index 000000000..5fd1285ed --- /dev/null +++ b/devsetup/tripleo/overcloud_net_config_scenario_a.j2 @@ -0,0 +1,86 @@ +#jinja2: trim_blocks:False +network_config: +- type: ovs_bridge + name: br-ex + mtu: {{ interface_mtu }} + use_dhcp: false + dns_servers: + {% if dns_server %} + - {{ dns_server }} + {% endif %} + - {{ gateway_ip }} + domain: [] + addresses: + - ip_netmask: {{ ctlplane_ip }}/{{ ctlplane_cidr }} + routes: [] + members: + - type: interface + name: {{ os_net_config_iface | default(nic1) }} + mtu: {{ interface_mtu }} + # force the MAC address of the bridge to this interface + primary: true + # external + - type: vlan + mtu: {{ interface_mtu }} + vlan_id: 44 + addresses: + - ip_netmask: {{ standalone_external_prefix | default("172.21.0", true)}}.{{ ip_address_suffix }}/24 + {% if adopted_external_net|default %} + routes: + - ip_netmask: {{ adopted_external_net }} + next_hop: {{ standalone_external_prefix | default("172.21.0", true) }}.{{ip_address_suffix}} + {% else %} + routes: [] + {% endif %} + # internal + - type: vlan + mtu: {{ interface_mtu }} + vlan_id: 20 + addresses: + - ip_netmask: {{ standalone_internal_prefix | default("172.17.0", true)}}.{{ ip_address_suffix }}/24 + {% if adopted_internal_net|default %} + routes: + - ip_netmask: {{ adopted_internal_net }} + next_hop: {{ standalone_internal_prefix | default("172.17.0", true) }}.{{ip_address_suffix}} + {% else %} + routes: [] + {% endif %} + # storage + - type: vlan + mtu: {{ interface_mtu }} + vlan_id: 21 + addresses: + - ip_netmask: {{ standalone_storage_prefix | default("172.18.0", true) }}.{{ ip_address_suffix }}/24 + {% if adopted_storage_net|default %} + routes: + - ip_netmask: {{ adopted_storage_net }} + next_hop: {{ standalone_storage_prefix | default("172.18.0", true) }}.{{ ip_address_suffix }} + {% else %} + routes: [] + {% endif %} + # storage_mgmt + - type: vlan + mtu: {{ interface_mtu }} + vlan_id: 23 + addresses: + - ip_netmask: {{ standalone_storage_mgmt_prefix | default("172.20.0", true) }}.{{ ip_address_suffix }}/24 + {% if adopted_storage_mgmt_net|default %} + routes: + - ip_netmask: {{ adopted_storage_mgmt_net }} + next_hop: {{ standalone_storage_mgmt_prefix | default("172.20.0", true) }}.{{ ip_address_suffix }} + {% else %} + routes: [] + {% endif %} + # tenant + - type: vlan + mtu: {{ interface_mtu }} + vlan_id: 22 + addresses: + - ip_netmask: {{ standalone_tenant_prefix | default("172.19.0", true) }}.{{ ip_address_suffix }}/24 + {% if adopted_tenant_net|default %} + routes: + - ip_netmask: {{ adopted_tenant_net }} + next_hop: {{ standalone_tenant_prefix | default("172.19.0", true) }}.{{ ip_address_suffix }} + {% else %} + routes: [] + {% endif %} diff --git a/scripts/gen-metallb-config.sh b/scripts/gen-metallb-config.sh index 7d2b066c1..cf4798db5 100644 --- a/scripts/gen-metallb-config.sh +++ b/scripts/gen-metallb-config.sh @@ -92,7 +92,7 @@ spec: EOF_CAT if [ -n "$IPV4_ENABLED" ]; then cat >> ${DEPLOY_DIR}/ipaddresspools.yaml <> ${DEPLOY_DIR}/ipaddresspools.yaml <> ${DEPLOY_DIR}/ipaddresspools.yaml <> ${DEPLOY_DIR}/internalapi.yaml <> ${DEPLOY_DIR}/internalapi.yaml <> ${DEPLOY_DIR}/storage.yaml <> ${DEPLOY_DIR}/storage.yaml <> ${DEPLOY_DIR}/tenant.yaml <> ${DEPLOY_DIR}/tenant.yaml <> ${DEPLOY_DIR}/storagemgmt.yaml <> ${DEPLOY_DIR}/storagemgmt.yaml < ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> "${DEPLOY_DIR}/${WORKER}_nncp.yaml" <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <> ${DEPLOY_DIR}/${WORKER}_nncp.yaml <