FEAT: RoCE and Hipersocket support for LPAR Cluster (#338)

Adding support for the Roce and Hipersocket for LPAR Cluster Introduced one new variable to all.yaml and two new variables to the host_vars_template In all.yaml `internal_ip` which need to defined when bringing up the cluster with hipersocket cards. In host_vars `mode` tells the network card mode like osa/roce/hipersocket `internal_ip` needs to be defined when mode is hipersocket --------- Signed-off-by: K Shiva Sai <[email protected]> Co-authored-by: K Shiva Sai <[email protected]>
IBM · Oct 15, 2024 · 41f3a59 · 41f3a59
1 parent fbbe4b0
commit 41f3a59
Show file tree

Hide file tree

Showing 12 changed files with 85 additions and 6 deletions.
diff --git a/docs/set-variables-group-vars.md b/docs/set-variables-group-vars.md
@@ -65,6 +65,7 @@
 **env.bastion.resources.vcpu** | How many virtual CPUs would you like to allocate to the bastion? Recommended 4 or more. | 4
 **env.bastion.resources.vcpu_model_option** | Configure the CPU model and CPU features exposed to the guest | --cpu host
 **env.bastion.networking.ip** | IPv4 address for the bastion. | 192.168.10.3
+**env.bastion.networking.internal_ip** | Private IPv4 address for bastion required when installing LPAR cluster with HiperSocket. Currently supports only when bastion is on LPAR or on zVM host. Incase of zVM bastion enable the HiperSocket prior to the playbook run with vmcp commands on the bastion. Alternative Option would be setting up the bridge port on OSA or RoCE. | 10.42.16.1  
 **env.bastion.networking.ipv6** | IPv6 address for the bastion if use_ipv6 variable is 'True'. | fd00::3
 **env.bastion.networking.mac** | MAC address for the bastion if use_dhcp variable is 'True'. | 52:54:00:18:1A:2B
 **env.bastion.networking.hostname** | Hostname of the bastion. Will be combined with env.bastion.networking.base_domain to create a Fully Qualified Domain Name (FQDN). | ocpz-bastion

diff --git a/docs/set-variables-host-vars.md b/docs/set-variables-host-vars.md
@@ -15,6 +15,8 @@
 :--- | :--- | :---
 **networking.hostname** | The hostname of the LPAR with RHEL installed natively (the KVM host). | kvm-host-01
 **networking.ip** | The IPv4 address of the LPAR with RHEL installed natively (the KVM host). | 192.168.10.2
+**networking.internal_ip** | The internal IPv4 address of the LPAR required when booting the LPAR with HiperSocket card. Currently supports only when bastion is on LPAR or on zVM host. Incase of zVM bastion enable the HiperSocket card prior to the playbook run with vmcp commands on the bastion. Alternative Option would be setting up the bridge port on OSA or RoCE.| 10.42.6.2
+**networking.mode** | Type of network card | osa/roce/hipersocket 
 **networking.ipv6** | IPv6 address for the bastion if use_ipv6 variable is 'True'. | fd00::3
 **networking.subnetmask** | The subnet that the LPAR resides in within your network. | 255.255.255.0
 **networking.gateway** | The IPv4 address of the gateway to the network where the KVM host resides. | 192.168.10.0

diff --git a/inventories/default/group_vars/all.yaml.template b/inventories/default/group_vars/all.yaml.template
@@ -63,6 +63,7 @@ env:
       vcpu_model_option: "--cpu host"
     networking:
       ip: #X
+      internal_ip: #X Required only when creating LPAR cluster with HiperSocket.
       ipv6: #X
       mac: #X
       hostname: #X

diff --git a/inventories/default/host_vars/KVMhostname1-here.yaml.template b/inventories/default/host_vars/KVMhostname1-here.yaml.template
@@ -1,5 +1,7 @@
 # Section 1 - KVM Host
 networking:
+  mode: #X Type of network type. Required only when creating LPAR cluster
+  internal_ip: #X Internal IP for the LPAR Required only when booting LPAR cluster with HiperSockets.
   hostname: #X
   ip: #X
   ipv6: #X

diff --git a/inventories/default/host_vars/KVMhostname2-here.yaml.template b/inventories/default/host_vars/KVMhostname2-here.yaml.template
@@ -1,5 +1,7 @@
 # Section 1 - KVM Host
 networking:
+  mode: #X Type of network type. Required only when creating LPAR cluster
+  internal_ip: #X Internal IP for the LPAR Required only when booting LPAR cluster with HiperSockets.
   hostname: #X
   ip: #X
   ipv6: #X

diff --git a/inventories/default/host_vars/KVMhostname3-here.yaml.template b/inventories/default/host_vars/KVMhostname3-here.yaml.template
@@ -1,5 +1,7 @@
 # Section 1 - KVM Host
 networking:
+  mode: #X Type of network type. Required only when creating LPAR cluster
+  internal_ip: #X Internal IP for the LPAR Required only when booting LPAR cluster with HiperSockets.
   hostname: #X
   ip: #X
   ipv6: #X

diff --git a/playbooks/5_setup_bastion.yaml b/playbooks/5_setup_bastion.yaml
@@ -66,6 +66,30 @@
   roles:
     - ssh_copy_id
 
+- name: 5 Setting up of HiperSocket card on bastion when booting LPAR nodes with HiperSocket card (if in lpar)
+  hosts: bastion
+  become: True
+  tasks:
+  - name: Getting the Hipervisor Details
+    command: lscpu
+    register: hipervisor_info
+  - name: Fail the play if bastion is on the KVM guest
+    fail:
+      msg: Currently not supported for LPAR clsuter with this particular mode when bastion is on KVM host.
+    when: "'KVM' in hipervisor_info.stdout"
+  - name: Check if vars file exists
+    stat:
+      path: "{{ inventory_dir }}/host_vars/{{ env.cluster.nodes.bootstrap.vm_name }}.yaml"
+    register: vars_file
+  - name: Include the vars
+    include_vars:
+      file: "{{ inventory_dir }}/host_vars/{{ env.cluster.nodes.bootstrap.vm_name }}.yaml"
+    when: vars_file.stat.exists
+  - name: Attaching HiperSocket card to bastion
+    ansible.builtin.include_tasks:
+      file: ../roles/bastion_setup_hipersocket_LPAR/tasks/main.yml
+    when: vars_file.stat.exists and networking.mode is defined and networking.mode | lower =='hipersocket'
+
 - name: 5 setup bastion - configure bastion node with essential services
   hosts: bastion
   tags: services, section_2

diff --git a/roles/bastion_setup_hipersocket_LPAR/tasks/main.yml b/roles/bastion_setup_hipersocket_LPAR/tasks/main.yml
@@ -0,0 +1,29 @@
+- name: Setting the Hipersocket card on the bastion
+  command:
+    cmd: "chzdev -e {{ lpar.networking.nic.card1.dev_num.split(',')[0].split('.')[-1] }}-{{ lpar.networking.nic.card1.dev_num.split(',')[-1].split('.')[-1]  }} layer2=1"
+
+- name: Add the IP address to the hipersocket card on bastion
+  command:
+    cmd: ip addr add '{{ env.bastion.networking.internal_ip }}/24' dev '{{ networking.device1 }}'
+  register: result
+  failed_when: "'exists' not in result.stderr and result.rc != 0"
+
+- name: Enabiling the IP forwarding to OSA when networking mode is hipersocket
+  command:
+    cmd: sysctl -w net.ipv4.ip_forward=1
+
+- name: Allowing masquerade on bastion when networking mode is hipersocket
+  command:
+    cmd: iptables -t nat -A POSTROUTING -o '{{ env.cluster.networking.interface }}' -j MASQUERADE
+
+- name: Enabiling the firewall masquerade
+  firewalld:
+    masquerade: yes
+    permanent: true
+    state: enabled
+    zone: public
+
+- name: Restart firewall service
+  systemd:
+    name: firewalld
+    state: restarted
diff --git a/roles/boot_LPAR/tasks/main.yaml b/roles/boot_LPAR/tasks/main.yaml
@@ -31,7 +31,7 @@
       set_fact:
         ipv6_string: "ip=[{{ node.networking.ipv6 }}]::[{{ node.networking.ipv6_gateway }}]:{{ node.networking.ipv6_prefix }}::{{ env.cluster.networking.interface }}:none"
       when: env.use_ipv6  == true
-    
+
     - name: set live disk lun
       set_fact:
         live_disk_lun: "{{ node.lpar.livedisk.lun if (node.lpar.livedisk.lun is defined and node.lpar.livedisk.lun is not none) else 'na' }}"
@@ -60,11 +60,11 @@
           --netset_ip {{ node.networking.ip }} \ 
           --netset_gateway {{ node.networking.gateway }} \
           --netset_network_type osa \
-          --netset_network_device "{{ node.lpar.networking.nic.card1.dev_num }}" \
+          --netset_network_device 0.0.1100,0.0.1101,0.0.1102 \
           --netset_password {{ node.lpar.livedisk.livedisk_root_pass }} \
           --netset_dns "{{ node.networking.nameserver1 }},{{ node.networking.nameserver2 }}" \
           --log_level DEBUG \
-          --cmdline 'rd.neednet=1 console=ttysclp0 {% if node.lpar.storage_group_1.type | lower == "fcp" %}coreos.inst.install_dev=sda{% endif %}{% if node.lpar.storage_group_1.type | lower == "dasd" %}coreos.inst.install_dev=/dev/dasda{% endif %} coreos.live.rootfs_url=http://{{ env.bastion.networking.ip }}:8080/bin/{{ rhcos_live_rootfs }} coreos.inst.ignition_url=http://{{ env.bastion.networking.ip }}:8080/ignition/{{ ignition }}.ign ip={{ node.networking.ip }}::{{ node.networking.gateway }}:{{ node.networking.subnetmask }}:{{ node.networking.hostname }}.{{ env.cluster.networking.metadata_name }}.{{ env.cluster.networking.base_domain }}:{{ node.networking.device1 }}:none nameserver={{ env.cluster.networking.nameserver1 }} cio_ignore=all,!condev zfcp.allow_lun_scan=0 rd.znet=qeth,{{ node.lpar.networking.nic.card1.dev_num }},layer2=1 {{ ipv6_string }} {% if node.lpar.storage_group_1.type | lower == "fcp" %}{{ rd_zfcp_string }}{% endif %}{% if node.lpar.storage_group_1.type | lower == "dasd" %}{{ rd_dasd_string }}{% endif %}'
+          --cmdline 'rd.neednet=1 console=ttysclp0 {% if node.lpar.storage_group_1.type | lower == "fcp" %}coreos.inst.install_dev=sda{% endif %}{% if node.lpar.storage_group_1.type | lower == "dasd" %}coreos.inst.install_dev=/dev/dasda{% endif %} coreos.live.rootfs_url=http://{% if node.networking.mode | lower == "hipersocket" %}{{ env.bastion.networking.internal_ip }}{% else %}{{ env.bastion.networking.ip }}{% endif %}:8080/bin/{{ rhcos_live_rootfs }} coreos.inst.ignition_url=http://{% if node.networking.mode | lower == "hipersocket" %}{{ env.bastion.networking.internal_ip }}{% else %}{{ env.bastion.networking.ip }}{% endif %}:8080/ignition/{{ ignition }}.ign ip={% if node.networking.mode | lower == "hipersocket" %}{{ node.networking.internal_ip }}{% else %}{{ node.networking.ip }}{% endif %}::{{ node.networking.nameserver1 }}:{{ node.networking.subnetmask }}:{{ node.networking.hostname }}.{{ env.cluster.networking.metadata_name }}.{{ env.cluster.networking.base_domain }}:{{ node.networking.device1 }}:none nameserver={{ env.cluster.networking.nameserver1 }} cio_ignore=all,!condev zfcp.allow_lun_scan=0 {% if node.networking.mode | lower != "roce" %}rd.znet=qeth,{{ node.lpar.networking.nic.card1.dev_num }},layer2=1{% endif %} {{ ipv6_string }} {% if node.lpar.storage_group_1.type | lower == "fcp" %}{{ rd_zfcp_string }}{% endif %}{% if node.lpar.storage_group_1.type | lower == "dasd" %}{{ rd_dasd_string }}{% endif %}'
 
     - name: Booting lpar node
       shell: |
@@ -85,8 +85,8 @@
           --netset_ip {{ node.networking.ip }} \ 
           --netset_gateway {{ node.networking.gateway }} \
           --netset_network_type osa \
-          --netset_network_device "{{ node.lpar.networking.nic.card1.dev_num }}" \
+          --netset_network_device 0.0.1100,0.0.1101,0.0.1102 \
           --netset_password {{ node.lpar.livedisk.livedisk_root_pass }} \
           --netset_dns "{{ node.networking.nameserver1 }},{{ node.networking.nameserver2 }}" \
           --log_level DEBUG \
-          --cmdline 'rd.neednet=1 console=ttysclp0 {% if node.lpar.storage_group_1.type | lower == "fcp" %}coreos.inst.install_dev=sda{% endif %}{% if node.lpar.storage_group_1.type | lower == "dasd" %}coreos.inst.install_dev=/dev/dasda{% endif %} coreos.live.rootfs_url=http://{{ env.bastion.networking.ip }}:8080/bin/{{ rhcos_live_rootfs }} coreos.inst.ignition_url=http://{{ env.bastion.networking.ip }}:8080/ignition/{{ ignition }}.ign ip={{ node.networking.ip }}::{{ node.networking.gateway }}:{{ node.networking.subnetmask }}:{{ node.networking.hostname }}.{{ env.cluster.networking.metadata_name }}.{{ env.cluster.networking.base_domain }}:{{ node.networking.device1 }}:none nameserver={{ env.cluster.networking.nameserver1 }} cio_ignore=all,!condev zfcp.allow_lun_scan=0 rd.znet=qeth,{{ node.lpar.networking.nic.card1.dev_num }},layer2=1 {{ ipv6_string }} {% if node.lpar.storage_group_1.type | lower == "fcp" %}{{ rd_zfcp_string }}{% endif %}{% if node.lpar.storage_group_1.type | lower == "dasd" %}{{ rd_dasd_string }}{% endif %}'
+          --cmdline 'rd.neednet=1 console=ttysclp0 {% if node.lpar.storage_group_1.type | lower == "fcp" %}coreos.inst.install_dev=sda{% endif %}{% if node.lpar.storage_group_1.type | lower == "dasd" %}coreos.inst.install_dev=/dev/dasda{% endif %} coreos.live.rootfs_url=http://{% if node.networking.mode | lower == "hipersocket" %}{{ env.bastion.networking.internal_ip }}{% else %}{{ env.bastion.networking.ip }}{% endif %}:8080/bin/{{ rhcos_live_rootfs }} coreos.inst.ignition_url=http://{% if node.networking.mode | lower == "hipersocket" %}{{ env.bastion.networking.internal_ip }}{% else %}{{ env.bastion.networking.ip }}{% endif %}:8080/ignition/{{ ignition }}.ign ip={% if node.networking.mode | lower == "hipersocket" %}{{ node.networking.internal_ip }}{% else %}{{ node.networking.ip }}{% endif %}::{{ node.networking.nameserver1 }}:{{ node.networking.subnetmask }}:{{ node.networking.hostname }}.{{ env.cluster.networking.metadata_name }}.{{ env.cluster.networking.base_domain }}:{{ node.networking.device1 }}:none nameserver={{ env.cluster.networking.nameserver1 }} cio_ignore=all,!condev zfcp.allow_lun_scan=0 {% if node.networking.mode | lower != "roce" %}rd.znet=qeth,{{ node.lpar.networking.nic.card1.dev_num }},layer2=1{% endif %} {{ ipv6_string }} {% if node.lpar.storage_group_1.type | lower == "fcp" %}{{ rd_zfcp_string }}{% endif %}{% if node.lpar.storage_group_1.type | lower == "dasd" %}{{ rd_dasd_string }}{% endif %}'
diff --git a/roles/check_dns/tasks/main.yaml b/roles/check_dns/tasks/main.yaml
@@ -5,6 +5,15 @@
   shell: "dig +short {{ env.bastion.networking.hostname }}.{{ env.bastion.networking.base_domain }} | tail -n1"
   register: bastion_lookup
   failed_when: env.bastion.networking.ip != bastion_lookup.stdout
+  when: env.bastion.networking.internal_ip is not defined or env.bastion.networking.internal_ip == None
+
+- name: Check internal cluster DNS resolution for the bastion with Internal_IP
+  tags: check_dns, dns
+  shell: "dig +short {{ env.bastion.networking.hostname }}.{{ env.bastion.networking.base_domain }} | tail -n1"
+  register: bastion_lookup
+  failed_when: env.bastion.networking.internal_ip != bastion_lookup.stdout
+  when: env.bastion.networking.internal_ip is defined and env.bastion.networking.internal_ip != None
+
 
 - name: Check internal cluster DNS resolution for external API and apps services
   tags: check_dns, dns

diff --git a/roles/dns/tasks/main.yaml b/roles/dns/tasks/main.yaml
@@ -16,6 +16,13 @@
   tags: dns
   ansible.builtin.set_fact:
     bastion_split_ip: "{{ env.bastion.networking.ip.split('.') }}"
+  when: env.bastion.networking.internal_ip is not defined or env.bastion.networking.internal_ip == None
+
+- name: Split Bastion Internal_IP addresses for use in templates
+  tags: dns
+  ansible.builtin.set_fact:
+    bastion_split_ip: "{{ env.bastion.networking.internal_ip.split('.') }}"
+  when: env.bastion.networking.internal_ip is defined and env.bastion.networking.internal_ip != None
 
 - name: Split Bootstrap IP addresses for use in templates
   tags: dns

diff --git a/roles/dns/templates/dns.db.j2 b/roles/dns/templates/dns.db.j2
@@ -11,7 +11,7 @@ $TTL 86400
 @ IN NS {{ env.bastion.networking.hostname }}.{{ env.bastion.networking.base_domain }}.
 
 ;IP Address for Name Server
-{{ env.bastion.networking.hostname }} IN A {{ env.bastion.networking.ip }}
+{{ env.bastion.networking.hostname }} IN A {% if env.bastion.networking.internal_ip is defined and env.bastion.networking.internal_ip!=None %}{{ env.bastion.networking.internal_ip }}{% else %}{{ env.bastion.networking.ip }}{% endif %}
 
 ;entry for bootstrap host.
 {% if env.cluster.nodes.bootstrap is defined %}