From 0de0e691e97f046de2cfc2cb7cc825cd73823df5 Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Wed, 26 Oct 2022 00:01:32 -0700
Subject: [PATCH 01/26] fail if interface script not exist and add prepend DNS
 to dhclient.conf if bastion is booting from DHCP.

---
 playbooks/tasks/dns_interface.yaml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/playbooks/tasks/dns_interface.yaml b/playbooks/tasks/dns_interface.yaml
index ce6c06d..a3b2af3 100644
--- a/playbooks/tasks/dns_interface.yaml
+++ b/playbooks/tasks/dns_interface.yaml
@@ -5,6 +5,10 @@
     path: "{{interface_script}}"
   register: stat_result
 
+- fail:
+    msg: "the network interface file {{interface_script}} not exist. Without it, the DNS config would not be correct."
+  when: not stat_result.stat.exists
+
 - name: Change DNS service in interface script
   lineinfile:
     path: "{{interface_script}}"
@@ -13,6 +17,24 @@
   when: stat_result.stat.exists==True
   register: interface_changed
 
+- name: Check if interface boots from DHCP
+  shell: cat "{{interface_script}}"
+  when: stat_result.stat.exists
+  register: interface_boot_from_dhcp_stat_result
+
+- name: Check if dhclient config exists
+  stat:
+    path: "/etc/dhcp/dhclient.conf"
+  register: dhclient_conf_stat_result
+
+- name: Prepend local DNS server to dhclient if booting from DHCP
+  lineinfile:
+    path: "/etc/dhcp/dhclient.conf"
+    regexp: "^prepend*{{groups['bastion'][0]}}"
+    line: "prepend domain-name-servers {{groups['bastion'][0]}} ;"
+  when: dhclient_conf_stat_result.stat.exists==True and interface_boot_from_dhcp_stat_result.stdout.find('BOOTPROTO=dhcp') != -1
+  register: interface_dhclient_changed
+
 - name: Restart NetworkManager service
   service:
     name: NetworkManager

From 998b3a7221555b461d7df54cccf475029f49b944 Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Wed, 26 Oct 2022 02:14:21 -0700
Subject: [PATCH 02/26] add interface name variable to customize the network
 interface name .

---
 inventory/vmware-airgapped-example.inv | 1 +
 inventory/vmware-example-410.inv       | 1 +
 inventory/vmware-example-48-ipi.inv    | 3 ++-
 inventory/vmware-example-48.inv        | 1 +
 inventory/vmware-example-49.inv        | 1 +
 playbooks/tasks/tftpboot_vm_menu.j2    | 2 +-
 6 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/inventory/vmware-airgapped-example.inv b/inventory/vmware-airgapped-example.inv
index 8386bd5..d3fde84 100644
--- a/inventory/vmware-airgapped-example.inv
+++ b/inventory/vmware-airgapped-example.inv
@@ -7,6 +7,7 @@ ansible_ssh_common_args='-o StrictHostKeyChecking=no'
 domain_name="uk.ibm.com"
 cluster_name="ocp45"
 default_gateway="10.99.92.1"
+interface_name="ens192"
 
 #
 # Indicate the method by which Red Hat CoreOS will be installed. This can be one of the following values:
diff --git a/inventory/vmware-example-410.inv b/inventory/vmware-example-410.inv
index 1f57235..ffc086b 100644
--- a/inventory/vmware-example-410.inv
+++ b/inventory/vmware-example-410.inv
@@ -7,6 +7,7 @@ ansible_ssh_common_args='-o StrictHostKeyChecking=no'
 domain_name="coc.ibm.com"
 cluster_name="ocp410"
 default_gateway="10.99.92.1"
+interface_name="ens192"
 
 #
 # Indicate the method by which Red Hat CoreOS will be installed. This can be one of the following values:
diff --git a/inventory/vmware-example-48-ipi.inv b/inventory/vmware-example-48-ipi.inv
index 16b498b..ca0e8e3 100644
--- a/inventory/vmware-example-48-ipi.inv
+++ b/inventory/vmware-example-48-ipi.inv
@@ -7,6 +7,7 @@ ansible_ssh_common_args='-o StrictHostKeyChecking=no'
 domain_name="coc.ibm.com"
 cluster_name="ocp48"
 default_gateway="10.99.92.1"
+interface_name="ens192"
 
 #
 # Indicate the method by which Red Hat CoreOS will be installed. This can be one of the following values:
@@ -210,4 +211,4 @@ vm_worker_disk=200
 # Ignored for IPI installations
 
 [workers]
-# Ignored for IPI installations
\ No newline at end of file
+# Ignored for IPI installations
diff --git a/inventory/vmware-example-48.inv b/inventory/vmware-example-48.inv
index 42c67bf..c5d2670 100644
--- a/inventory/vmware-example-48.inv
+++ b/inventory/vmware-example-48.inv
@@ -7,6 +7,7 @@ ansible_ssh_common_args='-o StrictHostKeyChecking=no'
 domain_name="coc.ibm.com"
 cluster_name="ocp48"
 default_gateway="10.99.92.1"
+interface_name="ens192"
 
 #
 # Indicate the method by which Red Hat CoreOS will be installed. This can be one of the following values:
diff --git a/inventory/vmware-example-49.inv b/inventory/vmware-example-49.inv
index 1870b6d..c96788f 100644
--- a/inventory/vmware-example-49.inv
+++ b/inventory/vmware-example-49.inv
@@ -7,6 +7,7 @@ ansible_ssh_common_args='-o StrictHostKeyChecking=no'
 domain_name="coc.ibm.com"
 cluster_name="ocp49"
 default_gateway="10.99.92.1"
+interface_name="ens192"
 
 #
 # Indicate the method by which Red Hat CoreOS will be installed. This can be one of the following values:
diff --git a/playbooks/tasks/tftpboot_vm_menu.j2 b/playbooks/tasks/tftpboot_vm_menu.j2
index 042bd3e..fa6dd90 100644
--- a/playbooks/tasks/tftpboot_vm_menu.j2
+++ b/playbooks/tasks/tftpboot_vm_menu.j2
@@ -6,4 +6,4 @@ label linux
   menu label ^Install RHEL CoreOS
   menu default
   kernel /images/{{rhcos_kernel_package | basename}}
-  append initrd=/images/{{rhcos_initramfs_package | basename}} coreos.live.rootfs_url=http://{{groups['bastion'][0]}}:{{http_server_port}}/{{rhcos_rootfs_package}} rd.neednet=1 coreos.inst.install_dev=/dev/sda coreos.inst.ignition_url=http://{{groups['bastion'][0]}}:{{http_server_port}}/{{ocp_install_dir}}/{{node_type}}.ign ip={{item}}::{{default_gateway}}:255.255.255.0:{{hostvars[item]['host']}}.{{cluster_name}}.{{domain_name}}:ens192:none nameserver={{groups['bastion'][0]}} 
\ No newline at end of file
+  append initrd=/images/{{rhcos_initramfs_package | basename}} coreos.live.rootfs_url=http://{{groups['bastion'][0]}}:{{http_server_port}}/{{rhcos_rootfs_package}} rd.neednet=1 coreos.inst.install_dev=/dev/sda coreos.inst.ignition_url=http://{{groups['bastion'][0]}}:{{http_server_port}}/{{ocp_install_dir}}/{{node_type}}.ign ip={{item}}::{{default_gateway}}:255.255.255.0:{{hostvars[item]['host']}}.{{cluster_name}}.{{domain_name}}:{{interface_name}}:none nameserver={{groups['bastion'][0]}} 

From 16f6ee4c5bc95ff3e3b084e1a9ea135a5341ca75 Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Wed, 26 Oct 2022 03:22:45 -0700
Subject: [PATCH 03/26] fix the master number cannot less than 3.

---
 playbooks/tasks/dns_server.j2 | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/playbooks/tasks/dns_server.j2 b/playbooks/tasks/dns_server.j2
index 0774b88..22a0267 100644
--- a/playbooks/tasks/dns_server.j2
+++ b/playbooks/tasks/dns_server.j2
@@ -9,12 +9,12 @@ address=/api.{{cluster_name}}.{{domain_name}}/{{groups['lb'][0]}}
 address=/.apps.{{cluster_name}}.{{domain_name}}/{{groups['lb'][0]}}
 
 {% if rhcos_installation_method|upper != "IPI" %}
-address=/etcd-0.{{cluster_name}}.{{domain_name}}/{{groups['masters'][0]}}
-address=/etcd-1.{{cluster_name}}.{{domain_name}}/{{groups['masters'][1]}}
-address=/etcd-2.{{cluster_name}}.{{domain_name}}/{{groups['masters'][2]}}
-srv-host=_etcd-server-ssl._tcp.{{cluster_name}}.{{domain_name}},etcd-0.{{cluster_name}}.{{domain_name}},2380,0,10
-srv-host=_etcd-server-ssl._tcp.{{cluster_name}}.{{domain_name}},etcd-1.{{cluster_name}}.{{domain_name}},2380,0,10
-srv-host=_etcd-server-ssl._tcp.{{cluster_name}}.{{domain_name}},etcd-2.{{cluster_name}}.{{domain_name}},2380,0,10
+{% for master in groups['masters'] %}
+address=/etcd-{{loop.index - 1}}.{{cluster_name}}.{{domain_name}}/{{master}}
+{% endfor %}
+{% for master in groups['masters'] %}
+srv-host=_etcd-server-ssl._tcp.{{cluster_name}}.{{domain_name}},etcd-{{loop.index - 1}}.{{cluster_name}}.{{domain_name}},2380,0,10
+{% endfor %}
 {% endif %}
 
 {% if dhcp_on_bastion %}
@@ -42,4 +42,4 @@ tftp-root=/tftpboot
 dhcp-boot=pxelinux.0
 {% endif %}
 
-{% endif %}
\ No newline at end of file
+{% endif %}

From b5344e7a6826e8d5c92f85795ef5a3d2939c5fc5 Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Wed, 26 Oct 2022 16:10:10 -0700
Subject: [PATCH 04/26] add multi upstream DNS server support for dnsmasq

---
 playbooks/tasks/dns_server.j2 | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/playbooks/tasks/dns_server.j2 b/playbooks/tasks/dns_server.j2
index 22a0267..7c2af23 100644
--- a/playbooks/tasks/dns_server.j2
+++ b/playbooks/tasks/dns_server.j2
@@ -1,7 +1,9 @@
 # {{ ansible_managed }}
 
 no-resolv
-server={{external_name_servers[0]}}
+{% for upstream_dns in external_name_servers %}
+server={{upstream_dns}}
+{% endfor %}
 local=/{{cluster_name}}.{{domain_name}}/
 
 address=/api-int.{{cluster_name}}.{{domain_name}}/{{groups['lb'][0]}}

From 9e9285d637342a9d39a31ca7b7513de78f84c908 Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Thu, 27 Oct 2022 20:25:18 -0700
Subject: [PATCH 05/26] add multi subnet support for crossing network master
 and worker nodes.

---
 .gitignore                                    |   3 +
 .../vmware-example-48-with-new-options.inv    | 238 ++++++++++++++++++
 playbooks/tasks/dns_interface.yaml            |   8 +-
 playbooks/tasks/dns_server.j2                 |   7 +-
 playbooks/tasks/tftpboot_vm_menu.j2           |   2 +-
 5 files changed, 251 insertions(+), 7 deletions(-)
 create mode 100644 inventory/vmware-example-48-with-new-options.inv

diff --git a/.gitignore b/.gitignore
index e289565..78e6d21 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,5 @@
 .DS_Store
 scrap
+
+# for sed inplace edit backup
+*.bak
diff --git a/inventory/vmware-example-48-with-new-options.inv b/inventory/vmware-example-48-with-new-options.inv
new file mode 100644
index 0000000..4c2249e
--- /dev/null
+++ b/inventory/vmware-example-48-with-new-options.inv
@@ -0,0 +1,238 @@
+[all:children]
+dhcp
+masters
+workers
+
+[all:vars]
+ansible_ssh_common_args='-o StrictHostKeyChecking=no'
+domain_name="ocp.mycom.com"
+cluster_name="ocpmas"
+default_gateway="10.11.80.1"
+node_interface_name="ens192"
+
+#
+# Indicate the method by which Red Hat CoreOS will be installed. This can be one of the following values:
+# - pxe: CoreOS will be loaded on the bootstrap, masters and workers using PXE-boot. This is the default.
+# - ova: Only for VMWare. CoreOS will be loaded using a VMWare template (imported OVA file).
+# - ipi: Only for VMWare. CoreOS and OpenShift will be installed via Installer-provisioned infrastructure. OpenShift
+#        will take care of creating the required VMs and automatically assigns hostnames. If you choose this installation
+#        method, please also complete the IPI section in the inventory file.
+#
+# For pxe and ova installs you have the option of letting the prepare script create the VMs for you by setting
+# vm_create_vms=True. This can only be done if you specify the vc_user and vc_password properties at the command line.
+# If you specify run_install=True and vm_create_vms=True, the script will start the virtual machines. Otherwise, you must
+# start the bootstrap, masters and workers yourself while the script is waiting for the bootstrap.
+# When rhcos_installation_method=ipi, run_install is assumed to be True as well.
+#
+rhcos_installation_method=pxe
+vm_create_vms=False
+run_install=False
+
+#
+# OCP Installation directory
+# Depicts the directory on the bastion (current) node in which the OpenShift installation
+# artifacts will be stored.
+#
+ocp_install_dir="/root/prepare-ocp/artifact/ocp_install"
+
+#
+# Proxy settings. If the nodes can only reach the internet through a proxy, the proxy server must be configured
+# in the OpenShift installation configuration file. Make sure that you specify all 3 properties: http_proxy, https_proxy
+# and no_proxy. Property no_proxy must contain the domain name, the k8s internal addresses (10.1.0.0/16 and 172.30.0.0/16),
+# the internal services domain (.svc) and the IP range of the cluster nodes (for example 192.168.1.0/24).
+# Additionally, if the bastion (and other non-cluster nodes) have not yet 
+# been configured with a global proxy environment variable, the preparation script can add this to the profile of all users.
+#
+#http_proxy="http://bastion.{{cluster_name}}.{{domain_name}}:3128"
+#https_proxy="http://bastion.{{cluster_name}}.{{domain_name}}:3128"
+#no_proxy=".{{domain_name}},10.1.0.0/16,{{service_network}},10.99.92.0/24,.svc"
+#configure_global_proxy=True
+
+#
+# OpenShift download URLs. These are the URLs used by the prepare script to download
+# the packages needed. If you want to install a different release of OpenShift, it is sufficient to change the
+# openshift_release property. The download scripts automatically select the relevant packages from the URL.
+#
+openshift_release="4.8"
+openshift_base_url="https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-{{openshift_release}}/"
+rhcos_base_url="https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/{{openshift_release}}/latest/"
+openshift_client_package_pattern=".*(openshift-client.*tar.gz).*"
+openshift_install_package_pattern=".*(openshift-install-linux.*tar.gz).*"
+rhcos_metal_package_pattern=".*(rhcos.*metal.*raw.gz).*"
+rhcos_kernel_package_pattern=".*(rhcos.*-kernel.x86_64).*"
+rhcos_initramfs_package_pattern=".*(rhcos.*initramfs.x86_64.img).*"
+rhcos_rootfs_package_pattern=".*(rhcos.*rootfs.x86_64.img).*"
+rhcos_ova_package_pattern=".*(rhcos.*.ova).*"
+
+#
+# Indicate whether to opt out of remote health checking of the OpenShift cluster
+#
+opt_out_health_checking=False
+
+#
+# Indicates whether the chrony NTP service must be enabled on the bastion node. If enabled,
+# all OpenShift nodes, bootstrap, masters and workers will synchronize their times with the
+# bastion node instead of a public NTP server. If you use the bastion node as the NTP server, 
+# you have to specify which range of servers will be allowed to synchronize
+# themselves with the bastion node; you can do this in the ntp_allow property.
+# Finally, specify a list of external servers with which the nodes will by synchronized, for example:
+# ntp_server=['0.rhel.pool.ntp.org', '1.rhel.pool.ntp.org'].
+#
+setup_chrony_server=False
+ntp_allow="10.11.80.0/24"
+ntp_servers=['10.11.80.187']
+override_chrony_settings_on_cluster_nodes=True
+
+#
+# A DNS server (dnsmasq) will be set up on the bastion node. Specify the upstream name server
+# that dnsmasq will use.
+#
+external_name_servers=['8.8.8.8']
+bastion_interface_script="/etc/sysconfig/network-scripts/ifcfg-eth0"
+
+#
+# Indicate whether DHCP and TFTP must run on the bastion node as part of dnsmasq. When using PXE boot, the masters 
+# and worker will get an fixed IP address from the dnsmasq DHCP server. Specify a range of addresses from which the 
+# DHCP server can issue an IP address. Every node configured in the bootstrap, masters and workers sections below will 
+# have a specific dhcp-server entry in the dnsmasq configuration, specifying the IP address they will be assigned by
+# the DHCP server. If the DHCP server runs on the bastion node and the RHCOS installation (rhcos_installation_method) is PXE,
+# it will also serve the RHCOS ISO using the TFTP server.
+#
+dhcp_on_bastion=True
+# to support multi subnet DHCP, the dhcp range need to be listed in a group which is listed on the bottom, like 
+# master/worker nodes.
+# dhcp_range=['10.11.80.51','10.11.80.60']
+
+#
+# Indicate whether the load balancer must be configured by the prepare script. If you choose to use an external load
+# balancer, set the manage_load_balancer property to False. You can still configure the load balancer under the [lb]
+# section.
+#
+manage_load_balancer=True
+
+#
+# Indicate the port that the HTTP server (nginx) on the bastion node will listen on. The HTTP server provides access
+# to the ignition files required at boot time of the bootstrap, masters and workers, as well as PXE boot assets such as
+# RHCOS ISO.
+#
+http_server_port=8090
+
+# Set up desktop on the bastion node
+setup_desktop=False
+
+# Set the bastion hostname as part of the preparation. If set to True, the bastion hostname will be set to
+# <bastion_host>.<cluster_name>.<domain_name>
+set_bastion_hostname=False
+
+#
+# If manage_nfs is True, the Ansible script will try to format and mount the NFS volume configured in the nfs_volume*
+# properties on the server indicated in the [nfs] section. If the value is False, the nfs server referred to in the [nfs]
+# section will still be used to configure the managed-nfs-storage storage class if specified. However in that case, it is assumed
+# that an external NFS server is available which is not configured by the Ansible playbooks.
+#
+# Volume parameters: these parameters indicate which volume the preparation scripts have to
+# format and mount. The "selector" parameter is used to find the device using the lsblk
+# command; you can specify the size (such as 500G) or any unique string that identifies
+# the device (such as sdb).
+#
+# The "nfs_volume_mount_path" indicates the mount point that is created for the device. Even if you are not managing NFS
+# yourself, but are using an external NFS server for NFS storge class, you should configure the path that must be mounted.
+#
+manage_nfs=True
+nfs_volume_selector="sdb"
+
+nfs_volume_mount_path="/nfs"
+
+#
+# Storageclass parameters: indicate whether NFS and/or Portworx storage classes must be created
+#
+
+# Create managed-nfs-storage storage class?
+create_nfs_sc=True
+
+# Install Portworx and create storage classes?
+create_portworx_sc=False
+
+#
+# Registry storage class and size to be used when image registry is configured. If NFS is used for
+# the image registry, the storage class is typically managed-nfs-storage. For OCS, it would be ocs-storagecluster-cephfs.
+#
+image_registry_storage_class=managed-nfs-storage
+image_registry_size=200Gi
+
+# This variable configures the subnet in which services will be created in the OpenShift Container Platform SDN.
+# Specify a private block that does not conflict with any existing network blocks in your infrastructure to which pods,
+# nodes, or the master might require access to, or the installation will fail. Defaults to 172.30.0.0/16,
+# and cannot be re-configured after deployment. If changing from the default, avoid 172.17.0.0/16,
+# which the docker0 network bridge uses by default, or modify the docker0 network.
+#
+service_network="172.30.0.0/16"
+
+#
+# Additional variables for IPI (Installer-provisioned Infrastructure) installations. 
+#
+apiVIP="10.11.48.51"
+ingressVIP="10.11.48.52"
+vm_number_of_masters=3
+vm_number_of_workers=3
+
+#
+# VMware envrionment specific properties. These are only used if you have selected the IPI installation method, or if
+# you want the prepare script to create the VMs or when using the vm_create.sh script to create these. 
+# You can ignore these properties if you manually create the VMs.
+#
+vc_vcenter="10.99.92.13"
+vc_datacenter="Datacenter1"
+vc_datastore="Datastore1"
+vc_cluster="Cluster1"
+vc_res_pool="resourcepool"
+vc_folder="fkocp48"
+vc_guest_id="rhel7_64Guest"
+vc_network="VM Network"
+vm_template="/Datacenter1/vm/Templates/RHCOS_4.8.8"
+
+# Bootstrap VM properties
+vm_bootstrap_mem=16384
+vm_bootstrap_cpu=4
+vm_bootstrap_disk=100
+
+# Master VM properties
+vm_master_mem=32768
+vm_master_cpu=8
+vm_master_disk=200
+
+# Worker VM properties
+vm_worker_mem=65536
+vm_worker_cpu=16
+vm_worker_disk=200
+
+# dhcp multi subnet list
+# format: 
+# subnet_tag address_begin address_end gateway
+
+[dhcp]
+sub1 begin="10.11.80.51" end="10.11.80.60" gateway="10.11.80.1"
+sub2 begin="10.11.60.51" end="10.11.60.60" gateway="10.11.60.1"
+
+[lb]
+10.11.80.187 host="bastion"
+
+[nfs]
+10.11.80.187 host="bastion"
+
+[bastion]
+10.11.80.187 host="bastion"
+
+[bootstrap]
+10.11.80.51 host="bootstrap" mac="00:50:56:ab:33:90" gateway="10.11.80.1"
+
+[masters]
+10.11.80.52 host="master-1" mac="00:50:56:ab:33:91" gateway="10.11.80.1"
+10.11.80.53 host="master-2" mac="00:50:56:ab:33:92" gateway="10.11.80.1"
+10.11.80.54 host="master-3" mac="00:50:56:ab:33:93" gateway="10.11.80.1"
+
+[workers]
+10.11.80.55 host="worker-1" mac="00:50:56:ab:33:94" gateway="10.11.80.1"
+10.11.80.56 host="worker-2" mac="00:50:56:ab:33:95" gateway="10.11.80.1"
+10.11.80.57 host="worker-3" mac="00:50:56:ab:33:96" gateway="10.11.80.1"
+
diff --git a/playbooks/tasks/dns_interface.yaml b/playbooks/tasks/dns_interface.yaml
index a3b2af3..bed3c20 100644
--- a/playbooks/tasks/dns_interface.yaml
+++ b/playbooks/tasks/dns_interface.yaml
@@ -2,23 +2,23 @@
 
 - name: Check if interface script exists
   stat:
-    path: "{{interface_script}}"
+    path: "{{bastion_interface_script}}"
   register: stat_result
 
 - fail:
-    msg: "the network interface file {{interface_script}} not exist. Without it, the DNS config would not be correct."
+    msg: "the network interface file {{bastion_interface_script}} not exist. Without it, the DNS config would not be correct."
   when: not stat_result.stat.exists
 
 - name: Change DNS service in interface script
   lineinfile:
-    path: "{{interface_script}}"
+    path: "{{bastion_interface_script}}"
     regexp: '^DNS1'
     line: "DNS1={{groups['bastion'][0]}}"
   when: stat_result.stat.exists==True
   register: interface_changed
 
 - name: Check if interface boots from DHCP
-  shell: cat "{{interface_script}}"
+  shell: cat "{{bastion_interface_script}}"
   when: stat_result.stat.exists
   register: interface_boot_from_dhcp_stat_result
 
diff --git a/playbooks/tasks/dns_server.j2 b/playbooks/tasks/dns_server.j2
index 7c2af23..2eca072 100644
--- a/playbooks/tasks/dns_server.j2
+++ b/playbooks/tasks/dns_server.j2
@@ -21,8 +21,11 @@ srv-host=_etcd-server-ssl._tcp.{{cluster_name}}.{{domain_name}},etcd-{{loop.inde
 
 {% if dhcp_on_bastion %}
 domain={{cluster_name}}.{{domain_name}}
-dhcp-range={{dhcp_range[0]}},{{dhcp_range[1]}},infinite
-dhcp-option=3,{{default_gateway}}
+
+{% for tag in groups['dhcp'] | sort %}
+dhcp-range={{tag}},{{hostvars[tag]['begin']}},{{hostvars[tag]['end']}},infinite
+dhcp-option={{tag}},3,{{hostvars[tag]['gateway']}}
+{% endfor %}
 
 {% if rhcos_installation_method|upper!="IPI" %}
 {% for host in groups['masters'] | sort %}
diff --git a/playbooks/tasks/tftpboot_vm_menu.j2 b/playbooks/tasks/tftpboot_vm_menu.j2
index fa6dd90..0b1cef1 100644
--- a/playbooks/tasks/tftpboot_vm_menu.j2
+++ b/playbooks/tasks/tftpboot_vm_menu.j2
@@ -6,4 +6,4 @@ label linux
   menu label ^Install RHEL CoreOS
   menu default
   kernel /images/{{rhcos_kernel_package | basename}}
-  append initrd=/images/{{rhcos_initramfs_package | basename}} coreos.live.rootfs_url=http://{{groups['bastion'][0]}}:{{http_server_port}}/{{rhcos_rootfs_package}} rd.neednet=1 coreos.inst.install_dev=/dev/sda coreos.inst.ignition_url=http://{{groups['bastion'][0]}}:{{http_server_port}}/{{ocp_install_dir}}/{{node_type}}.ign ip={{item}}::{{default_gateway}}:255.255.255.0:{{hostvars[item]['host']}}.{{cluster_name}}.{{domain_name}}:{{interface_name}}:none nameserver={{groups['bastion'][0]}} 
+  append initrd=/images/{{rhcos_initramfs_package | basename}} coreos.live.rootfs_url=http://{{groups['bastion'][0]}}:{{http_server_port}}/{{rhcos_rootfs_package}} rd.neednet=1 coreos.inst.install_dev=/dev/sda coreos.inst.ignition_url=http://{{groups['bastion'][0]}}:{{http_server_port}}/{{ocp_install_dir}}/{{node_type}}.ign ip={{item}}::{{hostvars[item]['gateway']}}:255.255.255.0:{{hostvars[item]['host']}}.{{cluster_name}}.{{domain_name}}:{{node_interface_name}}:none nameserver={{groups['bastion'][0]}} 

From 03c4199d0709e00d8571408bcd5a7d129a483290 Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Fri, 28 Oct 2022 00:46:58 -0700
Subject: [PATCH 06/26] add init script for bastion machine.

---
 common.sh       | 168 ++++++++++++++++++++++++++++++++++++++++++++++++
 init_bastion.sh |  47 ++++++++++++++
 2 files changed, 215 insertions(+)
 create mode 100755 common.sh
 create mode 100755 init_bastion.sh

diff --git a/common.sh b/common.sh
new file mode 100755
index 0000000..4cfa877
--- /dev/null
+++ b/common.sh
@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+guard_bash_error () {
+    set -Eeuo pipefail
+}
+
+# Log levels
+INFO=0
+WARN=1
+ERROR=2
+FATAL=3
+DEBUG=4
+DEFAULT_LOG_LEVEL=${ERROR}
+
+my_exit () {
+    echo "EXIT: - [HOST:$(hostname)]: - $(date +"%Y-%m-%d %H:%M:%S") - $1"
+    exit "$2"
+}
+
+msg () {
+    if [ "$1" -le ${DEFAULT_LOG_LEVEL} ]; then
+        echo "[HOST:$(hostname)]: - $(date +"%Y-%m-%d %H:%M:%S") - $2"
+    fi
+}
+
+info () {
+    msg ${INFO} "INFO: - $1"
+}
+
+warn () {
+    msg ${WARN} "WARNING: - $1"
+}
+
+error () {
+    msg ${ERROR} "ERROR: - $1"
+}
+
+fatal () {
+    msg ${FATAL} "FATAL: - $1"
+}
+
+debug () {
+    msg ${DEBUG} "DEBUG: - $1"
+}
+
+begin_banner () {
+    info "$1 - $2 phase - begin"
+}
+
+done_banner () {
+    info "$1 - $2 phase - done"
+}
+
+### turn path within script into absolute path
+### must pass the calling string of the script as the first parameter
+### e.g., ./path_to_script/script.sh
+### or, /root/path_to_script/script.sh
+### return the absolute path to the script with "echo" command
+turn_to_absolute_path () {
+    local SCRIPT_ABS_PATH_RAW
+    SCRIPT_ABS_PATH_RAW="$(dirname "$1")"
+    # turn SCRIPT_ABS_PATH into absolute path
+    case ${SCRIPT_ABS_PATH_RAW} in
+        /*) echo "${SCRIPT_ABS_PATH_RAW}" ;;
+        \.\.*) echo "$PWD/${SCRIPT_ABS_PATH_RAW}" ;;
+        \.*) echo "$PWD/${SCRIPT_ABS_PATH_RAW}" ;;
+        *) echo "$PWD" ;;
+    esac
+}
+
+### change CD to up to the project root directory
+### must pass the absolute path to the script as the first parameter
+change_CD_to_project_root () {
+    cd "$1"
+    local up_level=..
+    local my_loop=10 # guard not to loop forever
+    until [ -f "${up_level}/cook.sh" ] && [ ${my_loop} -gt 0 ]
+    do
+        up_level=${up_level}/..
+        my_loop=$((my_loop - 1))
+    done
+    if [ ${my_loop} -eq 0 ]; then
+        my_exit "Too many level up within the searching for DevOps directory,abort." 1
+    fi
+    cd "$1/${up_level}"
+}
+
+### check OS and distribution
+### return the OS distribution and ID with "echo" command
+check_dist_or_OS () {
+    local MY_THE_DISTRIBUTION_ID=""
+    local MY_THE_DISTRIBUTION_VERSION=""
+    if [ -e /etc/os-release ]; then
+        MY_THE_DISTRIBUTION_ID=$(grep -w "ID" /etc/os-release |awk -F"=" '{print $NF}'|sed 's/"//g')
+        if [ "${MY_THE_DISTRIBUTION_ID}" == "ubuntu" ]; then
+            MY_THE_DISTRIBUTION_VERSION=$(grep -w "VERSION_ID" /etc/os-release |awk -F"=" '{print $NF}'|sed 's/"//g')
+        else
+            MY_THE_DISTRIBUTION_VERSION=$(grep -w "VERSION_ID" /etc/os-release |awk -F"=" '{print $NF}'|awk -F"." '{print $1}'|sed 's/"//g')
+        fi
+        echo "${MY_THE_DISTRIBUTION_ID} ${MY_THE_DISTRIBUTION_VERSION}"
+    else
+        if type uname > /dev/null 2>&1; then
+             MY_THE_DISTRIBUTION_ID=$(uname -s)
+             MY_THE_DISTRIBUTION_VERSION=$(uname -r)
+             echo "${MY_THE_DISTRIBUTION_ID} ${MY_THE_DISTRIBUTION_VERSION}"
+        else
+             echo ""
+        fi
+    fi
+}
+
+### guard that the caller of the script must be root or has sudo right
+guard_root_or_sudo () {
+    if [[ $EUID -gt 0 ]] && ! sudo echo  >/dev/null 2>&1; then
+        return 1
+    else
+        return 0
+    fi
+}
+
+### init script with check if root or sudo
+init_with_root_or_sudo () {
+    guard_bash_error
+
+    if ! guard_root_or_sudo; then
+        my_exit "You must be root or you must be sudoer to prepare the env for CI/CD." 1
+    fi
+
+    SCRIPT_ABS_PATH=$(turn_to_absolute_path "$0")
+
+    # change_CD_to_project_root ${SCRIPT_ABS_PATH}
+
+    THE_DISTRIBUTION_ID_VERSION=$(check_dist_or_OS)
+    THE_DISTRIBUTION_ID=$(echo "${THE_DISTRIBUTION_ID_VERSION}"|awk '{print $1}')
+    THE_DISTRIBUTION_VERSION=$(echo "${THE_DISTRIBUTION_ID_VERSION}"|awk '{print $2}')
+}
+
+### init script without check if root or sudo
+init_without_root_or_sudo () {
+    guard_bash_error
+
+    SCRIPT_ABS_PATH=$(turn_to_absolute_path "$0")
+
+    # change_CD_to_project_root ${SCRIPT_ABS_PATH}
+
+    THE_DISTRIBUTION_ID_VERSION=$(check_dist_or_OS)
+    THE_DISTRIBUTION_ID=$(echo "${THE_DISTRIBUTION_ID_VERSION}"|awk '{print $1}')
+    THE_DISTRIBUTION_VERSION=$(echo "${THE_DISTRIBUTION_ID_VERSION}"|awk '{print $2}')
+}
+
+get_last_stable_nix_channel () {
+    local MY_CHANNEL_NAME_REGEX=""
+    case ${THE_DISTRIBUTION_ID} in
+      debian|ubuntu|rhel|centos) MY_CHANNEL_NAME_REGEX='s/.*\(nixos-[0-9][0-9].[0-9][0-9]\).*/\1/p' ;;
+      Darwin) MY_CHANNEL_NAME_REGEX='s/.*\(nixpkgs-[0-9][0-9].[0-9][0-9]-darwin\).*/\1/p' ;;
+      *) ;;
+    esac
+    local MY_LAST_NIX_STABLE_CHANNEL
+    MY_LAST_NIX_STABLE_CHANNEL=$(git ls-remote --heads https://github.com/NixOS/nixpkgs | awk '{print $NF}' | awk -F"/" '{print $NF}' | grep -v "\-unstable" | grep -v "\-small" | sed -n "${MY_CHANNEL_NAME_REGEX}" | sort | tail -1)
+    echo "${MY_LAST_NIX_STABLE_CHANNEL}"
+}
+
+switch_to_last_stable_nix_channel () {
+    nix-channel --remove nixpkgs
+    nix-channel --add "https://nixos.org/channels/$(get_last_stable_nix_channel)" nixpkgs
+    nix-channel --update
+}
diff --git a/init_bastion.sh b/init_bastion.sh
new file mode 100755
index 0000000..3bfd601
--- /dev/null
+++ b/init_bastion.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+if ! type dirname > /dev/null 2>&1; then
+    echo "Not even a linux or macOS, Windoze? We don't support it. Abort."
+    exit 1
+fi
+
+. "$(dirname "$0")"/common.sh
+
+init_with_root_or_sudo "$0"
+
+begin_banner "Top level" "project env prepare"
+
+    case ${THE_DISTRIBUTION_ID} in
+      debian)
+        [[ -e /proc/sys/kernel/unprivileged_userns_clone ]] && sudo sysctl kernel.unprivileged_userns_clone=1
+        curl -L https://nixos.org/nix/install | sh
+	     ;;
+      ubuntu)
+        [[ -e /proc/sys/kernel/unprivileged_userns_clone ]] && sudo sysctl kernel.unprivileged_userns_clone=1
+        curl -L https://nixos.org/nix/install | sh
+	     ;;
+      Darwin)
+        curl -L https://nixos.org/nix/install | sh
+	     ;;
+      rhel|centos)
+        if [ "X$THE_DISTRIBUTION_VERSION" != "X8" ]; then
+          my_exit "only support centos/RHEL 8.x" 126
+        fi
+
+        systemctl stop firewalld;systemctl disable firewalld
+        yum -y update
+        yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
+        yum install -y ansible bind-utils buildah chrony dnsmasq git \
+                       haproxy httpd-tools jq libvirt net-tools nfs-utils nginx podman \
+                       python3 python3-netaddr python3-passlib python3-pip python3-policycoreutils python3-pyvmomi python3-requests \
+                       screen sos syslinux-tftpboot wget yum-utils
+
+        LATEST_PIP=$(ls -a /usr/bin/pip*|sort|tail -1)
+        "$LATEST_PIP" install passlib
+
+	     ;;
+      *) ;;
+    esac
+
+done_banner "Top level" "project env prepare"
+

From 45848451f1f77fb868f82c5c5b80119716214d04 Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Fri, 28 Oct 2022 02:02:43 -0700
Subject: [PATCH 07/26] fix init_bastion on other platforms.

---
 init_bastion.sh | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/init_bastion.sh b/init_bastion.sh
index 3bfd601..de32c63 100755
--- a/init_bastion.sh
+++ b/init_bastion.sh
@@ -13,15 +13,13 @@ begin_banner "Top level" "project env prepare"
 
     case ${THE_DISTRIBUTION_ID} in
       debian)
-        [[ -e /proc/sys/kernel/unprivileged_userns_clone ]] && sudo sysctl kernel.unprivileged_userns_clone=1
-        curl -L https://nixos.org/nix/install | sh
+        my_exit "debian not supported yet." 222
 	     ;;
       ubuntu)
-        [[ -e /proc/sys/kernel/unprivileged_userns_clone ]] && sudo sysctl kernel.unprivileged_userns_clone=1
-        curl -L https://nixos.org/nix/install | sh
+        my_exit "ubuntu not supported yet." 222
 	     ;;
       Darwin)
-        curl -L https://nixos.org/nix/install | sh
+        my_exit "macOS not supported yet." 222
 	     ;;
       rhel|centos)
         if [ "X$THE_DISTRIBUTION_VERSION" != "X8" ]; then

From 3a4b933c5fbcc0d612060a4b0b93bf7b614330e2 Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Fri, 4 Nov 2022 07:59:37 -0700
Subject: [PATCH 08/26] add set tag for dhcp multi sub net

---
 inventory/vmware-example-48-with-new-options.inv | 14 +++++++-------
 playbooks/tasks/dns_server.j2                    | 10 +++++-----
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/inventory/vmware-example-48-with-new-options.inv b/inventory/vmware-example-48-with-new-options.inv
index 4c2249e..afebd1d 100644
--- a/inventory/vmware-example-48-with-new-options.inv
+++ b/inventory/vmware-example-48-with-new-options.inv
@@ -224,15 +224,15 @@ sub2 begin="10.11.60.51" end="10.11.60.60" gateway="10.11.60.1"
 10.11.80.187 host="bastion"
 
 [bootstrap]
-10.11.80.51 host="bootstrap" mac="00:50:56:ab:33:90" gateway="10.11.80.1"
+10.11.80.51 host="bootstrap" mac="00:50:56:ab:33:90" gateway="10.11.80.1" dhcp_sub_net="sub1"
 
 [masters]
-10.11.80.52 host="master-1" mac="00:50:56:ab:33:91" gateway="10.11.80.1"
-10.11.80.53 host="master-2" mac="00:50:56:ab:33:92" gateway="10.11.80.1"
-10.11.80.54 host="master-3" mac="00:50:56:ab:33:93" gateway="10.11.80.1"
+10.11.80.52 host="master-1" mac="00:50:56:ab:33:91" gateway="10.11.80.1" dhcp_sub_net="sub1"
+10.11.80.53 host="master-2" mac="00:50:56:ab:33:92" gateway="10.11.80.1" dhcp_sub_net="sub1"
+10.11.80.54 host="master-3" mac="00:50:56:ab:33:93" gateway="10.11.80.1" dhcp_sub_net="sub1"
 
 [workers]
-10.11.80.55 host="worker-1" mac="00:50:56:ab:33:94" gateway="10.11.80.1"
-10.11.80.56 host="worker-2" mac="00:50:56:ab:33:95" gateway="10.11.80.1"
-10.11.80.57 host="worker-3" mac="00:50:56:ab:33:96" gateway="10.11.80.1"
+10.11.80.55 host="worker-1" mac="00:50:56:ab:33:94" gateway="10.11.80.1" dhcp_sub_net="sub1"
+10.11.80.56 host="worker-2" mac="00:50:56:ab:33:95" gateway="10.11.80.1" dhcp_sub_net="sub1"
+10.11.80.57 host="worker-3" mac="00:50:56:ab:33:96" gateway="10.11.80.1" dhcp_sub_net="sub1"
 
diff --git a/playbooks/tasks/dns_server.j2 b/playbooks/tasks/dns_server.j2
index 2eca072..bc6009c 100644
--- a/playbooks/tasks/dns_server.j2
+++ b/playbooks/tasks/dns_server.j2
@@ -23,21 +23,21 @@ srv-host=_etcd-server-ssl._tcp.{{cluster_name}}.{{domain_name}},etcd-{{loop.inde
 domain={{cluster_name}}.{{domain_name}}
 
 {% for tag in groups['dhcp'] | sort %}
-dhcp-range={{tag}},{{hostvars[tag]['begin']}},{{hostvars[tag]['end']}},infinite
-dhcp-option={{tag}},3,{{hostvars[tag]['gateway']}}
+dhcp-range=set:{{tag}},{{hostvars[tag]['begin']}},{{hostvars[tag]['end']}},infinite
+dhcp-option=tag:{{tag}},3,{{hostvars[tag]['gateway']}}
 {% endfor %}
 
 {% if rhcos_installation_method|upper!="IPI" %}
 {% for host in groups['masters'] | sort %}
-dhcp-host={{hostvars[host]['mac']}},{{hostvars[host]['host']}},{{host}}
+dhcp-host={{hostvars[host]['mac']}},{{hostvars[host]['host']}},{{host}},set:{{hostvars[host]['dhcp_sub_net']}}
 {% endfor %}
 
 {% for host in groups['workers'] | sort %}
-dhcp-host={{hostvars[host]['mac']}},{{hostvars[host]['host']}},{{host}}
+dhcp-host={{hostvars[host]['mac']}},{{hostvars[host]['host']}},{{host}},set:{{hostvars[host]['dhcp_sub_net']}}
 {% endfor %}
 
 {% for host in groups['bootstrap'] | sort %}
-dhcp-host={{hostvars[host]['mac']}},{{hostvars[host]['host']}},{{host}}
+dhcp-host={{hostvars[host]['mac']}},{{hostvars[host]['host']}},{{host}},set:{{hostvars[host]['dhcp_sub_net']}}
 {% endfor %}
 {% endif %}
 

From 1a53e2bc2e5cca95466c2c7f8752a03dde52f5d9 Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Sat, 5 Nov 2022 00:24:46 -0700
Subject: [PATCH 09/26] add mirror registry.

---
 mirror_ocp.sh  | 161 +++++++++++++++++++++++++++++++++++++++++++++++++
 nginx.conf.tpl |  97 +++++++++++++++++++++++++++++
 2 files changed, 258 insertions(+)
 create mode 100755 mirror_ocp.sh
 create mode 100644 nginx.conf.tpl

diff --git a/mirror_ocp.sh b/mirror_ocp.sh
new file mode 100755
index 0000000..79fef55
--- /dev/null
+++ b/mirror_ocp.sh
@@ -0,0 +1,161 @@
+#!/usr/bin/env bash
+
+if ! type dirname > /dev/null 2>&1; then
+    echo "Not even a linux or macOS, Windoze? We don't support it. Abort."
+    exit 1
+fi
+
+. "$(dirname "$0")"/common.sh
+
+init_with_root_or_sudo "$0"
+
+begin_banner "Top level" "project env prepare"
+
+    case ${THE_DISTRIBUTION_ID} in
+      debian)
+        my_exit "debian not supported yet." 222
+	     ;;
+      ubuntu)
+        my_exit "ubuntu not supported yet." 222
+	     ;;
+      Darwin)
+        my_exit "macOS not supported yet." 222
+	     ;;
+      rhel|centos)
+        if [ "X$THE_DISTRIBUTION_VERSION" != "X8" ]; then
+          my_exit "only support centos/RHEL 8.x" 126
+        fi
+
+	# the pull secret file is needed to go forward
+	#if [ -z $pull_secret_file ];then
+  	#	pull_secret_file="/tmp/ocp_pullsecret.json"
+	#fi
+
+	if [ ! -e "${pull_secret_file-/tmp/ocp_pullsecret.json}" ];then
+  		echo "Pull secret file $pull_secret_file does not exist, please create the file or set the pull_secret_file environment variable to point to the file that holds the pull secret."
+  		exit 1
+	fi
+
+	# export environment variables for reusing
+	export REGISTRY_DOMAIN=chenjfocp.ibm.com
+	export REGISTRY_SERVER="registry.${REGISTRY_DOMAIN}"
+	export REGISTRY_PORT=5000
+	export LOCAL_REGISTRY="${REGISTRY_SERVER}:${REGISTRY_PORT}"
+	export EMAIL="admin@${REGISTRY_DOMAIN}"
+	export REGISTRY_USER="admin"
+	export REGISTRY_PASSWORD="passw0rd"
+
+	export OCP_RELEASE="4.8.52"
+	export RHCOS_RELEASE="4.8.14"
+	export LOCAL_REPOSITORY='ocp4/openshift4' 
+	export PRODUCT_REPO='openshift-release-dev' 
+	export MIRROR_DIR='/ocp4_downloads' 
+	export LOCAL_SECRET_JSON="${MIRROR_DIR}/ocp4_install/ocp_pullsecret.json"
+	export RELEASE_NAME="ocp-release"
+
+	export MIRROR_REGISTRY_HTTP_PORT=8090
+
+	# disable firewall first
+	systemctl stop firewalld;systemctl disable firewalld
+
+	# install the dependent tools
+	yum -y install wget podman httpd-tools jq nginx
+
+	# add host to /etc/hosts
+	PRIV_IP=$(hostname -I|awk '{print $1}')
+	grep "${PRIV_IP} ${REGISTRY_SERVER} registry" /etc/hosts > /dev/null || echo "${PRIV_IP} ${REGISTRY_SERVER} registry" >> /etc/hosts
+
+	# make mirror dir
+	mkdir -p ${MIRROR_DIR}/{clients,dependencies,ocp4_install}
+	mkdir -p ${MIRROR_DIR}/registry/{auth,certs,data,images}
+
+	# download oc client
+	wget -c https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-4.8/openshift-client-linux.tar.gz -O ${MIRROR_DIR}/clients/openshift-client-linux.tar.gz
+	wget -c https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-4.8/openshift-install-linux.tar.gz -O ${MIRROR_DIR}/clients/openshift-install-linux.tar.gz
+
+	# download coreos
+	# folowwing is for PXE boot
+	#wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-metal.x86_64.raw.gz -O ${MIRROR_DIR}/dependencies/rhcos-${RHCOS_RELEASE}-x86_64-metal.x86_64.raw.gz
+	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-live-kernel-x86_64 -O ${MIRROR_DIR}/dependencies/hcos-${RHCOS_RELEASE}-x86_64-live-kernel-x86_64
+	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-live-initramfs.x86_64.img -O ${MIRROR_DIR}/dependencies/hcos-${RHCOS_RELEASE}-x86_64-live-initramfs.x86_64.img
+	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-live-rootfs.x86_64.img -O ${MIRROR_DIR}/dependencies/hcos-${RHCOS_RELEASE}-x86_64-live-rootfs.x86_64.img
+	#wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-live.x86_64.iso -O ${MIRROR_DIR}/dependencies/hcos-${RHCOS_RELEASE}-x86_64-live.x86_64.iso
+
+	# following is for OVA install
+	#wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-vmware.x86_64.ova -O ${MIRROR_DIR}/dependencies/hcos-${RHCOS_RELEASE}-x86_64-vmware.x86_64.ova
+
+	# extract and install oc into /usr/local/bin
+	tar xvzf ${MIRROR_DIR}/clients/openshift-client-linux.tar.gz -C /usr/local/bin
+
+	# generate the self-signed certs for the mirrored registry
+	openssl req -newkey rsa:4096 -nodes -sha256 -keyout ${MIRROR_DIR}/registry/certs/registry.key -x509 -days 365 -out ${MIRROR_DIR}/registry/certs/registry.crt -subj "/C=US/ST=/L=/O=/CN=$REGISTRY_SERVER" -addext "subjectAltName = DNS:$REGISTRY_SERVER"
+
+	# generate the auth for the mirrored registry
+	htpasswd -bBc ${MIRROR_DIR}/registry/auth/htpasswd $REGISTRY_USER $REGISTRY_PASSWORD
+
+	# pull the registry docker image
+	podman pull docker.io/library/registry:2
+
+	# pull the nfs-provider image
+	podman pull gcr.io/k8s-staging-sig-storage/nfs-subdir-external-provisioner:v4.0.2
+
+	# run the registry container
+	podman run --name mirror-registry --publish $REGISTRY_PORT:5000 \
+     		--detach \
+     		--volume ${MIRROR_DIR}/registry/data:/var/lib/registry:z \
+     		--volume ${MIRROR_DIR}/registry/auth:/auth:z \
+     		--volume ${MIRROR_DIR}/registry/certs:/certs:z \
+     		--env "REGISTRY_AUTH=htpasswd" \
+     		--env "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \
+     		--env REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \
+     		--env REGISTRY_HTTP_TLS_CERTIFICATE=/certs/registry.crt \
+     		--env REGISTRY_HTTP_TLS_KEY=/certs/registry.key \
+     		docker.io/library/registry:2
+
+	# copy the generated self-signed certs of the mirrored registry to the trust store
+	/usr/bin/cp -f ${MIRROR_DIR}/registry/certs/registry.crt /etc/pki/ca-trust/source/anchors/
+	update-ca-trust
+
+	# list the mirrored registry catalog to verify it's working
+	curl -u $REGISTRY_USER:$REGISTRY_PASSWORD https://${LOCAL_REGISTRY}/v2/_catalog
+
+	# generate the pull secret for the mirroed registry
+	AUTH=$(echo -n "$REGISTRY_USER:$REGISTRY_PASSWORD" | base64 -w0)
+
+	CUST_REG='{"%s": {"auth":"%s", "email":"%s"}}\n'
+	printf "$CUST_REG" "$LOCAL_REGISTRY" "$AUTH" "$EMAIL" > /tmp/local_reg.json
+
+	jq --argjson authinfo "$(</tmp/local_reg.json)" '.auths += $authinfo' "${pull_secret_file-/tmp/ocp_pullsecret.json}" > ${MIRROR_DIR}/ocp4_install/ocp_pullsecret.json
+
+	# now really doing the mirror
+	oc adm -a ${LOCAL_SECRET_JSON} release mirror \
+     		--from=quay.io/${PRODUCT_REPO}/${RELEASE_NAME}:${OCP_RELEASE}-x86_64 \
+     		--to=${LOCAL_REGISTRY}/${LOCAL_REPOSITORY} \
+     		--to-release-image=${LOCAL_REGISTRY}/${LOCAL_REPOSITORY}:${OCP_RELEASE}
+
+
+	# list the mirrored registry catalog again to verify it's working
+	curl -u $REGISTRY_USER:$REGISTRY_PASSWORD https://${LOCAL_REGISTRY}/v2/_catalog
+
+	# generate nginx config based on template and variables
+	cp /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bak.by.mirror.registry
+	envsubst < ${SCRIPT_ABS_PATH}/nginx.conf.tpl > /etc/nginx/nginx.conf
+	ln -s ${MIRROR_DIR} /usr/share/nginx/html${MIRROR_DIR}
+	systemctl restart nginx;systemctl enable nginx
+
+	# check the http list entries
+	curl -L -s http://${REGISTRY_SERVER}:${MIRROR_REGISTRY_HTTP_PORT}${MIRROR_DIR} --list-only
+
+	# generate a systemd service the for mirror registry
+	#podman generate systemd mirror-registry -n > /etc/systemd/system/container-mirror-registry.service
+	#systemctl enable container-mirror-registry.service
+	#systemctl daemon-reload
+
+
+
+	     ;;
+      *) ;;
+    esac
+
+done_banner "Top level" "project env prepare"
+
diff --git a/nginx.conf.tpl b/nginx.conf.tpl
new file mode 100644
index 0000000..a559aec
--- /dev/null
+++ b/nginx.conf.tpl
@@ -0,0 +1,97 @@
+# Template nginx config to use to generate the config for OCP mirror
+# adapted from an original nginx stock config file
+
+# For more information on configuration, see:
+#   * Official English Documentation: http://nginx.org/en/docs/
+#   * Official Russian Documentation: http://nginx.org/ru/docs/
+
+user nginx;
+worker_processes auto;
+error_log /var/log/nginx/error.log;
+pid /run/nginx.pid;
+
+# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic.
+include /usr/share/nginx/modules/*.conf;
+
+events {
+    worker_connections 1024;
+}
+
+http {
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+
+    sendfile            on;
+    tcp_nopush          on;
+    tcp_nodelay         on;
+    keepalive_timeout   65;
+    types_hash_max_size 2048;
+
+    include             /etc/nginx/mime.types;
+    default_type        application/octet-stream;
+
+    # Load modular configuration files from the /etc/nginx/conf.d directory.
+    # See http://nginx.org/en/docs/ngx_core_module.html#include
+    # for more information.
+    include /etc/nginx/conf.d/*.conf;
+
+    server {
+        listen       ${MIRROR_REGISTRY_HTTP_PORT} default_server;
+        listen       [::]:${MIRROR_REGISTRY_HTTP_PORT} default_server;
+        server_name  _;
+        root         /usr/share/nginx/html;
+
+        # Load configuration files for the default server block.
+        include /etc/nginx/default.d/*.conf;
+
+        location / {
+        }
+
+        location ${MIRROR_DIR} {
+	    autoindex on;
+        }
+
+        error_page 404 /404.html;
+            location = /40x.html {
+        }
+
+        error_page 500 502 503 504 /50x.html;
+            location = /50x.html {
+        }
+    }
+
+# Settings for a TLS enabled server.
+#
+#    server {
+#        listen       443 ssl http2 default_server;
+#        listen       [::]:443 ssl http2 default_server;
+#        server_name  _;
+#        root         /usr/share/nginx/html;
+#
+#        ssl_certificate "/etc/pki/nginx/server.crt";
+#        ssl_certificate_key "/etc/pki/nginx/private/server.key";
+#        ssl_session_cache shared:SSL:1m;
+#        ssl_session_timeout  10m;
+#        ssl_ciphers PROFILE=SYSTEM;
+#        ssl_prefer_server_ciphers on;
+#
+#        # Load configuration files for the default server block.
+#        include /etc/nginx/default.d/*.conf;
+#
+#        location / {
+#        }
+#
+#        error_page 404 /404.html;
+#            location = /40x.html {
+#        }
+#
+#        error_page 500 502 503 504 /50x.html;
+#            location = /50x.html {
+#        }
+#    }
+
+}
+

From 5ca66b2604d942f1a503bb0cf3dfd24c42a843e6 Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Sat, 5 Nov 2022 01:18:02 -0700
Subject: [PATCH 10/26] fix wget rhcos output wrong file name problem.

---
 mirror_ocp.sh | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/mirror_ocp.sh b/mirror_ocp.sh
index 79fef55..f56dd2e 100755
--- a/mirror_ocp.sh
+++ b/mirror_ocp.sh
@@ -9,7 +9,7 @@ fi
 
 init_with_root_or_sudo "$0"
 
-begin_banner "Top level" "project env prepare"
+begin_banner "Top level" "create an OCP mirror registry"
 
     case ${THE_DISTRIBUTION_ID} in
       debian)
@@ -76,13 +76,13 @@ begin_banner "Top level" "project env prepare"
 	# download coreos
 	# folowwing is for PXE boot
 	#wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-metal.x86_64.raw.gz -O ${MIRROR_DIR}/dependencies/rhcos-${RHCOS_RELEASE}-x86_64-metal.x86_64.raw.gz
-	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-live-kernel-x86_64 -O ${MIRROR_DIR}/dependencies/hcos-${RHCOS_RELEASE}-x86_64-live-kernel-x86_64
-	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-live-initramfs.x86_64.img -O ${MIRROR_DIR}/dependencies/hcos-${RHCOS_RELEASE}-x86_64-live-initramfs.x86_64.img
-	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-live-rootfs.x86_64.img -O ${MIRROR_DIR}/dependencies/hcos-${RHCOS_RELEASE}-x86_64-live-rootfs.x86_64.img
-	#wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-live.x86_64.iso -O ${MIRROR_DIR}/dependencies/hcos-${RHCOS_RELEASE}-x86_64-live.x86_64.iso
+	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-live-kernel-x86_64 -O ${MIRROR_DIR}/dependencies/rhcos-${RHCOS_RELEASE}-x86_64-live-kernel-x86_64
+	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-live-initramfs.x86_64.img -O ${MIRROR_DIR}/dependencies/rhcos-${RHCOS_RELEASE}-x86_64-live-initramfs.x86_64.img
+	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-live-rootfs.x86_64.img -O ${MIRROR_DIR}/dependencies/rhcos-${RHCOS_RELEASE}-x86_64-live-rootfs.x86_64.img
+	#wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-live.x86_64.iso -O ${MIRROR_DIR}/dependencies/rhcos-${RHCOS_RELEASE}-x86_64-live.x86_64.iso
 
 	# following is for OVA install
-	#wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-vmware.x86_64.ova -O ${MIRROR_DIR}/dependencies/hcos-${RHCOS_RELEASE}-x86_64-vmware.x86_64.ova
+	#wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-vmware.x86_64.ova -O ${MIRROR_DIR}/dependencies/rhcos-${RHCOS_RELEASE}-x86_64-vmware.x86_64.ova
 
 	# extract and install oc into /usr/local/bin
 	tar xvzf ${MIRROR_DIR}/clients/openshift-client-linux.tar.gz -C /usr/local/bin
@@ -157,5 +157,5 @@ begin_banner "Top level" "project env prepare"
       *) ;;
     esac
 
-done_banner "Top level" "project env prepare"
+done_banner "Top level" "create an OCP mirror registry"
 

From be138b2c3fef1ab2deeb504bf6f5c892f44325cd Mon Sep 17 00:00:00 2001
From: hughjfchen <hugh.jf.chen@gmail.com>
Date: Mon, 7 Nov 2022 09:41:11 +0800
Subject: [PATCH 11/26] fix pull_secret_file unbound problem.

---
 mirror_ocp.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mirror_ocp.sh b/mirror_ocp.sh
index f56dd2e..331410c 100755
--- a/mirror_ocp.sh
+++ b/mirror_ocp.sh
@@ -32,7 +32,7 @@ begin_banner "Top level" "create an OCP mirror registry"
 	#fi
 
 	if [ ! -e "${pull_secret_file-/tmp/ocp_pullsecret.json}" ];then
-  		echo "Pull secret file $pull_secret_file does not exist, please create the file or set the pull_secret_file environment variable to point to the file that holds the pull secret."
+  		echo "Pull secret file ${pull_secret_file-/tmp/ocp_pullsecret.json} does not exist, please create the file or set the pull_secret_file environment variable to point to the file that holds the pull secret."
   		exit 1
 	fi
 

From c18728a06e7d065a2bea746f2d39661baaafbcc3 Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Fri, 2 Dec 2022 23:21:48 +0800
Subject: [PATCH 12/26] optimization

---
 mirror_ocp.sh | 109 ++++++++++++++++++++++++++++----------------------
 1 file changed, 61 insertions(+), 48 deletions(-)

diff --git a/mirror_ocp.sh b/mirror_ocp.sh
index 331410c..e840312 100755
--- a/mirror_ocp.sh
+++ b/mirror_ocp.sh
@@ -33,27 +33,29 @@ begin_banner "Top level" "create an OCP mirror registry"
 
 	if [ ! -e "${pull_secret_file-/tmp/ocp_pullsecret.json}" ];then
   		echo "Pull secret file ${pull_secret_file-/tmp/ocp_pullsecret.json} does not exist, please create the file or set the pull_secret_file environment variable to point to the file that holds the pull secret."
+        echo "You may also want to check the environment variables value within the script before really invoking the script."
   		exit 1
 	fi
 
 	# export environment variables for reusing
-	export REGISTRY_DOMAIN=chenjfocp.ibm.com
-	export REGISTRY_SERVER="registry.${REGISTRY_DOMAIN}"
-	export REGISTRY_PORT=5000
-	export LOCAL_REGISTRY="${REGISTRY_SERVER}:${REGISTRY_PORT}"
-	export EMAIL="admin@${REGISTRY_DOMAIN}"
-	export REGISTRY_USER="admin"
-	export REGISTRY_PASSWORD="passw0rd"
-
-	export OCP_RELEASE="4.8.52"
-	export RHCOS_RELEASE="4.8.14"
-	export LOCAL_REPOSITORY='ocp4/openshift4' 
-	export PRODUCT_REPO='openshift-release-dev' 
-	export MIRROR_DIR='/ocp4_downloads' 
-	export LOCAL_SECRET_JSON="${MIRROR_DIR}/ocp4_install/ocp_pullsecret.json"
-	export RELEASE_NAME="ocp-release"
-
-	export MIRROR_REGISTRY_HTTP_PORT=8090
+	export MY_REGISTRY_DOMAIN="${REGISTRY_DOMAIN-chenjfocp.ibm.com}"
+	export MY_REGISTRY_SERVER="${REGISTRY_SERVER-registry.$MY_REGISTRY_DOMAIN}"
+	export MY_REGISTRY_PORT=${REGISTRY_PORT-5000}
+	export MY_LOCAL_REGISTRY="${LOCAL_REGISTRY-$MY_REGISTRY_SERVER:$MY_REGISTRY_PORT}"
+	export MY_EMAIL="${EMAIL-admin@$MY_REGISTRY_DOMAIN}"
+	export MY_REGISTRY_USER="${REGISTRY_USER-admin}"
+	export MY_REGISTRY_PASSWORD="${REGISTRY_PASSWORD-passw0rd}"
+
+	export MY_OCP_RELEASE_MAIN_VERSION="${OCP_RELEASE_MAIN_VERSION-4.8}"
+	export MY_OCP_RELEASE="${OCP_RELEASE-$MY_OCP_RELEASE_MAIN_VERSION.52}"
+	export MY_RHCOS_RELEASE="${RHCOS_RELEASE-$MY_OCP_RELEASE_MAIN_VERSION.14}"
+	export MY_LOCAL_REPOSITORY="${LOCAL_REPOSITORY-ocp4/openshift4}"
+	export MY_PRODUCT_REPO="${PRODUCT_REPO-openshift-release-dev}"
+	export MY_MIRROR_DIR="${MIRROR_DIR-/ocp4_downloads}"
+	export MY_LOCAL_SECRET_JSON="${LOCAL_SECRET_JSON-$MY_MIRROR_DIR/ocp4_install/ocp_pullsecret.json}"
+	export MY_RELEASE_NAME="${RELEASE_NAME-ocp-release}"
+
+	export MY_MIRROR_REGISTRY_HTTP_PORT=${MIRROR_REGISTRY_HTTP_PORT-8090}
 
 	# disable firewall first
 	systemctl stop firewalld;systemctl disable firewalld
@@ -63,48 +65,53 @@ begin_banner "Top level" "create an OCP mirror registry"
 
 	# add host to /etc/hosts
 	PRIV_IP=$(hostname -I|awk '{print $1}')
-	grep "${PRIV_IP} ${REGISTRY_SERVER} registry" /etc/hosts > /dev/null || echo "${PRIV_IP} ${REGISTRY_SERVER} registry" >> /etc/hosts
+	grep "${PRIV_IP} ${MY_REGISTRY_SERVER} registry" /etc/hosts > /dev/null || echo "${PRIV_IP} ${MY_REGISTRY_SERVER} registry" >> /etc/hosts
 
 	# make mirror dir
-	mkdir -p ${MIRROR_DIR}/{clients,dependencies,ocp4_install}
-	mkdir -p ${MIRROR_DIR}/registry/{auth,certs,data,images}
+	mkdir -p ${MY_MIRROR_DIR}/{clients,dependencies,ocp4_install}
+	mkdir -p ${MY_MIRROR_DIR}/registry/{auth,certs,data,images}
 
 	# download oc client
-	wget -c https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-4.8/openshift-client-linux.tar.gz -O ${MIRROR_DIR}/clients/openshift-client-linux.tar.gz
-	wget -c https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-4.8/openshift-install-linux.tar.gz -O ${MIRROR_DIR}/clients/openshift-install-linux.tar.gz
+	wget -c https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-${MY_OCP_RELEASE_MAIN_VERSION}/openshift-client-linux.tar.gz -O ${MY_MIRROR_DIR}/clients/openshift-client-linux.tar.gz
+	wget -c https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-${MY_OCP_RELEASE_MAIN_VERSION}/openshift-install-linux.tar.gz -O ${MY_MIRROR_DIR}/clients/openshift-install-linux.tar.gz
 
 	# download coreos
 	# folowwing is for PXE boot
-	#wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-metal.x86_64.raw.gz -O ${MIRROR_DIR}/dependencies/rhcos-${RHCOS_RELEASE}-x86_64-metal.x86_64.raw.gz
-	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-live-kernel-x86_64 -O ${MIRROR_DIR}/dependencies/rhcos-${RHCOS_RELEASE}-x86_64-live-kernel-x86_64
-	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-live-initramfs.x86_64.img -O ${MIRROR_DIR}/dependencies/rhcos-${RHCOS_RELEASE}-x86_64-live-initramfs.x86_64.img
-	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-live-rootfs.x86_64.img -O ${MIRROR_DIR}/dependencies/rhcos-${RHCOS_RELEASE}-x86_64-live-rootfs.x86_64.img
-	#wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-live.x86_64.iso -O ${MIRROR_DIR}/dependencies/rhcos-${RHCOS_RELEASE}-x86_64-live.x86_64.iso
+	#wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-metal.x86_64.raw.gz -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-metal.x86_64.raw.gz
+	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-kernel-x86_64 -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-kernel-x86_64
+	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-initramfs.x86_64.img -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-initramfs.x86_64.img
+	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-rootfs.x86_64.img -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-rootfs.x86_64.img
+    # for liveCD installation
+	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live.x86_64.iso -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live.x86_64.iso
 
 	# following is for OVA install
-	#wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.8/latest/rhcos-${RHCOS_RELEASE}-x86_64-vmware.x86_64.ova -O ${MIRROR_DIR}/dependencies/rhcos-${RHCOS_RELEASE}-x86_64-vmware.x86_64.ova
+	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-vmware.x86_64.ova -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-vmware.x86_64.ova
 
 	# extract and install oc into /usr/local/bin
-	tar xvzf ${MIRROR_DIR}/clients/openshift-client-linux.tar.gz -C /usr/local/bin
+	tar xvzf ${MY_MIRROR_DIR}/clients/openshift-client-linux.tar.gz -C /usr/local/bin
 
 	# generate the self-signed certs for the mirrored registry
-	openssl req -newkey rsa:4096 -nodes -sha256 -keyout ${MIRROR_DIR}/registry/certs/registry.key -x509 -days 365 -out ${MIRROR_DIR}/registry/certs/registry.crt -subj "/C=US/ST=/L=/O=/CN=$REGISTRY_SERVER" -addext "subjectAltName = DNS:$REGISTRY_SERVER"
+	openssl req -newkey rsa:4096 -nodes -sha256 -keyout ${MY_MIRROR_DIR}/registry/certs/registry.key -x509 -days 3650 -out ${MY_MIRROR_DIR}/registry/certs/registry.crt -subj "/C=US/ST=/L=/O=/CN=$MY_REGISTRY_SERVER" -addext "subjectAltName = DNS:$MY_REGISTRY_SERVER"
 
 	# generate the auth for the mirrored registry
-	htpasswd -bBc ${MIRROR_DIR}/registry/auth/htpasswd $REGISTRY_USER $REGISTRY_PASSWORD
+	htpasswd -bBc ${MY_MIRROR_DIR}/registry/auth/htpasswd $MY_REGISTRY_USER $MY_REGISTRY_PASSWORD
 
 	# pull the registry docker image
 	podman pull docker.io/library/registry:2
 
 	# pull the nfs-provider image
-	podman pull gcr.io/k8s-staging-sig-storage/nfs-subdir-external-provisioner:v4.0.2
+	podman pull k8s.gcr.io/sig-storage/nfs-subdir-external-provisioner:v4.0.2
 
 	# run the registry container
-	podman run --name mirror-registry --publish $REGISTRY_PORT:5000 \
+    EXISTING_MIRROR_CONTAINER=$(podman ps --all --noheading | awk '$NF=="mirror-registry" {print $1}')
+    if [ "X${EXISTING_MIRROR_CONTAINER}" != "X" ]; then
+        podman rm -f ${EXISTING_MIRROR_CONTAINER}
+    fi
+	podman run --name mirror-registry --publish $MY_REGISTRY_PORT:5000 \
      		--detach \
-     		--volume ${MIRROR_DIR}/registry/data:/var/lib/registry:z \
-     		--volume ${MIRROR_DIR}/registry/auth:/auth:z \
-     		--volume ${MIRROR_DIR}/registry/certs:/certs:z \
+     		--volume ${MY_MIRROR_DIR}/registry/data:/var/lib/registry:z \
+     		--volume ${MY_MIRROR_DIR}/registry/auth:/auth:z \
+     		--volume ${MY_MIRROR_DIR}/registry/certs:/certs:z \
      		--env "REGISTRY_AUTH=htpasswd" \
      		--env "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \
      		--env REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \
@@ -113,38 +120,44 @@ begin_banner "Top level" "create an OCP mirror registry"
      		docker.io/library/registry:2
 
 	# copy the generated self-signed certs of the mirrored registry to the trust store
-	/usr/bin/cp -f ${MIRROR_DIR}/registry/certs/registry.crt /etc/pki/ca-trust/source/anchors/
+	/usr/bin/cp -f ${MY_MIRROR_DIR}/registry/certs/registry.crt /etc/pki/ca-trust/source/anchors/
 	update-ca-trust
 
 	# list the mirrored registry catalog to verify it's working
-	curl -u $REGISTRY_USER:$REGISTRY_PASSWORD https://${LOCAL_REGISTRY}/v2/_catalog
+	curl -u $MY_REGISTRY_USER:$MY_REGISTRY_PASSWORD https://${MY_LOCAL_REGISTRY}/v2/_catalog
 
 	# generate the pull secret for the mirroed registry
-	AUTH=$(echo -n "$REGISTRY_USER:$REGISTRY_PASSWORD" | base64 -w0)
+	AUTH=$(echo -n "$MY_REGISTRY_USER:$MY_REGISTRY_PASSWORD" | base64 -w0)
 
 	CUST_REG='{"%s": {"auth":"%s", "email":"%s"}}\n'
-	printf "$CUST_REG" "$LOCAL_REGISTRY" "$AUTH" "$EMAIL" > /tmp/local_reg.json
+	printf "$CUST_REG" "$MY_LOCAL_REGISTRY" "$AUTH" "$MY_EMAIL" > /tmp/local_reg.json
 
-	jq --argjson authinfo "$(</tmp/local_reg.json)" '.auths += $authinfo' "${pull_secret_file-/tmp/ocp_pullsecret.json}" > ${MIRROR_DIR}/ocp4_install/ocp_pullsecret.json
+	jq --argjson authinfo "$(</tmp/local_reg.json)" '.auths += $authinfo' "${pull_secret_file-/tmp/ocp_pullsecret.json}" > ${MY_MIRROR_DIR}/ocp4_install/ocp_pullsecret.json
 
 	# now really doing the mirror
-	oc adm -a ${LOCAL_SECRET_JSON} release mirror \
-     		--from=quay.io/${PRODUCT_REPO}/${RELEASE_NAME}:${OCP_RELEASE}-x86_64 \
-     		--to=${LOCAL_REGISTRY}/${LOCAL_REPOSITORY} \
-     		--to-release-image=${LOCAL_REGISTRY}/${LOCAL_REPOSITORY}:${OCP_RELEASE}
+	oc adm -a ${MY_LOCAL_SECRET_JSON} release mirror \
+     		--from=quay.io/${MY_PRODUCT_REPO}/${MY_RELEASE_NAME}:${MY_OCP_RELEASE}-x86_64 \
+     		--to=${MY_LOCAL_REGISTRY}/${MY_LOCAL_REPOSITORY} \
+     		--to-release-image=${MY_LOCAL_REGISTRY}/${MY_LOCAL_REPOSITORY}:${MY_OCP_RELEASE}
 
 
 	# list the mirrored registry catalog again to verify it's working
-	curl -u $REGISTRY_USER:$REGISTRY_PASSWORD https://${LOCAL_REGISTRY}/v2/_catalog
+	curl -u $MY_REGISTRY_USER:$MY_REGISTRY_PASSWORD https://${MY_LOCAL_REGISTRY}/v2/_catalog
 
 	# generate nginx config based on template and variables
 	cp /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bak.by.mirror.registry
 	envsubst < ${SCRIPT_ABS_PATH}/nginx.conf.tpl > /etc/nginx/nginx.conf
-	ln -s ${MIRROR_DIR} /usr/share/nginx/html${MIRROR_DIR}
+
+    # make the dir part of the MIRROR_DIR under the nginx default root dir first
+	# so that link succeed
+	MY_NGINX_DEFAULT_DOC_ROOT="/usr/share/nginx/html"
+	MY_MIRROR_DIR_DIR_PART=$(dirname $MY_MIRROR_DIR)
+	mkdir -p "$MY_NGINX_DEFAULT_DOC_ROOT/$MY_MIRROR_DIR_DIR_PART"
+	ln -s ${MY_MIRROR_DIR} "MY_NGINX_DEFAULT_DOC_ROOT/${MY_MIRROR_DIR}"
 	systemctl restart nginx;systemctl enable nginx
 
 	# check the http list entries
-	curl -L -s http://${REGISTRY_SERVER}:${MIRROR_REGISTRY_HTTP_PORT}${MIRROR_DIR} --list-only
+	curl -L -s http://${MY_REGISTRY_SERVER}:${MY_MIRROR_REGISTRY_HTTP_PORT}${MY_MIRROR_DIR} --list-only
 
 	# generate a systemd service the for mirror registry
 	#podman generate systemd mirror-registry -n > /etc/systemd/system/container-mirror-registry.service

From 80cdd84ef4ac5f293abae9c4cb5e54474fa6dd8f Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Sat, 3 Dec 2022 14:45:27 +0800
Subject: [PATCH 13/26] add task to generate coreos-installer install command

---
 playbooks/ocp4.yaml                           |  1 +
 playbooks/tasks/coreos_installer_command.j2   |  1 +
 playbooks/tasks/coreos_installer_command.yaml | 36 +++++++++++++++++++
 3 files changed, 38 insertions(+)
 create mode 100644 playbooks/tasks/coreos_installer_command.j2
 create mode 100644 playbooks/tasks/coreos_installer_command.yaml

diff --git a/playbooks/ocp4.yaml b/playbooks/ocp4.yaml
index f433a5a..4923352 100644
--- a/playbooks/ocp4.yaml
+++ b/playbooks/ocp4.yaml
@@ -78,6 +78,7 @@
         tags: tftp
       - include: tasks/pxe_links.yaml
       when: rhcos_installation_method|upper=="PXE"
+    - include: tasks/coreos_installer_command.yaml
     - include: tasks/dns_server.yaml
       tags: dns
     - include: tasks/dns_interface.yaml
diff --git a/playbooks/tasks/coreos_installer_command.j2 b/playbooks/tasks/coreos_installer_command.j2
new file mode 100644
index 0000000..a8dc95c
--- /dev/null
+++ b/playbooks/tasks/coreos_installer_command.j2
@@ -0,0 +1 @@
+sudo coreos-installer install /dev/sda --append-karg 'ip={{item}}::{{hostvars[item]['gateway']}}:255.255.255.0:{{hostvars[item]['host']}}.{{cluster_name}}.{{domain_name}}:{{node_interface_name}}:none nameserver={{groups['bastion'][0]}}' --ignition-url=http://{{groups['bastion'][0]}}:{{http_server_port}}/{{ocp_install_dir}}/{{node_type}}.ign --insecure-ignition
diff --git a/playbooks/tasks/coreos_installer_command.yaml b/playbooks/tasks/coreos_installer_command.yaml
new file mode 100644
index 0000000..13d55c1
--- /dev/null
+++ b/playbooks/tasks/coreos_installer_command.yaml
@@ -0,0 +1,36 @@
+---
+- name: Generate CoreOS installer command for bootstrap
+  template:
+    src: coreos_installer_command.j2
+    dest: "{{ocp_install_dir}}/coreos_installer_command.{{hostvars[item]['host']}}"
+    owner: root
+    group: root
+    mode: 0644
+  with_items:
+    - "{{groups['bootstrap']}}"
+  vars:
+    node_type: bootstrap
+
+- name: Generate CoreOS installer command for masters
+  template:
+    src: coreos_installer_command.j2
+    dest: "{{ocp_install_dir}}/coreos_installer_command.{{hostvars[item]['host']}}"
+    owner: root
+    group: root
+    mode: 0644
+  with_items:
+    - "{{groups['masters']}}"
+  vars:
+    node_type: master
+
+- name: Generate CoreOS installer command for workers
+  template:
+    src: coreos_installer_command.j2
+    dest: "{{ocp_install_dir}}/coreos_installer_command.{{hostvars[item]['host']}}"
+    owner: root
+    group: root
+    mode: 0644
+  with_items:
+    - "{{groups['workers']}}"
+  vars:
+    node_type: worker

From 97266e890d1e1e8a021557843173e1841b989d32 Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Sat, 3 Dec 2022 21:19:25 +0800
Subject: [PATCH 14/26] check if firewalld service exists before stop it.

---
 init_bastion.sh | 2 +-
 mirror_ocp.sh   | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/init_bastion.sh b/init_bastion.sh
index de32c63..7ec6028 100755
--- a/init_bastion.sh
+++ b/init_bastion.sh
@@ -26,7 +26,7 @@ begin_banner "Top level" "project env prepare"
           my_exit "only support centos/RHEL 8.x" 126
         fi
 
-        systemctl stop firewalld;systemctl disable firewalld
+        systemctl status firewalld > /dev/null 2>&1 && systemctl stop firewalld && systemctl disable firewalld
         yum -y update
         yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
         yum install -y ansible bind-utils buildah chrony dnsmasq git \
diff --git a/mirror_ocp.sh b/mirror_ocp.sh
index e840312..120eae7 100755
--- a/mirror_ocp.sh
+++ b/mirror_ocp.sh
@@ -58,7 +58,7 @@ begin_banner "Top level" "create an OCP mirror registry"
 	export MY_MIRROR_REGISTRY_HTTP_PORT=${MIRROR_REGISTRY_HTTP_PORT-8090}
 
 	# disable firewall first
-	systemctl stop firewalld;systemctl disable firewalld
+	systemctl status firewalld > /dev/null 2>&1 && systemctl stop firewalld && systemctl disable firewalld
 
 	# install the dependent tools
 	yum -y install wget podman httpd-tools jq nginx
@@ -81,11 +81,11 @@ begin_banner "Top level" "create an OCP mirror registry"
 	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-kernel-x86_64 -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-kernel-x86_64
 	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-initramfs.x86_64.img -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-initramfs.x86_64.img
 	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-rootfs.x86_64.img -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-rootfs.x86_64.img
-    # for liveCD installation
+        # for liveCD installation
 	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live.x86_64.iso -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live.x86_64.iso
 
 	# following is for OVA install
-	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-vmware.x86_64.ova -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-vmware.x86_64.ova
+	# wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-vmware.x86_64.ova -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-vmware.x86_64.ova
 
 	# extract and install oc into /usr/local/bin
 	tar xvzf ${MY_MIRROR_DIR}/clients/openshift-client-linux.tar.gz -C /usr/local/bin

From a369f2460080018d22dbed3c4c4c2b94ebda9116 Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Sun, 15 Jan 2023 20:19:00 +0800
Subject: [PATCH 15/26] integrate mirror_ocp into prepare.

---
 init_bastion.sh |   2 +-
 mirror_ocp.sh   | 269 ++++++++++++++++++++++++------------------------
 prepare.sh      |  10 ++
 3 files changed, 145 insertions(+), 136 deletions(-)

diff --git a/init_bastion.sh b/init_bastion.sh
index 7ec6028..fd14b89 100755
--- a/init_bastion.sh
+++ b/init_bastion.sh
@@ -34,7 +34,7 @@ begin_banner "Top level" "project env prepare"
                        python3 python3-netaddr python3-passlib python3-pip python3-policycoreutils python3-pyvmomi python3-requests \
                        screen sos syslinux-tftpboot wget yum-utils
 
-        LATEST_PIP=$(ls -a /usr/bin/pip*|sort|tail -1)
+        LATEST_PIP=$(find /usr/bin -name 'pip*'|sort|tail -1)
         "$LATEST_PIP" install passlib
 
 	     ;;
diff --git a/mirror_ocp.sh b/mirror_ocp.sh
index 120eae7..9c44b0f 100755
--- a/mirror_ocp.sh
+++ b/mirror_ocp.sh
@@ -14,159 +14,158 @@ begin_banner "Top level" "create an OCP mirror registry"
     case ${THE_DISTRIBUTION_ID} in
       debian)
         my_exit "debian not supported yet." 222
-	     ;;
+        ;;
       ubuntu)
         my_exit "ubuntu not supported yet." 222
-	     ;;
+        ;;
       Darwin)
         my_exit "macOS not supported yet." 222
-	     ;;
+        ;;
       rhel|centos)
         if [ "X$THE_DISTRIBUTION_VERSION" != "X8" ]; then
           my_exit "only support centos/RHEL 8.x" 126
         fi
 
-	# the pull secret file is needed to go forward
-	#if [ -z $pull_secret_file ];then
-  	#	pull_secret_file="/tmp/ocp_pullsecret.json"
-	#fi
-
-	if [ ! -e "${pull_secret_file-/tmp/ocp_pullsecret.json}" ];then
-  		echo "Pull secret file ${pull_secret_file-/tmp/ocp_pullsecret.json} does not exist, please create the file or set the pull_secret_file environment variable to point to the file that holds the pull secret."
-        echo "You may also want to check the environment variables value within the script before really invoking the script."
-  		exit 1
-	fi
-
-	# export environment variables for reusing
-	export MY_REGISTRY_DOMAIN="${REGISTRY_DOMAIN-chenjfocp.ibm.com}"
-	export MY_REGISTRY_SERVER="${REGISTRY_SERVER-registry.$MY_REGISTRY_DOMAIN}"
-	export MY_REGISTRY_PORT=${REGISTRY_PORT-5000}
-	export MY_LOCAL_REGISTRY="${LOCAL_REGISTRY-$MY_REGISTRY_SERVER:$MY_REGISTRY_PORT}"
-	export MY_EMAIL="${EMAIL-admin@$MY_REGISTRY_DOMAIN}"
-	export MY_REGISTRY_USER="${REGISTRY_USER-admin}"
-	export MY_REGISTRY_PASSWORD="${REGISTRY_PASSWORD-passw0rd}"
-
-	export MY_OCP_RELEASE_MAIN_VERSION="${OCP_RELEASE_MAIN_VERSION-4.8}"
-	export MY_OCP_RELEASE="${OCP_RELEASE-$MY_OCP_RELEASE_MAIN_VERSION.52}"
-	export MY_RHCOS_RELEASE="${RHCOS_RELEASE-$MY_OCP_RELEASE_MAIN_VERSION.14}"
-	export MY_LOCAL_REPOSITORY="${LOCAL_REPOSITORY-ocp4/openshift4}"
-	export MY_PRODUCT_REPO="${PRODUCT_REPO-openshift-release-dev}"
-	export MY_MIRROR_DIR="${MIRROR_DIR-/ocp4_downloads}"
-	export MY_LOCAL_SECRET_JSON="${LOCAL_SECRET_JSON-$MY_MIRROR_DIR/ocp4_install/ocp_pullsecret.json}"
-	export MY_RELEASE_NAME="${RELEASE_NAME-ocp-release}"
-
-	export MY_MIRROR_REGISTRY_HTTP_PORT=${MIRROR_REGISTRY_HTTP_PORT-8090}
-
-	# disable firewall first
-	systemctl status firewalld > /dev/null 2>&1 && systemctl stop firewalld && systemctl disable firewalld
-
-	# install the dependent tools
-	yum -y install wget podman httpd-tools jq nginx
-
-	# add host to /etc/hosts
-	PRIV_IP=$(hostname -I|awk '{print $1}')
-	grep "${PRIV_IP} ${MY_REGISTRY_SERVER} registry" /etc/hosts > /dev/null || echo "${PRIV_IP} ${MY_REGISTRY_SERVER} registry" >> /etc/hosts
-
-	# make mirror dir
-	mkdir -p ${MY_MIRROR_DIR}/{clients,dependencies,ocp4_install}
-	mkdir -p ${MY_MIRROR_DIR}/registry/{auth,certs,data,images}
-
-	# download oc client
-	wget -c https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-${MY_OCP_RELEASE_MAIN_VERSION}/openshift-client-linux.tar.gz -O ${MY_MIRROR_DIR}/clients/openshift-client-linux.tar.gz
-	wget -c https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-${MY_OCP_RELEASE_MAIN_VERSION}/openshift-install-linux.tar.gz -O ${MY_MIRROR_DIR}/clients/openshift-install-linux.tar.gz
-
-	# download coreos
-	# folowwing is for PXE boot
-	#wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-metal.x86_64.raw.gz -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-metal.x86_64.raw.gz
-	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-kernel-x86_64 -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-kernel-x86_64
-	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-initramfs.x86_64.img -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-initramfs.x86_64.img
-	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-rootfs.x86_64.img -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-rootfs.x86_64.img
-        # for liveCD installation
-	wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live.x86_64.iso -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live.x86_64.iso
-
-	# following is for OVA install
-	# wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-vmware.x86_64.ova -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-vmware.x86_64.ova
-
-	# extract and install oc into /usr/local/bin
-	tar xvzf ${MY_MIRROR_DIR}/clients/openshift-client-linux.tar.gz -C /usr/local/bin
-
-	# generate the self-signed certs for the mirrored registry
-	openssl req -newkey rsa:4096 -nodes -sha256 -keyout ${MY_MIRROR_DIR}/registry/certs/registry.key -x509 -days 3650 -out ${MY_MIRROR_DIR}/registry/certs/registry.crt -subj "/C=US/ST=/L=/O=/CN=$MY_REGISTRY_SERVER" -addext "subjectAltName = DNS:$MY_REGISTRY_SERVER"
-
-	# generate the auth for the mirrored registry
-	htpasswd -bBc ${MY_MIRROR_DIR}/registry/auth/htpasswd $MY_REGISTRY_USER $MY_REGISTRY_PASSWORD
-
-	# pull the registry docker image
-	podman pull docker.io/library/registry:2
-
-	# pull the nfs-provider image
-	podman pull k8s.gcr.io/sig-storage/nfs-subdir-external-provisioner:v4.0.2
-
-	# run the registry container
-    EXISTING_MIRROR_CONTAINER=$(podman ps --all --noheading | awk '$NF=="mirror-registry" {print $1}')
-    if [ "X${EXISTING_MIRROR_CONTAINER}" != "X" ]; then
-        podman rm -f ${EXISTING_MIRROR_CONTAINER}
-    fi
-	podman run --name mirror-registry --publish $MY_REGISTRY_PORT:5000 \
-     		--detach \
-     		--volume ${MY_MIRROR_DIR}/registry/data:/var/lib/registry:z \
-     		--volume ${MY_MIRROR_DIR}/registry/auth:/auth:z \
-     		--volume ${MY_MIRROR_DIR}/registry/certs:/certs:z \
-     		--env "REGISTRY_AUTH=htpasswd" \
-     		--env "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \
-     		--env REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \
-     		--env REGISTRY_HTTP_TLS_CERTIFICATE=/certs/registry.crt \
-     		--env REGISTRY_HTTP_TLS_KEY=/certs/registry.key \
-     		docker.io/library/registry:2
-
-	# copy the generated self-signed certs of the mirrored registry to the trust store
-	/usr/bin/cp -f ${MY_MIRROR_DIR}/registry/certs/registry.crt /etc/pki/ca-trust/source/anchors/
-	update-ca-trust
-
-	# list the mirrored registry catalog to verify it's working
-	curl -u $MY_REGISTRY_USER:$MY_REGISTRY_PASSWORD https://${MY_LOCAL_REGISTRY}/v2/_catalog
-
-	# generate the pull secret for the mirroed registry
-	AUTH=$(echo -n "$MY_REGISTRY_USER:$MY_REGISTRY_PASSWORD" | base64 -w0)
-
-	CUST_REG='{"%s": {"auth":"%s", "email":"%s"}}\n'
-	printf "$CUST_REG" "$MY_LOCAL_REGISTRY" "$AUTH" "$MY_EMAIL" > /tmp/local_reg.json
-
-	jq --argjson authinfo "$(</tmp/local_reg.json)" '.auths += $authinfo' "${pull_secret_file-/tmp/ocp_pullsecret.json}" > ${MY_MIRROR_DIR}/ocp4_install/ocp_pullsecret.json
-
-	# now really doing the mirror
-	oc adm -a ${MY_LOCAL_SECRET_JSON} release mirror \
-     		--from=quay.io/${MY_PRODUCT_REPO}/${MY_RELEASE_NAME}:${MY_OCP_RELEASE}-x86_64 \
-     		--to=${MY_LOCAL_REGISTRY}/${MY_LOCAL_REPOSITORY} \
-     		--to-release-image=${MY_LOCAL_REGISTRY}/${MY_LOCAL_REPOSITORY}:${MY_OCP_RELEASE}
+        # the pull secret file is needed to go forward
+        #if [ -z $pull_secret_file ];then
+        #   pull_secret_file="/tmp/ocp_pullsecret.json"
+        #fi
 
+        if [ ! -e "${pull_secret_file-/tmp/ocp_pullsecret.json}" ];then
+           echo "Pull secret file ${pull_secret_file-/tmp/ocp_pullsecret.json} does not exist, please create the file or set the pull_secret_file environment variable to point to the file that holds the pull secret."
+           echo "You may also want to check the environment variables value within the script before really invoking the script."
+           exit 1
+        fi
 
-	# list the mirrored registry catalog again to verify it's working
-	curl -u $MY_REGISTRY_USER:$MY_REGISTRY_PASSWORD https://${MY_LOCAL_REGISTRY}/v2/_catalog
+        # export environment variables for reusing
+        export MY_REGISTRY_DOMAIN="${REGISTRY_DOMAIN-$domain_name}"
+        export MY_REGISTRY_SERVER="${REGISTRY_SERVER-$air_gapped_registry_server}"
+        export MY_REGISTRY_PORT=${REGISTRY_PORT-5000}
+        export MY_LOCAL_REGISTRY="${LOCAL_REGISTRY-$MY_REGISTRY_SERVER:$MY_REGISTRY_PORT}"
+        export MY_EMAIL="${EMAIL-admin@$MY_REGISTRY_DOMAIN}"
+        export MY_REGISTRY_USER="${REGISTRY_USER-admin}"
+        export MY_REGISTRY_PASSWORD="${REGISTRY_PASSWORD-passw0rd}"
+
+        export MY_OCP_RELEASE_MAIN_VERSION="${OCP_RELEASE_MAIN_VERSION-$openshift_release}"
+        export MY_OCP_RELEASE="${OCP_RELEASE-$MY_OCP_RELEASE_MAIN_VERSION.52}"
+        export MY_RHCOS_RELEASE="${RHCOS_RELEASE-$MY_OCP_RELEASE_MAIN_VERSION.14}"
+        export MY_LOCAL_REPOSITORY="${LOCAL_REPOSITORY-ocp4/openshift4}"
+        export MY_PRODUCT_REPO="${PRODUCT_REPO-openshift-release-dev}"
+        export MY_MIRROR_DIR="${MIRROR_DIR-$air_gapped_download_dir}"
+        export MY_LOCAL_SECRET_JSON="${LOCAL_SECRET_JSON-$MY_MIRROR_DIR/ocp4_install/ocp_pullsecret.json}"
+        export MY_RELEASE_NAME="${RELEASE_NAME-ocp-release}"
+
+        export MY_MIRROR_REGISTRY_HTTP_PORT=${MIRROR_REGISTRY_HTTP_PORT-$http_server_port}
+
+        # disable firewall first
+        systemctl status firewalld > /dev/null 2>&1 && systemctl stop firewalld && systemctl disable firewalld
+
+        # install the dependent tools
+        yum -y install wget podman httpd-tools jq nginx
+
+        # add host to /etc/hosts
+        PRIV_IP=$(hostname -I|awk '{print $1}')
+        grep "${PRIV_IP} ${MY_REGISTRY_SERVER} registry" /etc/hosts > /dev/null || echo "${PRIV_IP} ${MY_REGISTRY_SERVER} registry" >> /etc/hosts
+
+        # make mirror dir
+        mkdir -p "${MY_MIRROR_DIR}/{clients,dependencies,ocp4_install}"
+        mkdir -p "${MY_MIRROR_DIR}/registry/{auth,certs,data,images}"
+
+        # download oc client
+        wget -c "https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-${MY_OCP_RELEASE_MAIN_VERSION}/openshift-client-linux.tar.gz" -O "${MY_MIRROR_DIR}/clients/openshift-client-linux.tar.gz"
+        wget -c "https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-${MY_OCP_RELEASE_MAIN_VERSION}/openshift-install-linux.tar.gz" -O "${MY_MIRROR_DIR}/clients/openshift-install-linux.tar.gz"
+
+        # download coreos
+        # folowwing is for PXE boot
+        #wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-metal.x86_64.raw.gz -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-metal.x86_64.raw.gz
+        wget -c "https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-kernel-x86_64" -O "${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-kernel-x86_64"
+        wget -c "https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-initramfs.x86_64.img" -O "${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-initramfs.x86_64.img"
+        wget -c "https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-rootfs.x86_64.img" -O "${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-rootfs.x86_64.img"
+        # for liveCD installation
+        wget -c "https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live.x86_64.iso" -O "${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live.x86_64.iso"
 
-	# generate nginx config based on template and variables
-	cp /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bak.by.mirror.registry
-	envsubst < ${SCRIPT_ABS_PATH}/nginx.conf.tpl > /etc/nginx/nginx.conf
+        # following is for OVA install
+        # wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-vmware.x86_64.ova -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-vmware.x86_64.ova
 
-    # make the dir part of the MIRROR_DIR under the nginx default root dir first
-	# so that link succeed
-	MY_NGINX_DEFAULT_DOC_ROOT="/usr/share/nginx/html"
-	MY_MIRROR_DIR_DIR_PART=$(dirname $MY_MIRROR_DIR)
-	mkdir -p "$MY_NGINX_DEFAULT_DOC_ROOT/$MY_MIRROR_DIR_DIR_PART"
-	ln -s ${MY_MIRROR_DIR} "MY_NGINX_DEFAULT_DOC_ROOT/${MY_MIRROR_DIR}"
-	systemctl restart nginx;systemctl enable nginx
+        # extract and install oc into /usr/local/bin
+        tar xvzf "${MY_MIRROR_DIR}/clients/openshift-client-linux.tar.gz" -C /usr/local/bin
 
-	# check the http list entries
-	curl -L -s http://${MY_REGISTRY_SERVER}:${MY_MIRROR_REGISTRY_HTTP_PORT}${MY_MIRROR_DIR} --list-only
+        # generate the self-signed certs for the mirrored registry
+        openssl req -newkey rsa:4096 -nodes -sha256 -keyout "${MY_MIRROR_DIR}/registry/certs/registry.key" -x509 -days 3650 -out "${MY_MIRROR_DIR}/registry/certs/registry.crt" -subj "/C=US/ST=/L=/O=/CN=$MY_REGISTRY_SERVER" -addext "subjectAltName = DNS:$MY_REGISTRY_SERVER"
 
-	# generate a systemd service the for mirror registry
-	#podman generate systemd mirror-registry -n > /etc/systemd/system/container-mirror-registry.service
-	#systemctl enable container-mirror-registry.service
-	#systemctl daemon-reload
+        # generate the auth for the mirrored registry
+        htpasswd -bBc "${MY_MIRROR_DIR}/registry/auth/htpasswd" "$MY_REGISTRY_USER" "$MY_REGISTRY_PASSWORD"
 
+        # pull the registry docker image
+        podman pull docker.io/library/registry:2
 
+        # pull the nfs-provider image
+        # THIS DOES NOT WORK WITHIN CHINA!!!
+        # podman pull k8s.gcr.io/sig-storage/nfs-subdir-external-provisioner:v4.0.2
 
-	     ;;
+        # run the registry container
+        EXISTING_MIRROR_CONTAINER=$(podman ps --all --noheading | awk '$NF=="mirror-registry" {print $1}')
+        if [ "X${EXISTING_MIRROR_CONTAINER}" != "X" ]; then
+            podman rm -f "${EXISTING_MIRROR_CONTAINER}"
+        fi
+        podman run --name mirror-registry --publish "$MY_REGISTRY_PORT:5000" \
+               --detach \
+               --volume "${MY_MIRROR_DIR}/registry/data":/var/lib/registry:z \
+               --volume "${MY_MIRROR_DIR}/registry/auth":/auth:z \
+               --volume "${MY_MIRROR_DIR}/registry/certs":/certs:z \
+               --env "REGISTRY_AUTH=htpasswd" \
+               --env "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \
+               --env REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \
+               --env REGISTRY_HTTP_TLS_CERTIFICATE=/certs/registry.crt \
+               --env REGISTRY_HTTP_TLS_KEY=/certs/registry.key \
+               docker.io/library/registry:2
+
+        # copy the generated self-signed certs of the mirrored registry to the trust store
+        /usr/bin/cp -f "${MY_MIRROR_DIR}/registry/certs/registry.crt" /etc/pki/ca-trust/source/anchors/
+        update-ca-trust
+
+        # list the mirrored registry catalog to verify it's working
+        curl -u "$MY_REGISTRY_USER:$MY_REGISTRY_PASSWORD" "https://${MY_LOCAL_REGISTRY}/v2/_catalog"
+
+        # generate the pull secret for the mirroed registry
+        AUTH=$(echo -n "$MY_REGISTRY_USER:$MY_REGISTRY_PASSWORD" | base64 -w0)
+
+        CUST_REG='{"%s": {"auth":"%s", "email":"%s"}}\n'
+        printf "$CUST_REG" "$MY_LOCAL_REGISTRY" "$AUTH" "$MY_EMAIL" > /tmp/local_reg.json
+
+        jq --argjson authinfo "$(</tmp/local_reg.json)" '.auths += $authinfo' "${pull_secret_file-/tmp/ocp_pullsecret.json}" > "${MY_MIRROR_DIR}/ocp4_install/ocp_pullsecret.json"
+
+        # now really doing the mirror
+        oc adm -a "${MY_LOCAL_SECRET_JSON}" release mirror \
+               --from="quay.io/${MY_PRODUCT_REPO}/${MY_RELEASE_NAME}:${MY_OCP_RELEASE}-x86_64" \
+               --to="${MY_LOCAL_REGISTRY}/${MY_LOCAL_REPOSITORY}" \
+               --to-release-image="${MY_LOCAL_REGISTRY}/${MY_LOCAL_REPOSITORY}:${MY_OCP_RELEASE}"
+
+
+        # list the mirrored registry catalog again to verify it's working
+        curl -u "$MY_REGISTRY_USER:$MY_REGISTRY_PASSWORD" "https://${MY_LOCAL_REGISTRY}/v2/_catalog"
+
+        # generate nginx config based on template and variables
+        cp /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bak.by.mirror.registry
+        envsubst < "${SCRIPT_ABS_PATH}/nginx.conf.tpl" > /etc/nginx/nginx.conf
+
+        # make the dir part of the MIRROR_DIR under the nginx default root dir first
+        # so that link succeed
+        MY_NGINX_DEFAULT_DOC_ROOT="/usr/share/nginx/html"
+        MY_MIRROR_DIR_DIR_PART=$(dirname "$MY_MIRROR_DIR")
+        mkdir -p "$MY_NGINX_DEFAULT_DOC_ROOT/$MY_MIRROR_DIR_DIR_PART"
+        ln -s "${MY_MIRROR_DIR}" "MY_NGINX_DEFAULT_DOC_ROOT/${MY_MIRROR_DIR}"
+        systemctl restart nginx;systemctl enable nginx
+
+        # check the http list entries
+        curl -L -s "http://${MY_REGISTRY_SERVER}:${MY_MIRROR_REGISTRY_HTTP_PORT}${MY_MIRROR_DIR}" --list-only
+
+        # generate a systemd service the for mirror registry
+        #podman generate systemd mirror-registry -n > /etc/systemd/system/container-mirror-registry.service
+        #systemctl enable container-mirror-registry.service
+        #systemctl daemon-reload
+
+        ;;
       *) ;;
     esac
 
diff --git a/prepare.sh b/prepare.sh
index 069c1cd..a63ff8f 100755
--- a/prepare.sh
+++ b/prepare.sh
@@ -72,6 +72,16 @@ pushd $SCRIPT_DIR > /dev/null
 # Run ansible playbook
 inventory_file=$(realpath $INVENTORY_FILE_PARAM)
 
+# add airgap install support if air_gapped_install=True
+is_airgap_install=$(grep 'air_gapped_install=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
+if [ "X$is_airgap_install" == "XTrue" ]; then
+  sh -c ". $inventory_file 2>/dev/null; pull_secret_file=$pull_secret_file $SCRIPT_DIR/mirror_ocp.sh"
+  # replace the pull_secret file with the new one which includes the mirror registry
+  air_gapped_download_dir=$(grep 'air_gapped_download_dir=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
+  echo "mirrored created, using the new pull_secret file: ${air_gapped_download_dir}/ocp4_install/ocp_pullsecret.json"
+  export pull_secret_file="${air_gapped_download_dir}/ocp4_install/ocp_pullsecret.json"
+fi
+
 ansible-playbook -i $inventory_file playbooks/ocp4.yaml \
   -e ansible_ssh_pass=$root_password \
   -e ocp_admin_password=$ocp_admin_password \

From d15e3048aa67439141948ccd49ca803cc699dece Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Sun, 15 Jan 2023 20:41:20 +0800
Subject: [PATCH 16/26] add init_bastion into prepare.sh

---
 init_bastion.sh | 5 ++---
 prepare.sh      | 3 +++
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/init_bastion.sh b/init_bastion.sh
index fd14b89..e3efc0c 100755
--- a/init_bastion.sh
+++ b/init_bastion.sh
@@ -9,7 +9,7 @@ fi
 
 init_with_root_or_sudo "$0"
 
-begin_banner "Top level" "project env prepare"
+begin_banner "Top level" "Init bastion machine"
 
     case ${THE_DISTRIBUTION_ID} in
       debian)
@@ -41,5 +41,4 @@ begin_banner "Top level" "project env prepare"
       *) ;;
     esac
 
-done_banner "Top level" "project env prepare"
-
+done_banner "Top level" "Init bastion machine"
diff --git a/prepare.sh b/prepare.sh
index a63ff8f..caa2fe7 100755
--- a/prepare.sh
+++ b/prepare.sh
@@ -72,6 +72,9 @@ pushd $SCRIPT_DIR > /dev/null
 # Run ansible playbook
 inventory_file=$(realpath $INVENTORY_FILE_PARAM)
 
+# init the bastion machine
+"$SCRIPT_DIR"/init_bastion.sh
+
 # add airgap install support if air_gapped_install=True
 is_airgap_install=$(grep 'air_gapped_install=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
 if [ "X$is_airgap_install" == "XTrue" ]; then

From bd29f3efcb4f13c7f26e358cf2d6582d57e33f91 Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Sun, 15 Jan 2023 21:00:29 +0800
Subject: [PATCH 17/26] use command line env

---
 prepare.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/prepare.sh b/prepare.sh
index caa2fe7..f1aac81 100755
--- a/prepare.sh
+++ b/prepare.sh
@@ -78,9 +78,13 @@ inventory_file=$(realpath $INVENTORY_FILE_PARAM)
 # add airgap install support if air_gapped_install=True
 is_airgap_install=$(grep 'air_gapped_install=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
 if [ "X$is_airgap_install" == "XTrue" ]; then
-  sh -c ". $inventory_file 2>/dev/null; pull_secret_file=$pull_secret_file $SCRIPT_DIR/mirror_ocp.sh"
-  # replace the pull_secret file with the new one which includes the mirror registry
+  domain_name=$(grep 'domain_nam=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
+  air_gapped_registry_server=$(grep 'air_gapped_registry_server=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
   air_gapped_download_dir=$(grep 'air_gapped_download_dir=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
+  http_server_port=$(grep 'http_server_port=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
+  openshift_release=$(grep 'openshift_release=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
+  sh -c "pull_secret_file=$pull_secret_file domain_name=$domain_name  openshift_release=$openshift_release air_gapped_registry_server=$air_gapped_registry_server air_gapped_download_dir=$air_gapped_download_dir http_server_port=$http_server_port $SCRIPT_DIR/mirror_ocp.sh"
+  # replace the pull_secret file with the new one which includes the mirror registry
   echo "mirrored created, using the new pull_secret file: ${air_gapped_download_dir}/ocp4_install/ocp_pullsecret.json"
   export pull_secret_file="${air_gapped_download_dir}/ocp4_install/ocp_pullsecret.json"
 fi

From 7139e7fcd64952dfd9eefeec0f371a67063d221f Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Sun, 15 Jan 2023 21:06:07 +0800
Subject: [PATCH 18/26] fix minor bug

---
 mirror_ocp.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mirror_ocp.sh b/mirror_ocp.sh
index 9c44b0f..a608386 100755
--- a/mirror_ocp.sh
+++ b/mirror_ocp.sh
@@ -68,8 +68,8 @@ begin_banner "Top level" "create an OCP mirror registry"
         grep "${PRIV_IP} ${MY_REGISTRY_SERVER} registry" /etc/hosts > /dev/null || echo "${PRIV_IP} ${MY_REGISTRY_SERVER} registry" >> /etc/hosts
 
         # make mirror dir
-        mkdir -p "${MY_MIRROR_DIR}/{clients,dependencies,ocp4_install}"
-        mkdir -p "${MY_MIRROR_DIR}/registry/{auth,certs,data,images}"
+        mkdir -p "${MY_MIRROR_DIR}"/{clients,dependencies,ocp4_install}
+        mkdir -p "${MY_MIRROR_DIR}"/registry/{auth,certs,data,images}
 
         # download oc client
         wget -c "https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-${MY_OCP_RELEASE_MAIN_VERSION}/openshift-client-linux.tar.gz" -O "${MY_MIRROR_DIR}/clients/openshift-client-linux.tar.gz"

From 5437e26e868f14bee16aaeaf1156cc48a7435f88 Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Sun, 15 Jan 2023 21:09:39 +0800
Subject: [PATCH 19/26] change rhcos version to 4.8.47

---
 mirror_ocp.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mirror_ocp.sh b/mirror_ocp.sh
index a608386..ef794ca 100755
--- a/mirror_ocp.sh
+++ b/mirror_ocp.sh
@@ -48,7 +48,7 @@ begin_banner "Top level" "create an OCP mirror registry"
 
         export MY_OCP_RELEASE_MAIN_VERSION="${OCP_RELEASE_MAIN_VERSION-$openshift_release}"
         export MY_OCP_RELEASE="${OCP_RELEASE-$MY_OCP_RELEASE_MAIN_VERSION.52}"
-        export MY_RHCOS_RELEASE="${RHCOS_RELEASE-$MY_OCP_RELEASE_MAIN_VERSION.14}"
+        export MY_RHCOS_RELEASE="${RHCOS_RELEASE-$MY_OCP_RELEASE_MAIN_VERSION.47}"
         export MY_LOCAL_REPOSITORY="${LOCAL_REPOSITORY-ocp4/openshift4}"
         export MY_PRODUCT_REPO="${PRODUCT_REPO-openshift-release-dev}"
         export MY_MIRROR_DIR="${MIRROR_DIR-$air_gapped_download_dir}"

From 3b3de022f878678cc19a000abb623a3b7fc2b488 Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Mon, 16 Jan 2023 11:54:58 +0800
Subject: [PATCH 20/26] optimize the integration of the init_bastion and
 mirror_ocp within the preapre

---
 mirror_ocp.sh |  3 +--
 prepare.sh    | 11 +++++++++--
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/mirror_ocp.sh b/mirror_ocp.sh
index ef794ca..993bbee 100755
--- a/mirror_ocp.sh
+++ b/mirror_ocp.sh
@@ -130,8 +130,7 @@ begin_banner "Top level" "create an OCP mirror registry"
         # generate the pull secret for the mirroed registry
         AUTH=$(echo -n "$MY_REGISTRY_USER:$MY_REGISTRY_PASSWORD" | base64 -w0)
 
-        CUST_REG='{"%s": {"auth":"%s", "email":"%s"}}\n'
-        printf "$CUST_REG" "$MY_LOCAL_REGISTRY" "$AUTH" "$MY_EMAIL" > /tmp/local_reg.json
+        printf '{"%s": {"auth":"%s", "email":"%s"}}\n' "$MY_LOCAL_REGISTRY" "$AUTH" "$MY_EMAIL" > /tmp/local_reg.json
 
         jq --argjson authinfo "$(</tmp/local_reg.json)" '.auths += $authinfo' "${pull_secret_file-/tmp/ocp_pullsecret.json}" > "${MY_MIRROR_DIR}/ocp4_install/ocp_pullsecret.json"
 
diff --git a/prepare.sh b/prepare.sh
index f1aac81..c9ec715 100755
--- a/prepare.sh
+++ b/prepare.sh
@@ -73,7 +73,7 @@ pushd $SCRIPT_DIR > /dev/null
 inventory_file=$(realpath $INVENTORY_FILE_PARAM)
 
 # init the bastion machine
-"$SCRIPT_DIR"/init_bastion.sh
+"$SCRIPT_DIR"/init_bastion.sh || my_exist "failed to initialize the bostion machine" 224
 
 # add airgap install support if air_gapped_install=True
 is_airgap_install=$(grep 'air_gapped_install=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
@@ -83,7 +83,14 @@ if [ "X$is_airgap_install" == "XTrue" ]; then
   air_gapped_download_dir=$(grep 'air_gapped_download_dir=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
   http_server_port=$(grep 'http_server_port=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
   openshift_release=$(grep 'openshift_release=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
-  sh -c "pull_secret_file=$pull_secret_file domain_name=$domain_name  openshift_release=$openshift_release air_gapped_registry_server=$air_gapped_registry_server air_gapped_download_dir=$air_gapped_download_dir http_server_port=$http_server_port $SCRIPT_DIR/mirror_ocp.sh"
+  pull_secret_file=$pull_secret_file \
+    domain_name=$domain_name \
+    openshift_release=$openshift_release \
+    air_gapped_registry_server=$air_gapped_registry_server \
+    air_gapped_download_dir=$air_gapped_download_dir \
+    http_server_port=$http_server_port \
+    "$SCRIPT_DIR/mirror_ocp.sh" \
+    || my_exist "failed to create the OCP mirror" 224
   # replace the pull_secret file with the new one which includes the mirror registry
   echo "mirrored created, using the new pull_secret file: ${air_gapped_download_dir}/ocp4_install/ocp_pullsecret.json"
   export pull_secret_file="${air_gapped_download_dir}/ocp4_install/ocp_pullsecret.json"

From c1dc4d92b572c50eec5a3f0cd892e5e89bd1ba20 Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Mon, 16 Jan 2023 11:59:01 +0800
Subject: [PATCH 21/26] add my_exit to prepare.sh

---
 prepare.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/prepare.sh b/prepare.sh
index c9ec715..839b936 100755
--- a/prepare.sh
+++ b/prepare.sh
@@ -72,6 +72,12 @@ pushd $SCRIPT_DIR > /dev/null
 # Run ansible playbook
 inventory_file=$(realpath $INVENTORY_FILE_PARAM)
 
+# Utilities function
+my_exit () {
+    echo "EXIT: - [HOST:$(hostname)]: - $(date +"%Y-%m-%d %H:%M:%S") - $1"
+    exit "$2"
+}
+
 # init the bastion machine
 "$SCRIPT_DIR"/init_bastion.sh || my_exist "failed to initialize the bostion machine" 224
 

From 0ff8c69d5a030f98c24c621cf55e9ae7b7edd90b Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Mon, 16 Jan 2023 12:01:28 +0800
Subject: [PATCH 22/26] fix typo.

---
 prepare.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/prepare.sh b/prepare.sh
index 839b936..4f64920 100755
--- a/prepare.sh
+++ b/prepare.sh
@@ -79,7 +79,7 @@ my_exit () {
 }
 
 # init the bastion machine
-"$SCRIPT_DIR"/init_bastion.sh || my_exist "failed to initialize the bostion machine" 224
+"$SCRIPT_DIR"/init_bastion.sh || my_exit "failed to initialize the bostion machine" 224
 
 # add airgap install support if air_gapped_install=True
 is_airgap_install=$(grep 'air_gapped_install=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
@@ -96,7 +96,7 @@ if [ "X$is_airgap_install" == "XTrue" ]; then
     air_gapped_download_dir=$air_gapped_download_dir \
     http_server_port=$http_server_port \
     "$SCRIPT_DIR/mirror_ocp.sh" \
-    || my_exist "failed to create the OCP mirror" 224
+    || my_exit "failed to create the OCP mirror" 224
   # replace the pull_secret file with the new one which includes the mirror registry
   echo "mirrored created, using the new pull_secret file: ${air_gapped_download_dir}/ocp4_install/ocp_pullsecret.json"
   export pull_secret_file="${air_gapped_download_dir}/ocp4_install/ocp_pullsecret.json"

From 596f1768d310593604ab3f8830046335b7b0e743 Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Mon, 16 Jan 2023 12:04:43 +0800
Subject: [PATCH 23/26] add --skip-broken to init bastion machine.

---
 init_bastion.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/init_bastion.sh b/init_bastion.sh
index e3efc0c..1d90b41 100755
--- a/init_bastion.sh
+++ b/init_bastion.sh
@@ -29,7 +29,7 @@ begin_banner "Top level" "Init bastion machine"
         systemctl status firewalld > /dev/null 2>&1 && systemctl stop firewalld && systemctl disable firewalld
         yum -y update
         yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
-        yum install -y ansible bind-utils buildah chrony dnsmasq git \
+        yum install -y --skip-broken ansible bind-utils buildah chrony dnsmasq git \
                        haproxy httpd-tools jq libvirt net-tools nfs-utils nginx podman \
                        python3 python3-netaddr python3-passlib python3-pip python3-policycoreutils python3-pyvmomi python3-requests \
                        screen sos syslinux-tftpboot wget yum-utils

From 6d6fa85bdd8ab958526de6273a3e17454d95a9fa Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Mon, 16 Jan 2023 12:15:30 +0800
Subject: [PATCH 24/26] fix minor bugs.

---
 init_bastion.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/init_bastion.sh b/init_bastion.sh
index 1d90b41..e3efc0c 100755
--- a/init_bastion.sh
+++ b/init_bastion.sh
@@ -29,7 +29,7 @@ begin_banner "Top level" "Init bastion machine"
         systemctl status firewalld > /dev/null 2>&1 && systemctl stop firewalld && systemctl disable firewalld
         yum -y update
         yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
-        yum install -y --skip-broken ansible bind-utils buildah chrony dnsmasq git \
+        yum install -y ansible bind-utils buildah chrony dnsmasq git \
                        haproxy httpd-tools jq libvirt net-tools nfs-utils nginx podman \
                        python3 python3-netaddr python3-passlib python3-pip python3-policycoreutils python3-pyvmomi python3-requests \
                        screen sos syslinux-tftpboot wget yum-utils

From e284a5706a2dd48a55349c32fa1b95175a14cb62 Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Mon, 16 Jan 2023 15:19:00 +0800
Subject: [PATCH 25/26] remove quote for environment variables.

---
 mirror_ocp.sh | 30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

diff --git a/mirror_ocp.sh b/mirror_ocp.sh
index 993bbee..0fd528c 100755
--- a/mirror_ocp.sh
+++ b/mirror_ocp.sh
@@ -38,22 +38,22 @@ begin_banner "Top level" "create an OCP mirror registry"
         fi
 
         # export environment variables for reusing
-        export MY_REGISTRY_DOMAIN="${REGISTRY_DOMAIN-$domain_name}"
-        export MY_REGISTRY_SERVER="${REGISTRY_SERVER-$air_gapped_registry_server}"
+        export MY_REGISTRY_DOMAIN=${REGISTRY_DOMAIN-$domain_name}
+        export MY_REGISTRY_SERVER=${REGISTRY_SERVER-$air_gapped_registry_server}
         export MY_REGISTRY_PORT=${REGISTRY_PORT-5000}
-        export MY_LOCAL_REGISTRY="${LOCAL_REGISTRY-$MY_REGISTRY_SERVER:$MY_REGISTRY_PORT}"
-        export MY_EMAIL="${EMAIL-admin@$MY_REGISTRY_DOMAIN}"
-        export MY_REGISTRY_USER="${REGISTRY_USER-admin}"
-        export MY_REGISTRY_PASSWORD="${REGISTRY_PASSWORD-passw0rd}"
-
-        export MY_OCP_RELEASE_MAIN_VERSION="${OCP_RELEASE_MAIN_VERSION-$openshift_release}"
-        export MY_OCP_RELEASE="${OCP_RELEASE-$MY_OCP_RELEASE_MAIN_VERSION.52}"
-        export MY_RHCOS_RELEASE="${RHCOS_RELEASE-$MY_OCP_RELEASE_MAIN_VERSION.47}"
-        export MY_LOCAL_REPOSITORY="${LOCAL_REPOSITORY-ocp4/openshift4}"
-        export MY_PRODUCT_REPO="${PRODUCT_REPO-openshift-release-dev}"
-        export MY_MIRROR_DIR="${MIRROR_DIR-$air_gapped_download_dir}"
-        export MY_LOCAL_SECRET_JSON="${LOCAL_SECRET_JSON-$MY_MIRROR_DIR/ocp4_install/ocp_pullsecret.json}"
-        export MY_RELEASE_NAME="${RELEASE_NAME-ocp-release}"
+        export MY_LOCAL_REGISTRY=${LOCAL_REGISTRY-$MY_REGISTRY_SERVER:$MY_REGISTRY_PORT}
+        export MY_EMAIL=${EMAIL-admin@$MY_REGISTRY_DOMAIN}
+        export MY_REGISTRY_USER=${REGISTRY_USER-admin}
+        export MY_REGISTRY_PASSWORD=${REGISTRY_PASSWORD-passw0rd}
+
+        export MY_OCP_RELEASE_MAIN_VERSION=${OCP_RELEASE_MAIN_VERSION-$openshift_release}
+        export MY_OCP_RELEASE=${OCP_RELEASE-$MY_OCP_RELEASE_MAIN_VERSION.52}
+        export MY_RHCOS_RELEASE=${RHCOS_RELEASE-$MY_OCP_RELEASE_MAIN_VERSION.47}
+        export MY_LOCAL_REPOSITORY=${LOCAL_REPOSITORY-ocp4/openshift4}
+        export MY_PRODUCT_REPO=${PRODUCT_REPO-openshift-release-dev}
+        export MY_MIRROR_DIR=${MIRROR_DIR-$air_gapped_download_dir}
+        export MY_LOCAL_SECRET_JSON=${LOCAL_SECRET_JSON-$MY_MIRROR_DIR/ocp4_install/ocp_pullsecret.json}
+        export MY_RELEASE_NAME=${RELEASE_NAME-ocp-release}
 
         export MY_MIRROR_REGISTRY_HTTP_PORT=${MIRROR_REGISTRY_HTTP_PORT-$http_server_port}
 

From 57f6db3e803ae13acd77ae5f6472ebca620e2ffb Mon Sep 17 00:00:00 2001
From: Hugh JF Chen <hugh.jf.chen@gmail.com>
Date: Mon, 16 Jan 2023 15:39:56 +0800
Subject: [PATCH 26/26] remove double quote for variable value.

---
 prepare.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/prepare.sh b/prepare.sh
index 4f64920..d4dc1d5 100755
--- a/prepare.sh
+++ b/prepare.sh
@@ -82,13 +82,13 @@ my_exit () {
 "$SCRIPT_DIR"/init_bastion.sh || my_exit "failed to initialize the bostion machine" 224
 
 # add airgap install support if air_gapped_install=True
-is_airgap_install=$(grep 'air_gapped_install=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
+is_airgap_install=$(grep 'air_gapped_install=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}'|tr -d '"')
 if [ "X$is_airgap_install" == "XTrue" ]; then
-  domain_name=$(grep 'domain_nam=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
-  air_gapped_registry_server=$(grep 'air_gapped_registry_server=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
-  air_gapped_download_dir=$(grep 'air_gapped_download_dir=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
-  http_server_port=$(grep 'http_server_port=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
-  openshift_release=$(grep 'openshift_release=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}')
+  domain_name=$(grep 'domain_nam=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}'|tr -d '"')
+  air_gapped_registry_server=$(grep 'air_gapped_registry_server=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}'|tr -d '"')
+  air_gapped_download_dir=$(grep 'air_gapped_download_dir=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}'|tr -d '"')
+  http_server_port=$(grep 'http_server_port=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}'|tr -d '"')
+  openshift_release=$(grep 'openshift_release=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}'|tr -d '"')
   pull_secret_file=$pull_secret_file \
     domain_name=$domain_name \
     openshift_release=$openshift_release \