diff --git a/.gitignore b/.gitignore index e289565..78e6d21 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,5 @@ .DS_Store scrap + +# for sed inplace edit backup +*.bak diff --git a/common.sh b/common.sh new file mode 100755 index 0000000..4cfa877 --- /dev/null +++ b/common.sh @@ -0,0 +1,168 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +guard_bash_error () { + set -Eeuo pipefail +} + +# Log levels +INFO=0 +WARN=1 +ERROR=2 +FATAL=3 +DEBUG=4 +DEFAULT_LOG_LEVEL=${ERROR} + +my_exit () { + echo "EXIT: - [HOST:$(hostname)]: - $(date +"%Y-%m-%d %H:%M:%S") - $1" + exit "$2" +} + +msg () { + if [ "$1" -le ${DEFAULT_LOG_LEVEL} ]; then + echo "[HOST:$(hostname)]: - $(date +"%Y-%m-%d %H:%M:%S") - $2" + fi +} + +info () { + msg ${INFO} "INFO: - $1" +} + +warn () { + msg ${WARN} "WARNING: - $1" +} + +error () { + msg ${ERROR} "ERROR: - $1" +} + +fatal () { + msg ${FATAL} "FATAL: - $1" +} + +debug () { + msg ${DEBUG} "DEBUG: - $1" +} + +begin_banner () { + info "$1 - $2 phase - begin" +} + +done_banner () { + info "$1 - $2 phase - done" +} + +### turn path within script into absolute path +### must pass the calling string of the script as the first parameter +### e.g., ./path_to_script/script.sh +### or, /root/path_to_script/script.sh +### return the absolute path to the script with "echo" command +turn_to_absolute_path () { + local SCRIPT_ABS_PATH_RAW + SCRIPT_ABS_PATH_RAW="$(dirname "$1")" + # turn SCRIPT_ABS_PATH into absolute path + case ${SCRIPT_ABS_PATH_RAW} in + /*) echo "${SCRIPT_ABS_PATH_RAW}" ;; + \.\.*) echo "$PWD/${SCRIPT_ABS_PATH_RAW}" ;; + \.*) echo "$PWD/${SCRIPT_ABS_PATH_RAW}" ;; + *) echo "$PWD" ;; + esac +} + +### change CD to up to the project root directory +### must pass the absolute path to the script as the first parameter +change_CD_to_project_root () { + cd "$1" + local up_level=.. + local my_loop=10 # guard not to loop forever + until [ -f "${up_level}/cook.sh" ] && [ ${my_loop} -gt 0 ] + do + up_level=${up_level}/.. + my_loop=$((my_loop - 1)) + done + if [ ${my_loop} -eq 0 ]; then + my_exit "Too many level up within the searching for DevOps directory,abort." 1 + fi + cd "$1/${up_level}" +} + +### check OS and distribution +### return the OS distribution and ID with "echo" command +check_dist_or_OS () { + local MY_THE_DISTRIBUTION_ID="" + local MY_THE_DISTRIBUTION_VERSION="" + if [ -e /etc/os-release ]; then + MY_THE_DISTRIBUTION_ID=$(grep -w "ID" /etc/os-release |awk -F"=" '{print $NF}'|sed 's/"//g') + if [ "${MY_THE_DISTRIBUTION_ID}" == "ubuntu" ]; then + MY_THE_DISTRIBUTION_VERSION=$(grep -w "VERSION_ID" /etc/os-release |awk -F"=" '{print $NF}'|sed 's/"//g') + else + MY_THE_DISTRIBUTION_VERSION=$(grep -w "VERSION_ID" /etc/os-release |awk -F"=" '{print $NF}'|awk -F"." '{print $1}'|sed 's/"//g') + fi + echo "${MY_THE_DISTRIBUTION_ID} ${MY_THE_DISTRIBUTION_VERSION}" + else + if type uname > /dev/null 2>&1; then + MY_THE_DISTRIBUTION_ID=$(uname -s) + MY_THE_DISTRIBUTION_VERSION=$(uname -r) + echo "${MY_THE_DISTRIBUTION_ID} ${MY_THE_DISTRIBUTION_VERSION}" + else + echo "" + fi + fi +} + +### guard that the caller of the script must be root or has sudo right +guard_root_or_sudo () { + if [[ $EUID -gt 0 ]] && ! sudo echo >/dev/null 2>&1; then + return 1 + else + return 0 + fi +} + +### init script with check if root or sudo +init_with_root_or_sudo () { + guard_bash_error + + if ! guard_root_or_sudo; then + my_exit "You must be root or you must be sudoer to prepare the env for CI/CD." 1 + fi + + SCRIPT_ABS_PATH=$(turn_to_absolute_path "$0") + + # change_CD_to_project_root ${SCRIPT_ABS_PATH} + + THE_DISTRIBUTION_ID_VERSION=$(check_dist_or_OS) + THE_DISTRIBUTION_ID=$(echo "${THE_DISTRIBUTION_ID_VERSION}"|awk '{print $1}') + THE_DISTRIBUTION_VERSION=$(echo "${THE_DISTRIBUTION_ID_VERSION}"|awk '{print $2}') +} + +### init script without check if root or sudo +init_without_root_or_sudo () { + guard_bash_error + + SCRIPT_ABS_PATH=$(turn_to_absolute_path "$0") + + # change_CD_to_project_root ${SCRIPT_ABS_PATH} + + THE_DISTRIBUTION_ID_VERSION=$(check_dist_or_OS) + THE_DISTRIBUTION_ID=$(echo "${THE_DISTRIBUTION_ID_VERSION}"|awk '{print $1}') + THE_DISTRIBUTION_VERSION=$(echo "${THE_DISTRIBUTION_ID_VERSION}"|awk '{print $2}') +} + +get_last_stable_nix_channel () { + local MY_CHANNEL_NAME_REGEX="" + case ${THE_DISTRIBUTION_ID} in + debian|ubuntu|rhel|centos) MY_CHANNEL_NAME_REGEX='s/.*\(nixos-[0-9][0-9].[0-9][0-9]\).*/\1/p' ;; + Darwin) MY_CHANNEL_NAME_REGEX='s/.*\(nixpkgs-[0-9][0-9].[0-9][0-9]-darwin\).*/\1/p' ;; + *) ;; + esac + local MY_LAST_NIX_STABLE_CHANNEL + MY_LAST_NIX_STABLE_CHANNEL=$(git ls-remote --heads https://github.com/NixOS/nixpkgs | awk '{print $NF}' | awk -F"/" '{print $NF}' | grep -v "\-unstable" | grep -v "\-small" | sed -n "${MY_CHANNEL_NAME_REGEX}" | sort | tail -1) + echo "${MY_LAST_NIX_STABLE_CHANNEL}" +} + +switch_to_last_stable_nix_channel () { + nix-channel --remove nixpkgs + nix-channel --add "https://nixos.org/channels/$(get_last_stable_nix_channel)" nixpkgs + nix-channel --update +} diff --git a/init_bastion.sh b/init_bastion.sh new file mode 100755 index 0000000..e3efc0c --- /dev/null +++ b/init_bastion.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +if ! type dirname > /dev/null 2>&1; then + echo "Not even a linux or macOS, Windoze? We don't support it. Abort." + exit 1 +fi + +. "$(dirname "$0")"/common.sh + +init_with_root_or_sudo "$0" + +begin_banner "Top level" "Init bastion machine" + + case ${THE_DISTRIBUTION_ID} in + debian) + my_exit "debian not supported yet." 222 + ;; + ubuntu) + my_exit "ubuntu not supported yet." 222 + ;; + Darwin) + my_exit "macOS not supported yet." 222 + ;; + rhel|centos) + if [ "X$THE_DISTRIBUTION_VERSION" != "X8" ]; then + my_exit "only support centos/RHEL 8.x" 126 + fi + + systemctl status firewalld > /dev/null 2>&1 && systemctl stop firewalld && systemctl disable firewalld + yum -y update + yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm + yum install -y ansible bind-utils buildah chrony dnsmasq git \ + haproxy httpd-tools jq libvirt net-tools nfs-utils nginx podman \ + python3 python3-netaddr python3-passlib python3-pip python3-policycoreutils python3-pyvmomi python3-requests \ + screen sos syslinux-tftpboot wget yum-utils + + LATEST_PIP=$(find /usr/bin -name 'pip*'|sort|tail -1) + "$LATEST_PIP" install passlib + + ;; + *) ;; + esac + +done_banner "Top level" "Init bastion machine" diff --git a/inventory/vmware-airgapped-example.inv b/inventory/vmware-airgapped-example.inv index 8386bd5..d3fde84 100644 --- a/inventory/vmware-airgapped-example.inv +++ b/inventory/vmware-airgapped-example.inv @@ -7,6 +7,7 @@ ansible_ssh_common_args='-o StrictHostKeyChecking=no' domain_name="uk.ibm.com" cluster_name="ocp45" default_gateway="10.99.92.1" +interface_name="ens192" # # Indicate the method by which Red Hat CoreOS will be installed. This can be one of the following values: diff --git a/inventory/vmware-example-410.inv b/inventory/vmware-example-410.inv index 1f57235..ffc086b 100644 --- a/inventory/vmware-example-410.inv +++ b/inventory/vmware-example-410.inv @@ -7,6 +7,7 @@ ansible_ssh_common_args='-o StrictHostKeyChecking=no' domain_name="coc.ibm.com" cluster_name="ocp410" default_gateway="10.99.92.1" +interface_name="ens192" # # Indicate the method by which Red Hat CoreOS will be installed. This can be one of the following values: diff --git a/inventory/vmware-example-48-ipi.inv b/inventory/vmware-example-48-ipi.inv index 16b498b..ca0e8e3 100644 --- a/inventory/vmware-example-48-ipi.inv +++ b/inventory/vmware-example-48-ipi.inv @@ -7,6 +7,7 @@ ansible_ssh_common_args='-o StrictHostKeyChecking=no' domain_name="coc.ibm.com" cluster_name="ocp48" default_gateway="10.99.92.1" +interface_name="ens192" # # Indicate the method by which Red Hat CoreOS will be installed. This can be one of the following values: @@ -210,4 +211,4 @@ vm_worker_disk=200 # Ignored for IPI installations [workers] -# Ignored for IPI installations \ No newline at end of file +# Ignored for IPI installations diff --git a/inventory/vmware-example-48-with-new-options.inv b/inventory/vmware-example-48-with-new-options.inv new file mode 100644 index 0000000..afebd1d --- /dev/null +++ b/inventory/vmware-example-48-with-new-options.inv @@ -0,0 +1,238 @@ +[all:children] +dhcp +masters +workers + +[all:vars] +ansible_ssh_common_args='-o StrictHostKeyChecking=no' +domain_name="ocp.mycom.com" +cluster_name="ocpmas" +default_gateway="10.11.80.1" +node_interface_name="ens192" + +# +# Indicate the method by which Red Hat CoreOS will be installed. This can be one of the following values: +# - pxe: CoreOS will be loaded on the bootstrap, masters and workers using PXE-boot. This is the default. +# - ova: Only for VMWare. CoreOS will be loaded using a VMWare template (imported OVA file). +# - ipi: Only for VMWare. CoreOS and OpenShift will be installed via Installer-provisioned infrastructure. OpenShift +# will take care of creating the required VMs and automatically assigns hostnames. If you choose this installation +# method, please also complete the IPI section in the inventory file. +# +# For pxe and ova installs you have the option of letting the prepare script create the VMs for you by setting +# vm_create_vms=True. This can only be done if you specify the vc_user and vc_password properties at the command line. +# If you specify run_install=True and vm_create_vms=True, the script will start the virtual machines. Otherwise, you must +# start the bootstrap, masters and workers yourself while the script is waiting for the bootstrap. +# When rhcos_installation_method=ipi, run_install is assumed to be True as well. +# +rhcos_installation_method=pxe +vm_create_vms=False +run_install=False + +# +# OCP Installation directory +# Depicts the directory on the bastion (current) node in which the OpenShift installation +# artifacts will be stored. +# +ocp_install_dir="/root/prepare-ocp/artifact/ocp_install" + +# +# Proxy settings. If the nodes can only reach the internet through a proxy, the proxy server must be configured +# in the OpenShift installation configuration file. Make sure that you specify all 3 properties: http_proxy, https_proxy +# and no_proxy. Property no_proxy must contain the domain name, the k8s internal addresses (10.1.0.0/16 and 172.30.0.0/16), +# the internal services domain (.svc) and the IP range of the cluster nodes (for example 192.168.1.0/24). +# Additionally, if the bastion (and other non-cluster nodes) have not yet +# been configured with a global proxy environment variable, the preparation script can add this to the profile of all users. +# +#http_proxy="http://bastion.{{cluster_name}}.{{domain_name}}:3128" +#https_proxy="http://bastion.{{cluster_name}}.{{domain_name}}:3128" +#no_proxy=".{{domain_name}},10.1.0.0/16,{{service_network}},10.99.92.0/24,.svc" +#configure_global_proxy=True + +# +# OpenShift download URLs. These are the URLs used by the prepare script to download +# the packages needed. If you want to install a different release of OpenShift, it is sufficient to change the +# openshift_release property. The download scripts automatically select the relevant packages from the URL. +# +openshift_release="4.8" +openshift_base_url="https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-{{openshift_release}}/" +rhcos_base_url="https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/{{openshift_release}}/latest/" +openshift_client_package_pattern=".*(openshift-client.*tar.gz).*" +openshift_install_package_pattern=".*(openshift-install-linux.*tar.gz).*" +rhcos_metal_package_pattern=".*(rhcos.*metal.*raw.gz).*" +rhcos_kernel_package_pattern=".*(rhcos.*-kernel.x86_64).*" +rhcos_initramfs_package_pattern=".*(rhcos.*initramfs.x86_64.img).*" +rhcos_rootfs_package_pattern=".*(rhcos.*rootfs.x86_64.img).*" +rhcos_ova_package_pattern=".*(rhcos.*.ova).*" + +# +# Indicate whether to opt out of remote health checking of the OpenShift cluster +# +opt_out_health_checking=False + +# +# Indicates whether the chrony NTP service must be enabled on the bastion node. If enabled, +# all OpenShift nodes, bootstrap, masters and workers will synchronize their times with the +# bastion node instead of a public NTP server. If you use the bastion node as the NTP server, +# you have to specify which range of servers will be allowed to synchronize +# themselves with the bastion node; you can do this in the ntp_allow property. +# Finally, specify a list of external servers with which the nodes will by synchronized, for example: +# ntp_server=['0.rhel.pool.ntp.org', '1.rhel.pool.ntp.org']. +# +setup_chrony_server=False +ntp_allow="10.11.80.0/24" +ntp_servers=['10.11.80.187'] +override_chrony_settings_on_cluster_nodes=True + +# +# A DNS server (dnsmasq) will be set up on the bastion node. Specify the upstream name server +# that dnsmasq will use. +# +external_name_servers=['8.8.8.8'] +bastion_interface_script="/etc/sysconfig/network-scripts/ifcfg-eth0" + +# +# Indicate whether DHCP and TFTP must run on the bastion node as part of dnsmasq. When using PXE boot, the masters +# and worker will get an fixed IP address from the dnsmasq DHCP server. Specify a range of addresses from which the +# DHCP server can issue an IP address. Every node configured in the bootstrap, masters and workers sections below will +# have a specific dhcp-server entry in the dnsmasq configuration, specifying the IP address they will be assigned by +# the DHCP server. If the DHCP server runs on the bastion node and the RHCOS installation (rhcos_installation_method) is PXE, +# it will also serve the RHCOS ISO using the TFTP server. +# +dhcp_on_bastion=True +# to support multi subnet DHCP, the dhcp range need to be listed in a group which is listed on the bottom, like +# master/worker nodes. +# dhcp_range=['10.11.80.51','10.11.80.60'] + +# +# Indicate whether the load balancer must be configured by the prepare script. If you choose to use an external load +# balancer, set the manage_load_balancer property to False. You can still configure the load balancer under the [lb] +# section. +# +manage_load_balancer=True + +# +# Indicate the port that the HTTP server (nginx) on the bastion node will listen on. The HTTP server provides access +# to the ignition files required at boot time of the bootstrap, masters and workers, as well as PXE boot assets such as +# RHCOS ISO. +# +http_server_port=8090 + +# Set up desktop on the bastion node +setup_desktop=False + +# Set the bastion hostname as part of the preparation. If set to True, the bastion hostname will be set to +# .. +set_bastion_hostname=False + +# +# If manage_nfs is True, the Ansible script will try to format and mount the NFS volume configured in the nfs_volume* +# properties on the server indicated in the [nfs] section. If the value is False, the nfs server referred to in the [nfs] +# section will still be used to configure the managed-nfs-storage storage class if specified. However in that case, it is assumed +# that an external NFS server is available which is not configured by the Ansible playbooks. +# +# Volume parameters: these parameters indicate which volume the preparation scripts have to +# format and mount. The "selector" parameter is used to find the device using the lsblk +# command; you can specify the size (such as 500G) or any unique string that identifies +# the device (such as sdb). +# +# The "nfs_volume_mount_path" indicates the mount point that is created for the device. Even if you are not managing NFS +# yourself, but are using an external NFS server for NFS storge class, you should configure the path that must be mounted. +# +manage_nfs=True +nfs_volume_selector="sdb" + +nfs_volume_mount_path="/nfs" + +# +# Storageclass parameters: indicate whether NFS and/or Portworx storage classes must be created +# + +# Create managed-nfs-storage storage class? +create_nfs_sc=True + +# Install Portworx and create storage classes? +create_portworx_sc=False + +# +# Registry storage class and size to be used when image registry is configured. If NFS is used for +# the image registry, the storage class is typically managed-nfs-storage. For OCS, it would be ocs-storagecluster-cephfs. +# +image_registry_storage_class=managed-nfs-storage +image_registry_size=200Gi + +# This variable configures the subnet in which services will be created in the OpenShift Container Platform SDN. +# Specify a private block that does not conflict with any existing network blocks in your infrastructure to which pods, +# nodes, or the master might require access to, or the installation will fail. Defaults to 172.30.0.0/16, +# and cannot be re-configured after deployment. If changing from the default, avoid 172.17.0.0/16, +# which the docker0 network bridge uses by default, or modify the docker0 network. +# +service_network="172.30.0.0/16" + +# +# Additional variables for IPI (Installer-provisioned Infrastructure) installations. +# +apiVIP="10.11.48.51" +ingressVIP="10.11.48.52" +vm_number_of_masters=3 +vm_number_of_workers=3 + +# +# VMware envrionment specific properties. These are only used if you have selected the IPI installation method, or if +# you want the prepare script to create the VMs or when using the vm_create.sh script to create these. +# You can ignore these properties if you manually create the VMs. +# +vc_vcenter="10.99.92.13" +vc_datacenter="Datacenter1" +vc_datastore="Datastore1" +vc_cluster="Cluster1" +vc_res_pool="resourcepool" +vc_folder="fkocp48" +vc_guest_id="rhel7_64Guest" +vc_network="VM Network" +vm_template="/Datacenter1/vm/Templates/RHCOS_4.8.8" + +# Bootstrap VM properties +vm_bootstrap_mem=16384 +vm_bootstrap_cpu=4 +vm_bootstrap_disk=100 + +# Master VM properties +vm_master_mem=32768 +vm_master_cpu=8 +vm_master_disk=200 + +# Worker VM properties +vm_worker_mem=65536 +vm_worker_cpu=16 +vm_worker_disk=200 + +# dhcp multi subnet list +# format: +# subnet_tag address_begin address_end gateway + +[dhcp] +sub1 begin="10.11.80.51" end="10.11.80.60" gateway="10.11.80.1" +sub2 begin="10.11.60.51" end="10.11.60.60" gateway="10.11.60.1" + +[lb] +10.11.80.187 host="bastion" + +[nfs] +10.11.80.187 host="bastion" + +[bastion] +10.11.80.187 host="bastion" + +[bootstrap] +10.11.80.51 host="bootstrap" mac="00:50:56:ab:33:90" gateway="10.11.80.1" dhcp_sub_net="sub1" + +[masters] +10.11.80.52 host="master-1" mac="00:50:56:ab:33:91" gateway="10.11.80.1" dhcp_sub_net="sub1" +10.11.80.53 host="master-2" mac="00:50:56:ab:33:92" gateway="10.11.80.1" dhcp_sub_net="sub1" +10.11.80.54 host="master-3" mac="00:50:56:ab:33:93" gateway="10.11.80.1" dhcp_sub_net="sub1" + +[workers] +10.11.80.55 host="worker-1" mac="00:50:56:ab:33:94" gateway="10.11.80.1" dhcp_sub_net="sub1" +10.11.80.56 host="worker-2" mac="00:50:56:ab:33:95" gateway="10.11.80.1" dhcp_sub_net="sub1" +10.11.80.57 host="worker-3" mac="00:50:56:ab:33:96" gateway="10.11.80.1" dhcp_sub_net="sub1" + diff --git a/inventory/vmware-example-48.inv b/inventory/vmware-example-48.inv index 42c67bf..c5d2670 100644 --- a/inventory/vmware-example-48.inv +++ b/inventory/vmware-example-48.inv @@ -7,6 +7,7 @@ ansible_ssh_common_args='-o StrictHostKeyChecking=no' domain_name="coc.ibm.com" cluster_name="ocp48" default_gateway="10.99.92.1" +interface_name="ens192" # # Indicate the method by which Red Hat CoreOS will be installed. This can be one of the following values: diff --git a/inventory/vmware-example-49.inv b/inventory/vmware-example-49.inv index 1870b6d..c96788f 100644 --- a/inventory/vmware-example-49.inv +++ b/inventory/vmware-example-49.inv @@ -7,6 +7,7 @@ ansible_ssh_common_args='-o StrictHostKeyChecking=no' domain_name="coc.ibm.com" cluster_name="ocp49" default_gateway="10.99.92.1" +interface_name="ens192" # # Indicate the method by which Red Hat CoreOS will be installed. This can be one of the following values: diff --git a/mirror_ocp.sh b/mirror_ocp.sh new file mode 100755 index 0000000..0fd528c --- /dev/null +++ b/mirror_ocp.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash + +if ! type dirname > /dev/null 2>&1; then + echo "Not even a linux or macOS, Windoze? We don't support it. Abort." + exit 1 +fi + +. "$(dirname "$0")"/common.sh + +init_with_root_or_sudo "$0" + +begin_banner "Top level" "create an OCP mirror registry" + + case ${THE_DISTRIBUTION_ID} in + debian) + my_exit "debian not supported yet." 222 + ;; + ubuntu) + my_exit "ubuntu not supported yet." 222 + ;; + Darwin) + my_exit "macOS not supported yet." 222 + ;; + rhel|centos) + if [ "X$THE_DISTRIBUTION_VERSION" != "X8" ]; then + my_exit "only support centos/RHEL 8.x" 126 + fi + + # the pull secret file is needed to go forward + #if [ -z $pull_secret_file ];then + # pull_secret_file="/tmp/ocp_pullsecret.json" + #fi + + if [ ! -e "${pull_secret_file-/tmp/ocp_pullsecret.json}" ];then + echo "Pull secret file ${pull_secret_file-/tmp/ocp_pullsecret.json} does not exist, please create the file or set the pull_secret_file environment variable to point to the file that holds the pull secret." + echo "You may also want to check the environment variables value within the script before really invoking the script." + exit 1 + fi + + # export environment variables for reusing + export MY_REGISTRY_DOMAIN=${REGISTRY_DOMAIN-$domain_name} + export MY_REGISTRY_SERVER=${REGISTRY_SERVER-$air_gapped_registry_server} + export MY_REGISTRY_PORT=${REGISTRY_PORT-5000} + export MY_LOCAL_REGISTRY=${LOCAL_REGISTRY-$MY_REGISTRY_SERVER:$MY_REGISTRY_PORT} + export MY_EMAIL=${EMAIL-admin@$MY_REGISTRY_DOMAIN} + export MY_REGISTRY_USER=${REGISTRY_USER-admin} + export MY_REGISTRY_PASSWORD=${REGISTRY_PASSWORD-passw0rd} + + export MY_OCP_RELEASE_MAIN_VERSION=${OCP_RELEASE_MAIN_VERSION-$openshift_release} + export MY_OCP_RELEASE=${OCP_RELEASE-$MY_OCP_RELEASE_MAIN_VERSION.52} + export MY_RHCOS_RELEASE=${RHCOS_RELEASE-$MY_OCP_RELEASE_MAIN_VERSION.47} + export MY_LOCAL_REPOSITORY=${LOCAL_REPOSITORY-ocp4/openshift4} + export MY_PRODUCT_REPO=${PRODUCT_REPO-openshift-release-dev} + export MY_MIRROR_DIR=${MIRROR_DIR-$air_gapped_download_dir} + export MY_LOCAL_SECRET_JSON=${LOCAL_SECRET_JSON-$MY_MIRROR_DIR/ocp4_install/ocp_pullsecret.json} + export MY_RELEASE_NAME=${RELEASE_NAME-ocp-release} + + export MY_MIRROR_REGISTRY_HTTP_PORT=${MIRROR_REGISTRY_HTTP_PORT-$http_server_port} + + # disable firewall first + systemctl status firewalld > /dev/null 2>&1 && systemctl stop firewalld && systemctl disable firewalld + + # install the dependent tools + yum -y install wget podman httpd-tools jq nginx + + # add host to /etc/hosts + PRIV_IP=$(hostname -I|awk '{print $1}') + grep "${PRIV_IP} ${MY_REGISTRY_SERVER} registry" /etc/hosts > /dev/null || echo "${PRIV_IP} ${MY_REGISTRY_SERVER} registry" >> /etc/hosts + + # make mirror dir + mkdir -p "${MY_MIRROR_DIR}"/{clients,dependencies,ocp4_install} + mkdir -p "${MY_MIRROR_DIR}"/registry/{auth,certs,data,images} + + # download oc client + wget -c "https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-${MY_OCP_RELEASE_MAIN_VERSION}/openshift-client-linux.tar.gz" -O "${MY_MIRROR_DIR}/clients/openshift-client-linux.tar.gz" + wget -c "https://mirror.openshift.com/pub/openshift-v4/clients/ocp/latest-${MY_OCP_RELEASE_MAIN_VERSION}/openshift-install-linux.tar.gz" -O "${MY_MIRROR_DIR}/clients/openshift-install-linux.tar.gz" + + # download coreos + # folowwing is for PXE boot + #wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-metal.x86_64.raw.gz -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-metal.x86_64.raw.gz + wget -c "https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-kernel-x86_64" -O "${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-kernel-x86_64" + wget -c "https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-initramfs.x86_64.img" -O "${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-initramfs.x86_64.img" + wget -c "https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-rootfs.x86_64.img" -O "${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live-rootfs.x86_64.img" + # for liveCD installation + wget -c "https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-live.x86_64.iso" -O "${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-live.x86_64.iso" + + # following is for OVA install + # wget -c https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/${MY_OCP_RELEASE_MAIN_VERSION}/latest/rhcos-${MY_RHCOS_RELEASE}-x86_64-vmware.x86_64.ova -O ${MY_MIRROR_DIR}/dependencies/rhcos-${MY_RHCOS_RELEASE}-x86_64-vmware.x86_64.ova + + # extract and install oc into /usr/local/bin + tar xvzf "${MY_MIRROR_DIR}/clients/openshift-client-linux.tar.gz" -C /usr/local/bin + + # generate the self-signed certs for the mirrored registry + openssl req -newkey rsa:4096 -nodes -sha256 -keyout "${MY_MIRROR_DIR}/registry/certs/registry.key" -x509 -days 3650 -out "${MY_MIRROR_DIR}/registry/certs/registry.crt" -subj "/C=US/ST=/L=/O=/CN=$MY_REGISTRY_SERVER" -addext "subjectAltName = DNS:$MY_REGISTRY_SERVER" + + # generate the auth for the mirrored registry + htpasswd -bBc "${MY_MIRROR_DIR}/registry/auth/htpasswd" "$MY_REGISTRY_USER" "$MY_REGISTRY_PASSWORD" + + # pull the registry docker image + podman pull docker.io/library/registry:2 + + # pull the nfs-provider image + # THIS DOES NOT WORK WITHIN CHINA!!! + # podman pull k8s.gcr.io/sig-storage/nfs-subdir-external-provisioner:v4.0.2 + + # run the registry container + EXISTING_MIRROR_CONTAINER=$(podman ps --all --noheading | awk '$NF=="mirror-registry" {print $1}') + if [ "X${EXISTING_MIRROR_CONTAINER}" != "X" ]; then + podman rm -f "${EXISTING_MIRROR_CONTAINER}" + fi + podman run --name mirror-registry --publish "$MY_REGISTRY_PORT:5000" \ + --detach \ + --volume "${MY_MIRROR_DIR}/registry/data":/var/lib/registry:z \ + --volume "${MY_MIRROR_DIR}/registry/auth":/auth:z \ + --volume "${MY_MIRROR_DIR}/registry/certs":/certs:z \ + --env "REGISTRY_AUTH=htpasswd" \ + --env "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \ + --env REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \ + --env REGISTRY_HTTP_TLS_CERTIFICATE=/certs/registry.crt \ + --env REGISTRY_HTTP_TLS_KEY=/certs/registry.key \ + docker.io/library/registry:2 + + # copy the generated self-signed certs of the mirrored registry to the trust store + /usr/bin/cp -f "${MY_MIRROR_DIR}/registry/certs/registry.crt" /etc/pki/ca-trust/source/anchors/ + update-ca-trust + + # list the mirrored registry catalog to verify it's working + curl -u "$MY_REGISTRY_USER:$MY_REGISTRY_PASSWORD" "https://${MY_LOCAL_REGISTRY}/v2/_catalog" + + # generate the pull secret for the mirroed registry + AUTH=$(echo -n "$MY_REGISTRY_USER:$MY_REGISTRY_PASSWORD" | base64 -w0) + + printf '{"%s": {"auth":"%s", "email":"%s"}}\n' "$MY_LOCAL_REGISTRY" "$AUTH" "$MY_EMAIL" > /tmp/local_reg.json + + jq --argjson authinfo "$( "${MY_MIRROR_DIR}/ocp4_install/ocp_pullsecret.json" + + # now really doing the mirror + oc adm -a "${MY_LOCAL_SECRET_JSON}" release mirror \ + --from="quay.io/${MY_PRODUCT_REPO}/${MY_RELEASE_NAME}:${MY_OCP_RELEASE}-x86_64" \ + --to="${MY_LOCAL_REGISTRY}/${MY_LOCAL_REPOSITORY}" \ + --to-release-image="${MY_LOCAL_REGISTRY}/${MY_LOCAL_REPOSITORY}:${MY_OCP_RELEASE}" + + + # list the mirrored registry catalog again to verify it's working + curl -u "$MY_REGISTRY_USER:$MY_REGISTRY_PASSWORD" "https://${MY_LOCAL_REGISTRY}/v2/_catalog" + + # generate nginx config based on template and variables + cp /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bak.by.mirror.registry + envsubst < "${SCRIPT_ABS_PATH}/nginx.conf.tpl" > /etc/nginx/nginx.conf + + # make the dir part of the MIRROR_DIR under the nginx default root dir first + # so that link succeed + MY_NGINX_DEFAULT_DOC_ROOT="/usr/share/nginx/html" + MY_MIRROR_DIR_DIR_PART=$(dirname "$MY_MIRROR_DIR") + mkdir -p "$MY_NGINX_DEFAULT_DOC_ROOT/$MY_MIRROR_DIR_DIR_PART" + ln -s "${MY_MIRROR_DIR}" "MY_NGINX_DEFAULT_DOC_ROOT/${MY_MIRROR_DIR}" + systemctl restart nginx;systemctl enable nginx + + # check the http list entries + curl -L -s "http://${MY_REGISTRY_SERVER}:${MY_MIRROR_REGISTRY_HTTP_PORT}${MY_MIRROR_DIR}" --list-only + + # generate a systemd service the for mirror registry + #podman generate systemd mirror-registry -n > /etc/systemd/system/container-mirror-registry.service + #systemctl enable container-mirror-registry.service + #systemctl daemon-reload + + ;; + *) ;; + esac + +done_banner "Top level" "create an OCP mirror registry" + diff --git a/nginx.conf.tpl b/nginx.conf.tpl new file mode 100644 index 0000000..a559aec --- /dev/null +++ b/nginx.conf.tpl @@ -0,0 +1,97 @@ +# Template nginx config to use to generate the config for OCP mirror +# adapted from an original nginx stock config file + +# For more information on configuration, see: +# * Official English Documentation: http://nginx.org/en/docs/ +# * Official Russian Documentation: http://nginx.org/ru/docs/ + +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log; +pid /run/nginx.pid; + +# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic. +include /usr/share/nginx/modules/*.conf; + +events { + worker_connections 1024; +} + +http { + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Load modular configuration files from the /etc/nginx/conf.d directory. + # See http://nginx.org/en/docs/ngx_core_module.html#include + # for more information. + include /etc/nginx/conf.d/*.conf; + + server { + listen ${MIRROR_REGISTRY_HTTP_PORT} default_server; + listen [::]:${MIRROR_REGISTRY_HTTP_PORT} default_server; + server_name _; + root /usr/share/nginx/html; + + # Load configuration files for the default server block. + include /etc/nginx/default.d/*.conf; + + location / { + } + + location ${MIRROR_DIR} { + autoindex on; + } + + error_page 404 /404.html; + location = /40x.html { + } + + error_page 500 502 503 504 /50x.html; + location = /50x.html { + } + } + +# Settings for a TLS enabled server. +# +# server { +# listen 443 ssl http2 default_server; +# listen [::]:443 ssl http2 default_server; +# server_name _; +# root /usr/share/nginx/html; +# +# ssl_certificate "/etc/pki/nginx/server.crt"; +# ssl_certificate_key "/etc/pki/nginx/private/server.key"; +# ssl_session_cache shared:SSL:1m; +# ssl_session_timeout 10m; +# ssl_ciphers PROFILE=SYSTEM; +# ssl_prefer_server_ciphers on; +# +# # Load configuration files for the default server block. +# include /etc/nginx/default.d/*.conf; +# +# location / { +# } +# +# error_page 404 /404.html; +# location = /40x.html { +# } +# +# error_page 500 502 503 504 /50x.html; +# location = /50x.html { +# } +# } + +} + diff --git a/playbooks/ocp4.yaml b/playbooks/ocp4.yaml index f433a5a..4923352 100644 --- a/playbooks/ocp4.yaml +++ b/playbooks/ocp4.yaml @@ -78,6 +78,7 @@ tags: tftp - include: tasks/pxe_links.yaml when: rhcos_installation_method|upper=="PXE" + - include: tasks/coreos_installer_command.yaml - include: tasks/dns_server.yaml tags: dns - include: tasks/dns_interface.yaml diff --git a/playbooks/tasks/coreos_installer_command.j2 b/playbooks/tasks/coreos_installer_command.j2 new file mode 100644 index 0000000..a8dc95c --- /dev/null +++ b/playbooks/tasks/coreos_installer_command.j2 @@ -0,0 +1 @@ +sudo coreos-installer install /dev/sda --append-karg 'ip={{item}}::{{hostvars[item]['gateway']}}:255.255.255.0:{{hostvars[item]['host']}}.{{cluster_name}}.{{domain_name}}:{{node_interface_name}}:none nameserver={{groups['bastion'][0]}}' --ignition-url=http://{{groups['bastion'][0]}}:{{http_server_port}}/{{ocp_install_dir}}/{{node_type}}.ign --insecure-ignition diff --git a/playbooks/tasks/coreos_installer_command.yaml b/playbooks/tasks/coreos_installer_command.yaml new file mode 100644 index 0000000..13d55c1 --- /dev/null +++ b/playbooks/tasks/coreos_installer_command.yaml @@ -0,0 +1,36 @@ +--- +- name: Generate CoreOS installer command for bootstrap + template: + src: coreos_installer_command.j2 + dest: "{{ocp_install_dir}}/coreos_installer_command.{{hostvars[item]['host']}}" + owner: root + group: root + mode: 0644 + with_items: + - "{{groups['bootstrap']}}" + vars: + node_type: bootstrap + +- name: Generate CoreOS installer command for masters + template: + src: coreos_installer_command.j2 + dest: "{{ocp_install_dir}}/coreos_installer_command.{{hostvars[item]['host']}}" + owner: root + group: root + mode: 0644 + with_items: + - "{{groups['masters']}}" + vars: + node_type: master + +- name: Generate CoreOS installer command for workers + template: + src: coreos_installer_command.j2 + dest: "{{ocp_install_dir}}/coreos_installer_command.{{hostvars[item]['host']}}" + owner: root + group: root + mode: 0644 + with_items: + - "{{groups['workers']}}" + vars: + node_type: worker diff --git a/playbooks/tasks/dns_interface.yaml b/playbooks/tasks/dns_interface.yaml index ce6c06d..bed3c20 100644 --- a/playbooks/tasks/dns_interface.yaml +++ b/playbooks/tasks/dns_interface.yaml @@ -2,17 +2,39 @@ - name: Check if interface script exists stat: - path: "{{interface_script}}" + path: "{{bastion_interface_script}}" register: stat_result +- fail: + msg: "the network interface file {{bastion_interface_script}} not exist. Without it, the DNS config would not be correct." + when: not stat_result.stat.exists + - name: Change DNS service in interface script lineinfile: - path: "{{interface_script}}" + path: "{{bastion_interface_script}}" regexp: '^DNS1' line: "DNS1={{groups['bastion'][0]}}" when: stat_result.stat.exists==True register: interface_changed +- name: Check if interface boots from DHCP + shell: cat "{{bastion_interface_script}}" + when: stat_result.stat.exists + register: interface_boot_from_dhcp_stat_result + +- name: Check if dhclient config exists + stat: + path: "/etc/dhcp/dhclient.conf" + register: dhclient_conf_stat_result + +- name: Prepend local DNS server to dhclient if booting from DHCP + lineinfile: + path: "/etc/dhcp/dhclient.conf" + regexp: "^prepend*{{groups['bastion'][0]}}" + line: "prepend domain-name-servers {{groups['bastion'][0]}} ;" + when: dhclient_conf_stat_result.stat.exists==True and interface_boot_from_dhcp_stat_result.stdout.find('BOOTPROTO=dhcp') != -1 + register: interface_dhclient_changed + - name: Restart NetworkManager service service: name: NetworkManager diff --git a/playbooks/tasks/dns_server.j2 b/playbooks/tasks/dns_server.j2 index 0774b88..bc6009c 100644 --- a/playbooks/tasks/dns_server.j2 +++ b/playbooks/tasks/dns_server.j2 @@ -1,7 +1,9 @@ # {{ ansible_managed }} no-resolv -server={{external_name_servers[0]}} +{% for upstream_dns in external_name_servers %} +server={{upstream_dns}} +{% endfor %} local=/{{cluster_name}}.{{domain_name}}/ address=/api-int.{{cluster_name}}.{{domain_name}}/{{groups['lb'][0]}} @@ -9,30 +11,33 @@ address=/api.{{cluster_name}}.{{domain_name}}/{{groups['lb'][0]}} address=/.apps.{{cluster_name}}.{{domain_name}}/{{groups['lb'][0]}} {% if rhcos_installation_method|upper != "IPI" %} -address=/etcd-0.{{cluster_name}}.{{domain_name}}/{{groups['masters'][0]}} -address=/etcd-1.{{cluster_name}}.{{domain_name}}/{{groups['masters'][1]}} -address=/etcd-2.{{cluster_name}}.{{domain_name}}/{{groups['masters'][2]}} -srv-host=_etcd-server-ssl._tcp.{{cluster_name}}.{{domain_name}},etcd-0.{{cluster_name}}.{{domain_name}},2380,0,10 -srv-host=_etcd-server-ssl._tcp.{{cluster_name}}.{{domain_name}},etcd-1.{{cluster_name}}.{{domain_name}},2380,0,10 -srv-host=_etcd-server-ssl._tcp.{{cluster_name}}.{{domain_name}},etcd-2.{{cluster_name}}.{{domain_name}},2380,0,10 +{% for master in groups['masters'] %} +address=/etcd-{{loop.index - 1}}.{{cluster_name}}.{{domain_name}}/{{master}} +{% endfor %} +{% for master in groups['masters'] %} +srv-host=_etcd-server-ssl._tcp.{{cluster_name}}.{{domain_name}},etcd-{{loop.index - 1}}.{{cluster_name}}.{{domain_name}},2380,0,10 +{% endfor %} {% endif %} {% if dhcp_on_bastion %} domain={{cluster_name}}.{{domain_name}} -dhcp-range={{dhcp_range[0]}},{{dhcp_range[1]}},infinite -dhcp-option=3,{{default_gateway}} + +{% for tag in groups['dhcp'] | sort %} +dhcp-range=set:{{tag}},{{hostvars[tag]['begin']}},{{hostvars[tag]['end']}},infinite +dhcp-option=tag:{{tag}},3,{{hostvars[tag]['gateway']}} +{% endfor %} {% if rhcos_installation_method|upper!="IPI" %} {% for host in groups['masters'] | sort %} -dhcp-host={{hostvars[host]['mac']}},{{hostvars[host]['host']}},{{host}} +dhcp-host={{hostvars[host]['mac']}},{{hostvars[host]['host']}},{{host}},set:{{hostvars[host]['dhcp_sub_net']}} {% endfor %} {% for host in groups['workers'] | sort %} -dhcp-host={{hostvars[host]['mac']}},{{hostvars[host]['host']}},{{host}} +dhcp-host={{hostvars[host]['mac']}},{{hostvars[host]['host']}},{{host}},set:{{hostvars[host]['dhcp_sub_net']}} {% endfor %} {% for host in groups['bootstrap'] | sort %} -dhcp-host={{hostvars[host]['mac']}},{{hostvars[host]['host']}},{{host}} +dhcp-host={{hostvars[host]['mac']}},{{hostvars[host]['host']}},{{host}},set:{{hostvars[host]['dhcp_sub_net']}} {% endfor %} {% endif %} @@ -42,4 +47,4 @@ tftp-root=/tftpboot dhcp-boot=pxelinux.0 {% endif %} -{% endif %} \ No newline at end of file +{% endif %} diff --git a/playbooks/tasks/tftpboot_vm_menu.j2 b/playbooks/tasks/tftpboot_vm_menu.j2 index 042bd3e..0b1cef1 100644 --- a/playbooks/tasks/tftpboot_vm_menu.j2 +++ b/playbooks/tasks/tftpboot_vm_menu.j2 @@ -6,4 +6,4 @@ label linux menu label ^Install RHEL CoreOS menu default kernel /images/{{rhcos_kernel_package | basename}} - append initrd=/images/{{rhcos_initramfs_package | basename}} coreos.live.rootfs_url=http://{{groups['bastion'][0]}}:{{http_server_port}}/{{rhcos_rootfs_package}} rd.neednet=1 coreos.inst.install_dev=/dev/sda coreos.inst.ignition_url=http://{{groups['bastion'][0]}}:{{http_server_port}}/{{ocp_install_dir}}/{{node_type}}.ign ip={{item}}::{{default_gateway}}:255.255.255.0:{{hostvars[item]['host']}}.{{cluster_name}}.{{domain_name}}:ens192:none nameserver={{groups['bastion'][0]}} \ No newline at end of file + append initrd=/images/{{rhcos_initramfs_package | basename}} coreos.live.rootfs_url=http://{{groups['bastion'][0]}}:{{http_server_port}}/{{rhcos_rootfs_package}} rd.neednet=1 coreos.inst.install_dev=/dev/sda coreos.inst.ignition_url=http://{{groups['bastion'][0]}}:{{http_server_port}}/{{ocp_install_dir}}/{{node_type}}.ign ip={{item}}::{{hostvars[item]['gateway']}}:255.255.255.0:{{hostvars[item]['host']}}.{{cluster_name}}.{{domain_name}}:{{node_interface_name}}:none nameserver={{groups['bastion'][0]}} diff --git a/prepare.sh b/prepare.sh index 069c1cd..d4dc1d5 100755 --- a/prepare.sh +++ b/prepare.sh @@ -72,6 +72,36 @@ pushd $SCRIPT_DIR > /dev/null # Run ansible playbook inventory_file=$(realpath $INVENTORY_FILE_PARAM) +# Utilities function +my_exit () { + echo "EXIT: - [HOST:$(hostname)]: - $(date +"%Y-%m-%d %H:%M:%S") - $1" + exit "$2" +} + +# init the bastion machine +"$SCRIPT_DIR"/init_bastion.sh || my_exit "failed to initialize the bostion machine" 224 + +# add airgap install support if air_gapped_install=True +is_airgap_install=$(grep 'air_gapped_install=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}'|tr -d '"') +if [ "X$is_airgap_install" == "XTrue" ]; then + domain_name=$(grep 'domain_nam=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}'|tr -d '"') + air_gapped_registry_server=$(grep 'air_gapped_registry_server=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}'|tr -d '"') + air_gapped_download_dir=$(grep 'air_gapped_download_dir=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}'|tr -d '"') + http_server_port=$(grep 'http_server_port=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}'|tr -d '"') + openshift_release=$(grep 'openshift_release=' "$inventory_file"|grep -v '#'|awk -F'=' '{print $2}'|tr -d '"') + pull_secret_file=$pull_secret_file \ + domain_name=$domain_name \ + openshift_release=$openshift_release \ + air_gapped_registry_server=$air_gapped_registry_server \ + air_gapped_download_dir=$air_gapped_download_dir \ + http_server_port=$http_server_port \ + "$SCRIPT_DIR/mirror_ocp.sh" \ + || my_exit "failed to create the OCP mirror" 224 + # replace the pull_secret file with the new one which includes the mirror registry + echo "mirrored created, using the new pull_secret file: ${air_gapped_download_dir}/ocp4_install/ocp_pullsecret.json" + export pull_secret_file="${air_gapped_download_dir}/ocp4_install/ocp_pullsecret.json" +fi + ansible-playbook -i $inventory_file playbooks/ocp4.yaml \ -e ansible_ssh_pass=$root_password \ -e ocp_admin_password=$ocp_admin_password \