From 868364da02b7162fa0aae45d5cc42a251bd50b56 Mon Sep 17 00:00:00 2001 From: wangzheng422 Date: Thu, 12 Oct 2023 15:31:22 +0800 Subject: [PATCH] ocp --- redhat/notes/2023/2023.10.qcow2.modify.md | 42 + .../ocp4/4.13/4.13.helper.node.oc.mirror.md | 859 ++++++++++++++++++ redhat/ocp4/4.13/4.13.pdns.helper.md | 165 ++++ 3 files changed, 1066 insertions(+) create mode 100644 redhat/notes/2023/2023.10.qcow2.modify.md create mode 100644 redhat/ocp4/4.13/4.13.helper.node.oc.mirror.md create mode 100644 redhat/ocp4/4.13/4.13.pdns.helper.md diff --git a/redhat/notes/2023/2023.10.qcow2.modify.md b/redhat/notes/2023/2023.10.qcow2.modify.md new file mode 100644 index 00000000..8c8f33eb --- /dev/null +++ b/redhat/notes/2023/2023.10.qcow2.modify.md @@ -0,0 +1,42 @@ +# modify qcow2 + +```bash +# https://platform9.com/blog/how-to-create-customized-qcow-images/ +qemu-img info rhcos-openstack.x86_64.qcow2 +# image: rhcos-openstack.x86_64.qcow2 +# file format: qcow2 +# virtual size: 16 GiB (17179869184 bytes) +# disk size: 2.46 GiB +# cluster_size: 65536 +# Format specific information: +# compat: 1.1 +# compression type: zlib +# lazy refcounts: false +# refcount bits: 16 +# corrupt: false +# extended l2: false + +virt-filesystems --long -h -a rhcos-openstack.x86_64.qcow2 +# Name Type VFS Label Size Parent +# /dev/sda2 filesystem vfat EFI-SYSTEM 127M - +# /dev/sda3 filesystem ext4 boot 350M - +# /dev/sda4 filesystem xfs root 3.3G - + + +# https://docs.openstack.org/image-guide/modify-images.html +modprobe nbd max_part=16 + +qemu-nbd -c /dev/nbd0 rhcos-openstack.x86_64.qcow2 + +partprobe /dev/nbd0 + +ls -l /dev/nbd0* +# brw-rw----. 1 root disk 43, 0 Oct 12 13:55 /dev/nbd0 +# brw-rw----. 1 root disk 43, 1 Oct 12 13:55 /dev/nbd0p1 +# brw-rw----. 1 root disk 43, 2 Oct 12 13:55 /dev/nbd0p2 +# brw-rw----. 1 root disk 43, 3 Oct 12 13:55 /dev/nbd0p3 +# brw-rw----. 1 root disk 43, 4 Oct 12 13:55 /dev/nbd0p4 + +mount /dev/nbd0p2 /mnt/2 + +``` \ No newline at end of file diff --git a/redhat/ocp4/4.13/4.13.helper.node.oc.mirror.md b/redhat/ocp4/4.13/4.13.helper.node.oc.mirror.md new file mode 100644 index 00000000..735f3661 --- /dev/null +++ b/redhat/ocp4/4.13/4.13.helper.node.oc.mirror.md @@ -0,0 +1,859 @@ +# helper node created + +# init setting for helper node + +```bash +ssh -tt -D 8801 -R 18801:10.147.17.89:5085 root@172.21.6.11 'bash -l -c byobu' + +# init setting for helper node +cat << EOF > ~/.ssh/config +StrictHostKeyChecking no +UserKnownHostsFile=/dev/null +EOF + +chmod 600 ~/.ssh/config + +cat << EOF >> /etc/hosts +127.0.0.1 registry.ocp4.redhat.ren nexus.ocp4.redhat.ren git.ocp4.redhat.ren quaylab.infra.wzhlab.top +EOF + +dnf -y install byobu htop jq ipmitool nmstate + +systemctl disable --now firewalld + +dnf groupinstall -y development server 'server with gui' + +dnf -y install qemu-kvm libvirt libguestfs-tools virt-install virt-viewer virt-manager tigervnc-server + +systemctl enable --now libvirtd + +# create thin pool +pvcreate -y /dev/sdb +vgcreate vgdata /dev/sdb + +# https://access.redhat.com/articles/766133 +lvcreate -y -n poolA -L 500G vgdata +lvcreate -y -n poolA_meta -L 1G vgdata +lvconvert -y --thinpool vgdata/poolA --poolmetadata vgdata/poolA_meta + # Thin pool volume with chunk size 64.00 KiB can address at most <15.88 TiB of data. + # WARNING: Converting vgdata/poolA and vgdata/poolA_meta to thin pool's data and metadata volumes with metadata wiping. + # THIS WILL DESTROY CONTENT OF LOGICAL VOLUME (filesystem etc.) + # Converted vgdata/poolA and vgdata/poolA_meta to thin pool. + +lvextend -l +100%FREE vgdata/poolA + # Rounding size to boundary between physical extents: <1.09 GiB. + # Size of logical volume vgdata/poolA_tmeta changed from 1.00 GiB (256 extents) to <1.09 GiB (279 extents). + # Size of logical volume vgdata/poolA_tdata changed from 500.00 GiB (128000 extents) to <1.09 TiB (285457 extents). + # Logical volume vgdata/poolA successfully resized. + +``` + +# setup ntp server on helper node + +```bash +# setup ntp server on helper node +# sed -i "s/#allow.*/allow 192.168.0.0\/16/" /etc/chrony.conf +sed -i "s/#allow.*/allow all/" /etc/chrony.conf +systemctl enable --now chronyd + +chronyc tracking +# Reference ID : CA760182 (202.118.1.130) +# Stratum : 2 +# Ref time (UTC) : Mon May 02 03:55:48 2022 +# System time : 0.000000530 seconds fast of NTP time +# Last offset : -0.003027542 seconds +# RMS offset : 0.003027542 seconds +# Frequency : 36.009 ppm slow +# Residual freq : +61.371 ppm +# Skew : 25.290 ppm +# Root delay : 0.016805194 seconds +# Root dispersion : 0.002184978 seconds +# Update interval : 0.8 seconds +# Leap status : Normal + +chronyc sources -v +# .-- Source mode '^' = server, '=' = peer, '#' = local clock. +# / .- Source state '*' = current best, '+' = combined, '-' = not combined, +# | / 'x' = may be in error, '~' = too variable, '?' = unusable. +# || .- xxxx [ yyyy ] +/- zzzz +# || Reachability register (octal) -. | xxxx = adjusted offset, +# || Log2(Polling interval) --. | | yyyy = measured offset, +# || \ | | zzzz = estimated error. +# || | | \ +# MS Name/IP address Stratum Poll Reach LastRx Last sample +# =============================================================================== +# ^- time.cloudflare.com 3 6 17 41 +28ms[ +28ms] +/- 157ms +# ^* 202.118.1.130 1 6 17 42 -9871ns[-3037us] +/- 8572us +# ^- time.cloudflare.com 3 6 17 40 +35ms[ +35ms] +/- 162ms +# ^- makaki.miuku.net 2 6 17 40 +46ms[ +46ms] +/- 110ms + +chronyc sourcestats -v +# .- Number of sample points in measurement set. +# / .- Number of residual runs with same sign. +# | / .- Length of measurement set (time). +# | | / .- Est. clock freq error (ppm). +# | | | / .- Est. error in freq. +# | | | | / .- Est. offset. +# | | | | | | On the -. +# | | | | | | samples. \ +# | | | | | | | +# Name/IP Address NP NR Span Frequency Freq Skew Offset Std Dev +# ============================================================================== +# time.cloudflare.com 4 3 7 -1249.574 60710.363 -54ms 8100us +# 202.118.1.130 4 3 6 +61.371 5439.969 +3713us 581us +# time.cloudflare.com 4 3 7 -3223.009 204771 -185ms 29ms +# makaki.miuku.net 4 3 7 +6244.955 92563.305 +411ms 12ms + +chronyc makestep +# 200 OK +``` + +# setup network + +```bash +cat << 'EOF' > /data/kvm/bridge.sh +#!/usr/bin/env bash + +PUB_CONN='ens192' +PUB_IP='172.21.6.11/24' +PUB_GW='172.21.6.254' +PUB_DNS='172.21.1.1' + +nmcli con down "$PUB_CONN" +nmcli con delete "$PUB_CONN" +nmcli con down baremetal +nmcli con delete baremetal +# RHEL 8.1 appends the word "System" in front of the connection,delete in case it exists +nmcli con down "System $PUB_CONN" +nmcli con delete "System $PUB_CONN" +nmcli connection add ifname baremetal type bridge con-name baremetal ipv4.method 'manual' \ + ipv4.address "$PUB_IP" \ + ipv4.gateway "$PUB_GW" \ + ipv4.dns "$PUB_DNS" + +nmcli con add type bridge-slave ifname "$PUB_CONN" master baremetal +nmcli con down "$PUB_CONN";pkill dhclient;dhclient baremetal +nmcli con up baremetal +EOF + +bash /data/kvm/bridge.sh + +nmcli con mod baremetal -ipv4.address '192.168.7.11/24' +nmcli con mod baremetal +ipv4.address '192.168.77.11/24' +nmcli con mod baremetal ipv6.method manual +ipv6.address 'fd03::11/64' + +nmcli con up baremetal +# nmcli networking off; nmcli networking on +``` + +# set cert key + +```bash +mkdir -p /etc/crts/ && cd /etc/crts + +# https://access.redhat.com/documentation/en-us/red_hat_codeready_workspaces/2.1/html/installation_guide/installing-codeready-workspaces-in-tls-mode-with-self-signed-certificates_crw +openssl genrsa -out /etc/crts/wzhlab.top.ca.key 4096 + +openssl req -x509 \ + -new -nodes \ + -key /etc/crts/wzhlab.top.ca.key \ + -sha256 \ + -days 36500 \ + -out /etc/crts/wzhlab.top.ca.crt \ + -subj /CN="Local wzh lab Signer" \ + -reqexts SAN \ + -extensions SAN \ + -config <(cat /etc/pki/tls/openssl.cnf \ + <(printf '[SAN]\nbasicConstraints=critical, CA:TRUE\nkeyUsage=keyCertSign, cRLSign, digitalSignature')) + +openssl genrsa -out /etc/crts/wzhlab.top.key 2048 + +openssl req -new -sha256 \ + -key /etc/crts/wzhlab.top.key \ + -subj "/O=Local wzh lab /CN=*.infra.wzhlab.top" \ + -reqexts SAN \ + -config <(cat /etc/pki/tls/openssl.cnf \ + <(printf "\n[SAN]\nsubjectAltName=DNS:*.infra.wzhlab.top,DNS:*.wzhlab.top\nbasicConstraints=critical, CA:FALSE\nkeyUsage=digitalSignature, keyEncipherment, keyAgreement, dataEncipherment\nextendedKeyUsage=serverAuth")) \ + -out /etc/crts/wzhlab.top.csr + +openssl x509 \ + -req \ + -sha256 \ + -extfile <(printf "subjectAltName=DNS:*.infra.wzhlab.top,DNS:*.wzhlab.top\nbasicConstraints=critical, CA:FALSE\nkeyUsage=digitalSignature, keyEncipherment, keyAgreement, dataEncipherment\nextendedKeyUsage=serverAuth") \ + -days 36500 \ + -in /etc/crts/wzhlab.top.csr \ + -CA /etc/crts/wzhlab.top.ca.crt \ + -CAkey /etc/crts/wzhlab.top.ca.key \ + -CAcreateserial -out /etc/crts/wzhlab.top.crt + +openssl x509 -in /etc/crts/wzhlab.top.crt -text + +/bin/cp -f /etc/crts/wzhlab.top.ca.crt /etc/pki/ca-trust/source/anchors/ +update-ca-trust extract + +``` + +# 下载需要的 openshift4 软件 + +```bash +dnf -y install git +cd /data +rm -rf /data/ocp4 +# scripts can be found here: +# https://github.com/wangzheng422/openshift4-shell +# bash helper.node.sh -v 4.10.12 -m 4.10 -f file + +# on vultr +cd /data +bash helper.node.client.sh -v 4.12.9 -m 4.12 + +# cd /data +# bash helper.node.sh -v 4.12.9 -m 4.12 -f file + +# on helper +cd /data +bash helper.node.ansible.sh -m 4.12 + +mkdir -p /data/ocp4/clients +cd /data/ocp4/clients +rsync -P -arz -e "ssh -J root@zerotier.wzhlab.top" 'root@66.42.107.212:/data/ocp4/clients/*' ./ + +install /data/ocp4/clients/butane-amd64 /usr/local/bin/butane +install /data/ocp4/clients/coreos-installer_amd64 /usr/local/bin/coreos-installer + +export BUILDNUMBER=4.12.9 +mkdir -p /data/ocp-$BUILDNUMBER +cd /data/ocp-$BUILDNUMBER +rsync -P -arz -e "ssh -J root@zerotier.wzhlab.top" "root@66.42.107.212:/data/ocp-$BUILDNUMBER/*" ./ + +tar -xzf openshift-client-linux-${BUILDNUMBER}.tar.gz -C /usr/local/bin/ +tar -xzf openshift-install-linux-${BUILDNUMBER}.tar.gz -C /usr/local/bin/ +tar -xzf oc-mirror.tar.gz -C /usr/local/bin/ +chmod +x /usr/local/bin/oc-mirror + +``` + +# setup dns + +```bash +# copy helper ansible project to /data/ocp4/ocp4-upi-helpernode-master + +yum -y install ansible git unzip podman python3 + +mkdir -p /data/ocp4/ocp4-upi-helpernode-master + +mkdir -p /data/sno + +NODE_SSH_KEY="$(cat ~/.ssh/id_rsa.pub)" +INSTALL_IMAGE_REGISTRY=quaylab.infra.redhat.ren:8443 + +PULL_SECRET='{"auths":{"registry.redhat.io": {"auth": "ZHVtbXk6ZHVtbXk=","email": "noemail@localhost"},"registry.ocp4.redhat.ren:5443": {"auth": "ZHVtbXk6ZHVtbXk=","email": "noemail@localhost"},"'${INSTALL_IMAGE_REGISTRY}'": {"auth": "'$( echo -n 'admin:shadowman' | openssl base64 )'","email": "noemail@localhost"}}}' + +NTP_SERVER=192.168.7.11 +HELP_SERVER=192.168.7.11 +KVM_HOST=192.168.7.11 +API_VIP=192.168.7.100 +INGRESS_VIP=192.168.7.101 +CLUSTER_PROVISION_IP=192.168.7.103 +BOOTSTRAP_IP=192.168.7.12 + +ACM_DEMO_MNGED_CLUSTER=acm-demo-man01 +ACM_DEMO_MNGED_SNO_IP=192.168.7.23 + +echo $PULL_SECRET + +# 定义单节点集群的节点信息 +SNO_CLUSTER_NAME=acm-demo-hub +SNO_BASE_DOMAIN=redhat.ren +SNO_IP=192.168.7.13 +SNO_GW=192.168.7.11 +SNO_NETMAST=255.255.255.0 +SNO_NETMAST_S=24 +SNO_HOSTNAME=acm-demo-hub-master +SNO_IF=enp1s0 +SNO_IF_MAC=`printf '00:60:2F:%02X:%02X:%02X' $[RANDOM%256] $[RANDOM%256] $[RANDOM%256]` +SNO_DNS=192.168.7.11 +SNO_DISK=/dev/vda +SNO_CORE_PWD=redhat + +# echo ${SNO_IF_MAC} > /data/sno/sno.mac + +cd /data/ocp4/ansible-helper + +cat > var.yaml << EOF +helper: + ip_addr: 192.168.7.11 + nic: baremetal +pdns: + bind: 0.0.0.0 + port: 5301 + recursor_port: 53 + forward: 172.21.1.1 + static: + - base_domain: infra.redhat.ren + record: + - name: registry + ip_addr: 192.168.7.11 + - name: nexus + ip_addr: 192.168.7.11 + - name: quay + ip_addr: 192.168.7.11 + - name: quaylab + ip_addr: 192.168.7.11 +ntp: + server: 192.168.7.11 +cluster: + - base_domain: acm-demo-hub.redhat.ren + node: + # - ip_addr: 192.168.7.12 + - ip_addr: 192.168.7.13 + name: sno-master-01 + # - ip_addr: 192.168.7.14 + # - ip_addr: 192.168.7.15 + - base_domain: acm-demo-one.redhat.ren + node: + - ip_addr: 192.168.7.22 + name: one-bootstrap + - ip_addr: 192.168.7.23 + name: one-master-01 + - ip_addr: 192.168.7.24 + name: one-master-02 + - ip_addr: 192.168.7.25 + name: one-master-03 + - base_domain: acm-demo-two.redhat.ren + node: + - ip_addr: 192.168.7.33 + name: two-bootstrap + - ip_addr: 192.168.7.34 + name: two-master-01 + - ip_addr: 192.168.7.35 + name: two-master-02 + - ip_addr: 192.168.7.36 + name: two-master-03 +ptr: + - addr: 192.168.7 + domain: ptr.redhat.ren +EOF + +cd /data/ocp4/ansible-helper +ansible-playbook -e @var.yaml helper.yaml + + +# dig @127.0.0.1 -p 5301 -x 192.168.7.23 + +``` + +# 配置quay镜像仓库 + +## using official version +```bash +# https://docs.openshift.com/container-platform/4.10/installing/disconnected_install/installing-mirroring-creating-registry.html + +mkdir -p /data/quay +cd /data/ocp4/clients +tar zvxf mirror-registry.tar.gz +./mirror-registry install -v \ + --initPassword redhatadmin --initUser admin \ + --quayHostname quaylab.infra.wzhlab.top --quayRoot /data/quay \ + --targetHostname quaylab.infra.wzhlab.top \ + --sslKey /etc/crts/infra.wzhlab.top.key --sslCert /etc/crts/infra.wzhlab.top.crt +# PLAY RECAP ******************************************************************************************************************************************************************************************$root@quaylab.infra.redhat.ren : ok=52 changed=20 unreachable=0 failed=0 skipped=7 rescued=0 ignored=0 + +# INFO[2022-05-02 18:01:10] Quay installed successfully, permanent data is stored in /data/quay +# INFO[2022-05-02 18:01:10] Quay is available at https://quaylab.infra.redhat.ren:8443 with credentials (admin, shadowman) + +ls -hl /data/quay +# total 4.1G +# -rw-r--r--. 1 root root 2.1G May 2 17:28 image-archive.tar +# -rw-r--r--. 1 root root 3.4M Mar 9 21:52 pause.tar +# drwxrwxr-x+ 3 root root 22 May 2 17:29 pg-data +# -rw-r--r--. 1 root root 585M Mar 9 21:54 postgres.tar +# drwxr-xr-x. 2 root root 90 May 2 18:00 quay-config +# drwxr-xr-x. 2 root root 60 May 2 17:29 quay-rootCA +# drwxrwxr-x+ 2 root root 6 May 2 17:29 quay-storage +# -rw-r--r--. 1 root root 1.1G Mar 9 21:53 quay.tar +# -rw-r--r--. 1 root root 430M Mar 9 21:54 redis.tar + +# to uninstall, do not use in setup +./mirror-registry uninstall -v \ + --autoApprove true --quayRoot /data/quay \ + --targetHostname quaylab.infra.wzhlab.top \ + -- + +# https://quaylab.infra.redhat.ren:8443/ +``` + +## using upstream + +```bash +# https://github.com/quay/mirror-registry/releases/download/v1.3.2/mirror-registry-online.tar.gz + +# rm -rf /data/quay +mkdir -p /data/quay +cd /data/swap + +export http_proxy="http://127.0.0.1:18801" +export https_proxy=${http_proxy} + +wget -O mirror-registry-offline.tar.gz https://github.com/quay/mirror-registry/releases/download/v1.3.2/mirror-registry-offline.tar.gz + +unset http_proxy +unset https_proxy + +tar zvxf mirror-registry-offline.tar.gz + +./mirror-registry install -v \ + --initPassword redhatadmin --initUser admin \ + --quayHostname quaylab.infra.wzhlab.top --quayRoot /data/quay \ + --targetHostname quaylab.infra.wzhlab.top \ + --sslKey /etc/crts/infra.wzhlab.top.key --sslCert /etc/crts/infra.wzhlab.top.crt +# PLAY RECAP ****************************************************************************************************************************************************************root@quaylab.infra.redhat.ren : ok=39 changed=21 unreachable=0 failed=0 skipped=20 rescued=0 ignored=0 + +# INFO[2022-12-13 13:55:24] Quay installed successfully, permanent data is stored in /data/quay +# INFO[2022-12-13 13:55:24] Quay is available at https://quaylab.infra.redhat.ren:8443 with credentials (admin, redhatadmin) + +# to uninstall, do not use in setup +./mirror-registry uninstall -v \ + --autoApprove true --quayRoot /data/quay \ + --targetHostname quaylab.infra.wzhlab.top \ + -- + + +ls -hl /data/quay +# total 0 +# drwxrwxr-x+ 3 root root 22 Dec 10 00:08 pg-data +# drwxr-xr-x. 2 root root 56 Dec 10 00:08 quay-config +# drwxrwxr-x+ 2 root root 6 Dec 10 00:08 quay-storage + +cat << EOF >> /data/quay/quay-config/config.yaml + +FEATURE_PROXY_CACHE: true +BROWSER_API_CALLS_XHR_ONLY: false +FEATURE_QUOTA_MANAGEMENT: false + +EOF + +# sed -i 's/DEFAULT_TAG_EXPIRATION:.*/DEFAULT_TAG_EXPIRATION: 4w/' /data/quay/quay-config/config.yaml + +cat << EOF > /etc/systemd/system/quay-app.service +[Unit] +Description=Quay Container +Wants=network.target +After=network-online.target quay-pod.service quay-postgres.service quay-redis.service +Requires=quay-pod.service quay-postgres.service quay-redis.service + +[Service] +Type=simple +TimeoutStartSec=5m +ExecStartPre=-/bin/rm -f %t/%n-pid %t/%n-cid + +# ExecStart=/usr/bin/podman run \ +# --name quay-app \ +# -v /data/quay/quay-config:/quay-registry/conf/stack:Z \ +# -v /data/quay/quay-storage:/datastorage:Z \ +# --pod=quay-pod \ +# --conmon-pidfile %t/%n-pid \ +# --cidfile %t/%n-cid \ +# --cgroups=no-conmon \ +# --replace \ +# -e HTTP_PROXY=http://192.168.77.11:18801 \ +# -e HTTPS_PROXY=http://192.168.77.11:18801 \ +# -e NO_PROXY=localhost,127.0.0.1,10.1.0.0/16,172.30.0.0/16,192.168.0.0/16 \ +# registry.redhat.io/quay/quay-rhel8:v3.7.11 + +ExecStart=/usr/bin/podman run \ + --name quay-app \ + -v /data/quay/quay-config:/quay-registry/conf/stack:Z \ + -v /data/quay/quay-storage:/datastorage:Z \ + --pod=quay-pod \ + --conmon-pidfile %t/%n-pid \ + --cidfile %t/%n-cid \ + --cgroups=no-conmon \ + --replace \ + registry.redhat.io/quay/quay-rhel8:v3.7.11 + +ExecStop=-/usr/bin/podman stop --ignore --cidfile %t/%n-cid -t 10 +ExecStopPost=-/usr/bin/podman rm --ignore -f --cidfile %t/%n-cid +PIDFile=%t/%n-pid +KillMode=none +Restart=always +RestartSec=30 + +[Install] +WantedBy=multi-user.target default.target +EOF + +systemctl daemon-reload + +systemctl restart quay-app + +``` + +## using docker registry + +```bash + +/bin/rm -rf /data/registry +mkdir -p /data/registry + +podman run --restart always --name local-registry -p 5443:5443 \ + -d --restart=always \ + -v /data/registry/:/var/lib/registry:z \ + -v /etc/crts:/certs:z \ + -e REGISTRY_HTTP_ADDR=0.0.0.0:5443 \ + -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/wzhlab.top.crt \ + -e REGISTRY_HTTP_TLS_KEY=/certs/wzhlab.top.key \ + docker.io/library/registry:2 + +podman run --restart always --name local-registry -p 5443:5443 \ + -d --restart=always \ + -v /data/registry/:/var/lib/registry:z \ + -v /etc/crts:/certs:z \ + -e REGISTRY_HTTP_ADDR=0.0.0.0:5443 \ + -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/infra.wzhlab.top.crt \ + -e REGISTRY_HTTP_TLS_KEY=/certs/infra.wzhlab.top.key \ + docker.io/library/registry:2 + +podman generate systemd --files --name local-registry +# /root/container-local-registry.service +/bin/cp -Zf container-local-registry.service /etc/systemd/system/ + +systemctl daemon-reload + +systemctl enable --now container-local-registry.service +systemctl status container-local-registry.service + + + +cat << EOF > /etc/systemd/system/container-local-registry.service +[Unit] +Description=local-registry +Wants=network.target +After=network-online.target + +[Service] +Type=simple +TimeoutStartSec=5m +ExecStartPre=-/bin/rm -f %t/%n-pid %t/%n-cid +ExecStart=/usr/bin/podman run --name local-registry -p 5443:5443 \ + -v /data/registry/:/var/lib/registry:z \ + -v /etc/crts:/certs:z \ + --conmon-pidfile %t/%n-pid \ + --cidfile %t/%n-cid \ + --replace \ + --privileged \ + -e REGISTRY_HTTP_ADDR=0.0.0.0:5443 \ + -e REGISTRY_HTTP_TLS_CERTIFICATE=/certs/infra.wzhlab.top.crt \ + -e REGISTRY_HTTP_TLS_KEY=/certs/infra.wzhlab.top.key \ + docker.io/library/registry:2 + +ExecStop=-/usr/bin/podman stop --ignore --cidfile %t/%n-cid -t 10 +ExecStopPost=-/usr/bin/podman rm --ignore -f --cidfile %t/%n-cid +PIDFile=%t/%n-pid +KillMode=none +Restart=always +RestartSec=30 + +[Install] +WantedBy=multi-user.target default.target +EOF + +systemctl daemon-reload + +systemctl restart container-local-registry + + +``` + +# download client binary + +```bash + +export BUILDNUMBER=4.12.9 + +mkdir -p /data/ocp-${BUILDNUMBER} +cd /data/ocp-${BUILDNUMBER} + +wget -O openshift-client-linux-${BUILDNUMBER}.tar.gz https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/${BUILDNUMBER}/openshift-client-linux-${BUILDNUMBER}.tar.gz +wget -O openshift-install-linux-${BUILDNUMBER}.tar.gz https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/${BUILDNUMBER}/openshift-install-linux-${BUILDNUMBER}.tar.gz +wget -O oc-mirror.tar.gz https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/${BUILDNUMBER}/oc-mirror.tar.gz + +tar -xzf openshift-client-linux-${BUILDNUMBER}.tar.gz -C /usr/local/bin/ +tar -xzf openshift-install-linux-${BUILDNUMBER}.tar.gz -C /usr/local/bin/ + +wget -O oc-mirror.tar.gz https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/${BUILDNUMBER}/oc-mirror.tar.gz +tar -xzf oc-mirror.tar.gz -C /usr/local/bin/ +chmod +x /usr/local/bin/oc-mirror + +wget -O rhcos-live.x86_64.iso https://mirror.openshift.com/pub/openshift-v4/x86_64/dependencies/rhcos/${BUILDNUMBER%.*}/latest/rhcos-live.x86_64.iso + +``` + +# import ocp content to docker registry + +```bash +export BUILDNUMBER=4.12.9 + +oc adm release mirror -a $SEC_FILE \ + --from=quay.io/openshift-release-dev/ocp-release:$BUILDNUMBER-x86_64 \ + --to=quaylab.infra.wzhlab.top:5443/ocp4/openshift4 + +``` + +# import ocp content into quay + +```bash + +export BUILDNUMBER=4.11.19 + +pushd /data/ocp4/${BUILDNUMBER} +tar -xzf openshift-client-linux-${BUILDNUMBER}.tar.gz -C /usr/local/bin/ +tar -xzf openshift-install-linux-${BUILDNUMBER}.tar.gz -C /usr/local/bin/ +tar -xzf oc-mirror.tar.gz -C /usr/local/bin/ +chmod +x /usr/local/bin/oc-mirror +install -m 755 /data/ocp4/clients/butane-amd64 /usr/local/bin/butane +install -m 755 /data/ocp4/clients/coreos-installer_amd64 /usr/local/bin/coreos-installer +popd + +# cd /data/swap +# wget -O oc-mirror.tar.gz https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp/${BUILDNUMBER}/oc-mirror.tar.gz +# tar -xzf oc-mirror.tar.gz -C /usr/local/bin/ +# chmod +x /usr/local/bin/oc-mirror + + +SEC_FILE="$XDG_RUNTIME_DIR/containers/auth.json" +# $XDG_RUNTIME_DIR/containers +# echo $XDG_RUNTIME_DIR +# /run/user/0 +# export XDG_RUNTIME_DIR=/run/user/0 +mkdir -p ${SEC_FILE%/*} + +# copy the password file +# vi $SEC_FILE + +# OR +# vi ~/.docker/config.json +SEC_FILE="$HOME/.docker/config.json" +mkdir -p ${SEC_FILE%/*} + +# podman login quaylab.infra.wzhlab.top:8443 --username admin --password redhatadmin + +# oc mirror init --registry quaylab.infra.redhat.ren:8443/mirror/oc-mirror-metadata > /data/ocp4/mirror.yaml + +cat > /data/ocp4/mirror.yaml << EOF +apiVersion: mirror.openshift.io/v1alpha2 +kind: ImageSetConfiguration +# archiveSize: 4 +mirror: + platform: + architectures: + - amd64 + # - arm64 + channels: + - name: stable-4.11 + type: ocp + minVersion: 4.11.21 + maxVersion: 4.11.21 + shortestPath: true + # - name: stable-4.10 + # type: ocp + # minVersion: 4.10.45 + # maxVersion: 4.10.45 + # shortestPath: true + graph: false + # additionalImages: + # - name: registry.redhat.io/redhat/redhat-operator-index:v4.10 + # - name: registry.redhat.io/redhat/certified-operator-index:v4.10 + # - name: registry.redhat.io/redhat/community-operator-index:v4.10 + # - name: registry.redhat.io/redhat/redhat-marketplace-index:v4.10 +EOF + +mkdir -p /data/install/mirror-tmp +cd /data/install/mirror-tmp + +oc-mirror --config /data/ocp4/mirror.yaml docker://quaylab.infra.wzhlab.top:8443 + +``` + +注意，默认情况下，他会创建如下几个repo, quay默认是私有的，我们要手动把他们搞成public的。 +- openshift/release +- operator-framework/opm (这个其实没啥用) + +![](imgs/2022-09-07-10-00-26.png) + +# setup nexus + +```bash +## import nexus fs +mkdir -p /data/ccn +cd /data/ccn + +podman create --name swap quay.io/wangzheng422/qimgs:nexus-fs-image-2022-01-14-2155 ls +podman cp swap:/nexus-image.tgz - > /data/ccn/nexus-image.tgz.tar +podman rm -fv swap +tar vxf nexus-image.tgz.tar +tar zxf nexus-image.tgz +rm -f nexus-image.tgz* + +chown -R 200 /data/ccn/nexus-image + +## run the nexus for image +podman run -d -p 8082:8081 -p 8083:8083 -it --name nexus-image -v /data/ccn/nexus-image:/nexus-data:Z docker.io/sonatype/nexus3:3.38.1 +# podman run -d -p 8082:8081 -p 8083:8083 -it --name nexus-image --privileged --cap-add all -v /home/ccn/nexus-image:/nexus-data:Z docker.io/sonatype/nexus3:3.38.1 +# podman run -d -p 8082:8081 -p 8083:8083 -it --name nexus-image -v /data/ccn/nexus-image:/nexus-data:Z docker.io/sonatype/nexus3:3.33.1 + +podman generate systemd --files --name nexus-image +# /root/container-local-registry.service +/bin/cp -Zf container-nexus-image.service /etc/systemd/system/ + +systemctl daemon-reload + +systemctl enable --now container-nexus-image.service +systemctl status container-nexus-image.service + +# get the admin password +cat /data/ccn/nexus-image/admin.password && echo +# 84091bcd-c82f-44a3-8b7b-dfc90f5b7da1 + +# open http://nexus.infra.redhat.ren:8082 +``` + +# setup assisted install service ( AIS ) + +```bash + +# https://github.com/openshift/assisted-service/blob/master/docs/user-guide/assisted-service-on-local.md + +# https://github.com/openshift/assisted-service/tree/master/deploy/podman + +podman version +# Version: 3.4.2 +# API Version: 3.4.2 +# Go Version: go1.16.12 +# Built: Wed Feb 2 07:59:28 2022 +# OS/Arch: linux/amd64 + +/bin/cp -f /data/ocp4/rhcos-live.x86_64.iso /var/www/html/install/ + +mkdir -p /data/assisted-service/ +cd /data/assisted-service/ + +export http_proxy="http://127.0.0.1:18801" +export https_proxy=${http_proxy} + +wget https://raw.githubusercontent.com/openshift/assisted-service/master/deploy/podman/configmap.yml +wget https://raw.githubusercontent.com/openshift/assisted-service/master/deploy/podman/pod.yml + +/bin/cp -f configmap.yml configmap.yml.bak +/bin/cp -f pod.yml pod.yml.bak + +unset http_proxy +unset https_proxy + +sed -i 's/ SERVICE_BASE_URL:.*/ SERVICE_BASE_URL: "http:\/\/172.21.6.11:8090"/' configmap.yml + +cat /data/ocp4/4.10.12/release.txt | grep " machine-os " + # machine-os 410.84.202204261500-0 Red Hat Enterprise Linux CoreOS + +cat << EOF > /data/assisted-service/os_image.json +[{ + "openshift_version": "4.10", + "cpu_architecture": "x86_64", + "url": "http://172.21.6.11:8080/install/rhcos-live.x86_64.iso", + "rootfs_url": "http://172.21.6.11:8080/install/rootfs.img", + "version": "410.84.202204261500-0" +}] +EOF +cat << EOF > /data/assisted-service/release.json +[{ + "openshift_version": "4.10", + "cpu_architecture": "x86_64", + "url": "quaylab.infra.redhat.ren/ocp4/openshift4:4.10.12-x86_64", + "version": "4.10.12", + "default": true +}] +EOF + +cat configmap.yml.bak \ + | python3 -c 'import json, yaml, sys; print(json.dumps(yaml.load(sys.stdin)))' \ + | jq --arg OSIMAGE "$(jq -c . /data/assisted-service/os_image.json)" '. | .data.OS_IMAGES = $OSIMAGE ' \ + | jq --arg RELEASE_IMAGES "$(jq -c . /data/assisted-service/release.json)" '. | .data.RELEASE_IMAGES = $RELEASE_IMAGES ' \ + | python3 -c 'import yaml, sys; print(yaml.dump(yaml.load(sys.stdin), default_flow_style=False))' \ + > configmap.yml + +cat pod.yml.bak \ + | python3 -c 'import json, yaml, sys; print(json.dumps(yaml.load(sys.stdin)))' \ + | jq ' .spec.containers[1].ports[0].hostPort = 8180 ' \ + | python3 -c 'import yaml, sys; print(yaml.dump(yaml.load(sys.stdin), default_flow_style=False))' \ + > pod.yml + +# 启动本地assisted service +cd /data/assisted-service/ +podman play kube --configmap configmap.yml pod.yml + +# 注入离线镜像仓库的证书 +podman cp /etc/crts/redhat.ren.ca.crt assisted-installer-service:/etc/pki/ca-trust/source/anchors/quaylab.crt +podman exec assisted-installer-service update-ca-trust + +# 用以下命令，停止/删除本地assisted service +cd /data/assisted-service/ +podman play kube --down pod.yml + +# assisted service启动以后，会下载安装介质，我们看看下载占用的空间。 +podman exec assisted-installer-image-service du -h /data +# 1.1G /data + + +``` + + +## 准备vnc环境 + +```bash +dnf -y install tigervnc-server + +vncpasswd + +cat << EOF > ~/.vnc/config +session=gnome +securitytypes=vncauth,tlsvnc +# desktop=sandbox +geometry=1440x855 +alwaysshared +EOF + +cat << EOF >> /etc/tigervnc/vncserver.users +:2=root +EOF + +systemctl start vncserver@:2 +# 如果你想停掉vnc server，这么做 +systemctl stop vncserver@:2 + +systemctl restart vncserver@:2 + +systemctl enable --now vncserver@:2 + +# firewall-cmd --permanent --add-port=6001/tcp +# firewall-cmd --permanent --add-port=5901/tcp +# firewall-cmd --reload + +# connect vnc at port 5901 +# export DISPLAY=:1 + +vncserver :1 -geometry 1280x800 + +ausearch -m avc --start recent -i + +restorecon -RFv ~/.vnc + +audit2allow -a -M wzh-tigervnc +semodule -i wzh-tigervnc.pp + + +python2 -m SimpleHTTPServer 5180 + +python3 -m http.server 5180 + + +``` diff --git a/redhat/ocp4/4.13/4.13.pdns.helper.md b/redhat/ocp4/4.13/4.13.pdns.helper.md new file mode 100644 index 00000000..023efb91 --- /dev/null +++ b/redhat/ocp4/4.13/4.13.pdns.helper.md @@ -0,0 +1,165 @@ +# helper node upgrade + +based on previous research, helper node will be upgrade, it will include only +- pdns ( power dns ) with lua plugin +- ntp / chronyd + +this will support +- multi-cluster on same helper + +it will NOT +- generate / copy ignition +- generate / copy iso + +the source code will be integrade into project openshift4-shell. + +```bash +cd /data/ocp4/ansible-helper + +cat > var.yaml << EOF +helper: + ip_addr: 192.168.77.11 + nic: baremetal +pdns: + bind: 0.0.0.0 + port: 5301 + recursor_port: 53 + forward: 172.21.1.1 + static: + - base_domain: infra.redhat.ren + record: + - name: registry + ip_addr: 192.168.77.11 + - name: nexus + ip_addr: 192.168.77.11 + - name: quay + ip_addr: 192.168.77.11 + - name: quaylab + ip_addr: 192.168.77.11 + - base_domain: infra.wzhlab.top + record: + - name: registry + ip_addr: 192.168.77.11 + - name: nexus + ip_addr: 192.168.77.11 + - name: quay + ip_addr: 192.168.77.11 + - name: quaylab + ip_addr: 192.168.77.11 + - name: panlab-satellite-server + ip_addr: 172.21.6.171 + - base_domain: chatgtpcat.tech + record: + - name: wutonglab + ip_addr: 172.21.6.99 +ntp: + server: 192.168.77.11 +cluster: + - base_domain: acm-demo-hub.wzhlab.top + node: + # - ip_addr: 192.168.77.12 + - ip_addr: 192.168.77.13 + name: sno-master-01 + # - ip_addr: 192.168.77.14 + # - ip_addr: 192.168.77.15 + - base_domain: gpu-sno.wzhlab.top + node: + - ip_addr: 192.168.77.14 + name: gpu-master-01 + - base_domain: acm-demo-one.wzhlab.top + node: + - ip_addr: 192.168.77.22 + name: one-bootstrap + - ip_addr: 192.168.77.23 + name: one-master-01 + - ip_addr: 192.168.77.24 + name: one-master-02 + - ip_addr: 192.168.77.25 + name: one-master-03 + - ip_addr: 192.168.77.26 + name: one-worker-01 + - base_domain: acm-demo-two.wzhlab.top + node: + - ip_addr: 192.168.77.32 + name: two-bootstrap + - ip_addr: 192.168.77.33 + name: two-master-01 + - ip_addr: 192.168.77.34 + name: two-master-02 + - ip_addr: 192.168.77.35 + name: two-master-03 + - base_domain: osp-demo.wzhlab.top + node: + - ip_addr: 192.168.77.42 + name: osp-bootstrap + - ip_addr: 192.168.77.43 + name: osp-master-01 + - ip_addr: 192.168.77.44 + name: osp-master-02 + - ip_addr: 192.168.77.45 + name: osp-master-03 + - ip_addr: 192.168.77.46 + name: osp-worker-01 + - ip_addr: 192.168.77.47 + name: osp-worker-02 + - base_domain: factory.wzhlab.top + node: + - ip_addr: 192.168.12.22 + name: bootstrap + - ip_addr: 192.168.12.23 + name: master-02 + - ip_addr: 192.168.12.24 + name: master-02 + - ip_addr: 192.168.12.25 + name: master-03 + - base_domain: edge01.wzhlab.top + node: + - ip_addr: 192.168.12.33 + name: sno-master + # below set for hypershift lab + # api: + # - ip_addr: 192.168.12.23 + # - ip_addr: 192.168.12.24 + # - ip_addr: 192.168.12.25 + # below set for factor lab + # api: + # - ip_addr: 192.168.12.33 + api_int: + - ip_addr: 192.168.12.34 + apps: + - ip_addr: 192.168.12.35 + - base_domain: edge02.wzhlab.top + node: + - ip_addr: 192.168.12.43 + name: sno-master + - base_domain: edge03.wzhlab.top + node: + - ip_addr: 192.168.12.53 + name: sno-master + api: + - ip_addr: 192.168.12.23 + - ip_addr: 192.168.12.24 + - ip_addr: 192.168.12.25 + apps: + - ip_addr: 192.168.12.23 + - ip_addr: 192.168.12.24 + - ip_addr: 192.168.12.25 + +ptr: + - addr: 192.168.77 + domain: ptr01.wzhlab.top + - addr: 192.168.12 + domain: ptr02.wzhlab.top + - addr: 172.21.6 + domain: ptr03.wzhlab.top +EOF + +cd /data/ocp4/ansible-helper +# ansible-playbook -vvv -e @var.yaml helper.yaml +ansible-playbook -e @var.yaml helper.yaml + +``` + +webconsole: https://172.21.6.99/ + +credential: administrator@wutonglab.local / Panpan0- \ No newline at end of file