Port s390x/other arches fixes from master with some tweaks

As an  effort to get an rhcos 4.2 image built and also to get the pipeline going with the main focus as s390x,
this contains ports of fixes for issues specified in coreos/fedora-coreos-tracker#255
As an addition there were some changes that needed to be made in the kickstart file to get anaconda to work for s390x.
One caveat is that the fedora image used with the installation needs to have anaconda patched with s390 fixes for this to
work.

deps-s390x.txt

@@ -0,0 +1 @@
+src/deps-s390x.txt

src/cmd-buildextend-installer

@@ -67,11 +67,6 @@ os.mkdir(tmpisoisolinux)
 
 def generate_iso():
     arch = platform.machine()
-
-    if arch in ("s390", "s390x"):
-        print(f"{arch} ISO generation is not supported at the moment")
-        sys.exit(1)
-
     tmpisofile = os.path.join(tmpdir, iso_name)
 
     # Find the directory under `/usr/lib/modules/<kver>` where the
@@ -128,6 +123,18 @@ def generate_iso():
     elif arch == "ppc64le":
         genisoargs += ['-r', '-l', '-sysid', 'PPC',
                        '-chrp-boot', '-graft-points']
+    elif arch == "s390x":
+        # combine kernel, initramfs and cmdline using lorax/mk-s390-cdboot tool
+        run_verbose(['/usr/bin/mk-s390-cdboot',
+                     '-i', os.path.join(tmpisoimages, 'vmlinuz'),
+                     '-r', os.path.join(tmpisoimages, 'initramfs.img'),
+                     '-p', os.path.join(tmpisoroot, 'zipl.prm'),
+                     '-o', os.path.join(tmpisoimages, 'fcos.img')])
+        genisoargs = ['/usr/bin/xorrisofs', '-verbose',
+                      '-volset', f"{name_version}",
+                      '-rock', '-J', '-joliet-long',
+                      '-no-emul-boot', '-eltorito-boot',
+                      os.path.join(os.path.relpath(tmpisoimages, tmpisoroot), 'fcos.img')]
 
     ### For x86_64 and aarch64 UEFI booting
     if arch in ("x86_64", "aarch64"):

src/cmd-run

@@ -102,7 +102,7 @@ set -- -machine accel=kvm -cpu host -smp "${VM_NCPUS}" "$@"
 
 if [ -n "${VM_SRV_MNT}" ]; then
     set -- --fsdev local,id=var-srv,path="${VM_SRV_MNT}",security_model=mapped,readonly \
-        -device virtio-9p-pci,fsdev=var-srv,mount_tag=/var/srv "$@"
+        -device virtio-9p-"${devtype}",fsdev=var-srv,mount_tag=/var/srv "$@"
     ign_var_srv_mount=',{
 "name": "var-srv.mount",
 "enabled": true,
@@ -225,6 +225,6 @@ fi
 # shellcheck disable=SC2086
 exec ${QEMU_KVM} -name coreos -m "${VM_MEMORY}" -nographic \
               -netdev user,id=eth0,hostname=coreos"${hostfwd:-}" \
-              -device virtio-net-pci,netdev=eth0 \
-              -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \
+              -device virtio-net-"${devtype}",netdev=eth0 \
+              -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-"${devtype}",rng=rng0 \
               "$@"

src/cmdlib.sh

@@ -41,12 +41,13 @@ export basearch
 # Get target architecture
 arch=$(uname -m)
 export arch
+devtype=pci
 
 case $arch in
     "x86_64")  VM_TERMINAL="ttyS0"    ;;
     "ppc64le") VM_TERMINAL="hvc0"     ;;
     "aarch64") VM_TERMINAL="ttyAMA0"  ;;
-    "s390x")   VM_TERMINAL="ttysclp0" ;;
+    "s390x")   VM_TERMINAL="ttysclp0"; devtype=ccw ;;
     *)         fatal "Architecture $(arch) not supported"
 esac
 export VM_TERMINAL
@@ -59,7 +60,7 @@ else
         "x86_64")  QEMU_KVM="qemu-system-$(arch) -accel kvm"                          ;;
         "aarch64") QEMU_KVM="qemu-system-$(arch) -accel kvm -M virt,gic-version=host" ;;
         "ppc64le") QEMU_KVM="qemu-system-ppc64 -accel kvm"                            ;;
-        "s390x")   QEMU_KVM="qemu-system-$(arch) -accel kvm"                          ;;
+        "s390x")   QEMU_KVM="qemu-system-$(arch) -accel kvm -M s390-ccw-virtio"       ;;
         *)         fatal "Architecture $(arch) not supported"
     esac
 fi
@@ -334,8 +335,9 @@ runvm() {
     [ -n "${ISFEDORA}" ] && filter='^#FEDORA '
     [ -n "${ISEL}" ]     && filter='^#EL7 '
     rpms=$(sed "s/${filter}//" "${DIR}"/vmdeps.txt | grep -v '^#')
+    archrpms=$(sed "s/${filter}//" "${DIR}"/vmdeps-"$(arch)".txt | grep -v '^#')
     # shellcheck disable=SC2086
-    supermin --prepare --use-installed -o "${vmpreparedir}" $rpms
+    supermin --prepare --use-installed -o "${vmpreparedir}" $rpms $archrpms
 
     # include COSA in the image
     find /usr/lib/coreos-assembler/ -type f > "${vmpreparedir}/hostfiles"
@@ -381,23 +383,26 @@ EOF
         srcvirtfs=("-virtfs" "local,id=source,path=${workdir}/src/config,security_model=none,mount_tag=source")
     fi
 
-    pcibus=pci.0
+    scsibus="bus=pci.0,addr=0x3"
     arch_args=
-    if [ "$(arch)" = "aarch64" ]; then
-        # 'pci' bus doesn't work on aarch64
-        pcibus=pcie.0
-        arch_args='-bios /usr/share/AAVMF/AAVMF_CODE.fd'
-    fi
+    case $arch in
+        "aarch64")
+            # 'pci' bus doesn't work on aarch64
+            scsibus="bus=pcie.0,addr=0x3"
+            arch_args='-bios /usr/share/AAVMF/AAVMF_CODE.fd'
+	    ;;
+        "s390x") scsibus="devno=fe.0.0003" ;;
+    esac
 
     #shellcheck disable=SC2086
     ${QEMU_KVM} ${arch_args:-} \
         -nodefaults -nographic -m 2048 -no-reboot -cpu host \
         -kernel "${vmbuilddir}/kernel" \
         -initrd "${vmbuilddir}/initrd" \
         -netdev user,id=eth0,hostname=supermin \
-        -device virtio-net-pci,netdev=eth0 \
-        -device virtio-scsi-pci,id=scsi0,bus=${pcibus},addr=0x3 \
-        -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-pci,rng=rng0 \
+        -device virtio-net-"${devtype}",netdev=eth0 \
+        -device virtio-scsi-"${devtype}",id=scsi0,"${scsibus}" \
+        -object rng-random,filename=/dev/urandom,id=rng0 -device virtio-rng-"${devtype}",rng=rng0 \
         -drive if=none,id=drive-scsi0-0-0-0,snapshot=on,file="${vmbuilddir}/root" \
         -device scsi-hd,bus=scsi0.0,channel=0,scsi-id=0,lun=0,drive=drive-scsi0-0-0-0,id=scsi0-0-0-0,bootindex=1 \
         "${cachedisk[@]}" \

src/deps-s390x.txt

@@ -0,0 +1,5 @@
+# For building iso image
+lorax xorriso
+
+# Deps needed by supermin
+s390utils-base

src/gf-anaconda-cleanup

@@ -26,7 +26,12 @@ set -x
 coreos_gf_run "${src}"
 # We don't have a way to do this with Anaconda/kickstart right now.
 # This bootstraps us to be able to do all of the mounts.
-coreos_gf set-label /dev/sda2 boot
+if [ "$arch" != "s390x" ]; then
+    coreos_gf set-label /dev/sda2 boot
+else
+    coreos_gf set-label /dev/sda1 boot
+fi
+
 coreos_gf_run_mount "${src}"
 
 fb=/boot/ignition.firstboot

src/gf-platformid

@@ -41,19 +41,22 @@ coreos_gf_run_mount "${tmp_dest}"
 # * grub config
 # * BLS config (for subsequent config regeneration)
 # First, the grub config.
-if [ "$(coreos_gf exists '/boot/efi')" == 'true' ]; then
-    grubcfg_path=$(coreos_gf glob-expand /boot/efi/EFI/*/grub.cfg)
-else
-    grubcfg_path=/boot/loader/grub.cfg
+# No grub on s390x
+if [ "$arch" != "s390x" ]; then
+    if [ "$(coreos_gf exists '/boot/efi')" == 'true' ]; then
+        grubcfg_path=$(coreos_gf glob-expand /boot/efi/EFI/*/grub.cfg)
+    else
+        grubcfg_path=/boot/loader/grub.cfg
+    fi
+    coreos_gf download "${grubcfg_path}" "${tmpd}"/grub.cfg
+    # Remove any platformid currently there
+    sed -i -e 's, ignition.platform.id=[a-zA-Z0-9]*,,g' "${tmpd}"/grub.cfg
+    # Insert our new platformid
+    # Match linux16, linux and linuxefi since only linux is available on aarch64
+    # and linuxefi is available in grub2.cfg for UEFI
+    sed -i -e 's,^\(linux\(16\|efi\)\? .*\),\1 coreos.oem.id='"${platformid}"' ignition.platform.id='"${platformid}"',' "${tmpd}"/grub.cfg
+    coreos_gf upload "${tmpd}"/grub.cfg "${grubcfg_path}"
 fi
-coreos_gf download "${grubcfg_path}" "${tmpd}"/grub.cfg
-# Remove any platformid currently there
-sed -i -e 's, ignition.platform.id=[a-zA-Z0-9]*,,g' "${tmpd}"/grub.cfg
-# Insert our new platformid
-# Match linux16, linux and linuxefi since only linux is available on aarch64
-# and linuxefi is available in grub2.cfg for UEFI
-sed -i -e 's,^\(linux\(16\|efi\)\? .*\),\1 coreos.oem.id='"${platformid}"' ignition.platform.id='"${platformid}"',' "${tmpd}"/grub.cfg
-coreos_gf upload "${tmpd}"/grub.cfg "${grubcfg_path}"
 # Now the BLS version
 blscfg_path=$(coreos_gf glob-expand /boot/loader/entries/ostree-*.conf)
 coreos_gf download "${blscfg_path}" "${tmpd}"/bls.conf

src/image-base.ks

@@ -18,12 +18,12 @@ firewall --disabled
 # rw and $ignition_firstboot are used by https://github.com/coreos/ignition-dracut/
 # Console settings are so we see output everywhere
 # %%KARGS%% is for distro-specific arguments
-bootloader --timeout=1 --append="console=ttyS0,115200n8 console=tty0 rootflags=defaults,prjquota rw %%KARGS%% $ignition_firstboot"
+bootloader --timeout=1 --append="console=%%TERM%%,115200n8 console=tty0 rootflags=defaults,prjquota rw %%KARGS%% $ignition_firstboot"
 # Anaconda currently writes out configs for this which we don't want to persist; see below
 network --bootproto=dhcp --onboot=on
 
 zerombr
-clearpart --initlabel --all --disklabel=gpt
+clearpart --initlabel --all %%DISKLABEL%%
 
 # https://github.com/coreos/fedora-coreos-tracker/issues/18
 # See also coreos-growpart.service defined in fedora-coreos-base.yaml

src/virt-install

@@ -118,7 +118,15 @@ extra_kargs = image_conf.get('extra-kargs', [])
 
 with open('/usr/lib/coreos-assembler/image-base.ks') as basef:
     buf = basef.read()
+    # grub can detect /boot/ignition.firstboot file at first boot and add those parameters on the fly to $ignition_firstboot
+    # zipl requires those to be added and a re-run of zipl program at build time
+    if platform.machine() == "s390x":
+        extra_kargs.append("ignition.firstboot rd.neednet=1 ip=dhcp coreos.oem.id=qemu ignition.platform.id=qemu")
+        buf = buf.replace('%%DISKLABEL%%','')
+    else:
+        buf = buf.replace('%%DISKLABEL%%',' disklabel=gpt')
     buf = buf.replace('%%KARGS%%', ' '.join(extra_kargs))
+    buf = buf.replace('%%TERM%%', os.environ.get('VM_TERMINAL', 'ttyS0'))
     ks_tmp.write(buf)
 
 # To support different distro builds using Fedora anaconda,
@@ -302,7 +310,7 @@ try:
                           location_arg,
                           "--disk=path={},cache=unsafe".format(args.dest),
                           "--initrd-inject={}".format(ks_tmp.name),
-                          "--extra-args", "ks=file://{} console=tty0 console=ttyS0,115200n8 inst.cmdline inst.notmux".format(os.path.basename(ks_tmp.name))])
+                          "--extra-args", "ks=file://{} console=tty0 console={},115200n8 inst.cmdline inst.notmux".format(os.path.basename(ks_tmp.name), os.environ.get('VM_TERMINAL', 'ttyS0'))])
     if args.variant == 'metal-uefi':
         vinstall_args.append('--boot=uefi')
     run_sync_verbose(vinstall_args)

src/vmdeps-aarch64.txt

@@ -0,0 +1 @@
+grub2 grub2-efi-aa64

src/vmdeps-ppc64le.txt

@@ -0,0 +1 @@
+grub2

src/vmdeps-s390x.txt

@@ -0,0 +1 @@
+s390utils-base haveged

src/vmdeps-x86_64.txt

@@ -0,0 +1 @@
+grub2 grub2-efi-x64

src/vmdeps.txt

@@ -17,3 +17,5 @@ selinux-policy selinux-policy-targeted policycoreutils
 
 # coreos-assembler
 #FEDORA python3 python3-gobject-base buildah podman skopeo iptables iptables-libs
+
+gdisk xfsprogs e2fsprogs dosfstools shim

vmdeps-aarch64.txt

@@ -0,0 +1 @@
+src/vmdeps-aarch64.txt

vmdeps-ppc64le.txt

@@ -0,0 +1 @@
+src/vmdeps-ppc64le.txt

vmdeps-s390x.txt

@@ -0,0 +1 @@
+src/vmdeps-s390x.txt

vmdeps-x86_64.txt

@@ -0,0 +1 @@
+src/vmdeps-x86_64.txt