Updating ISO building for Ubuntu 22.04

images/iso/autoinstall/README.md

@@ -0,0 +1,30 @@
+# Ubuntu Autoinstall
+
+Link: https://canonical-subiquity.readthedocs-hosted.com/en/latest/intro-to-autoinstall.html
+
+## DHCP/Static IP
+
+* Automated DHCP: `nocloud-dhcp/user-data`
+* Static IP: `nocloud-nodhcp/user-data` (there is
+  *interactive-sections* which will invoke text UI to enter IP
+  address manually or select the DHCP).
+
+
+## Note
+
+The `autoinstall.yaml` is expected to be present in the root of ISO
+Image. Its format is missing main `autoinstall:` header in 22.04. In
+later version (>= 24) it is expected to have different indentation:
+
+```yaml
+# Autoinstall configuration
+autoinstall:
+  version: 1
+
+# Storage configuration with LVM
+  storage:
+    layout:
+      name: lvm
+...
+```
+

images/iso/autoinstall/nocloud-dhcp/user-data

@@ -0,0 +1,51 @@
+#cloud-config
+# https://ubuntu.com/server/docs/install/autoinstall-reference
+autoinstall:
+  version: 1
+
+  storage:
+    layout:
+      name: lvm
+
+  network:
+    version: 2
+    ethernets:
+      enp0s3:
+        dhcp4: true
+
+  identity:
+    realname: 'Ubuntu ONA User'
+    username: ona-ubuntu
+    password: '$y$j9T$vTPss2y8DIwI64Qhspa2u1$0WErBalivR6wnQ82NIMmjzQDEEr7m6vfsESbVLzgwa1'
+    hostname: ona-default
+
+  locale: en_US.UTF-8
+  timezone: UTC
+
+  ssh:
+    install-server: true
+
+  # iptables-persistent settings
+  early-commands:
+    - echo 'iptables-persistent iptables-persistent/autosave_v6 boolean false' | debconf-set-selections
+    - echo 'iptables-persistent iptables-persistent/autosave_v4 boolean false' | debconf-set-selections
+
+  packages:
+    - apt-transport-https
+    - iptables-persistent
+    - ipset
+    - libjansson4
+    - libltdl7
+    - liblzo2-2
+    - libnet1
+    - libyaml-0-2
+    - nano
+    - ntp
+    - ntpdate
+    - snmp
+    - tcpdump
+    - net-tools
+
+  late-commands:
+    - cp -r /cdrom/ona/ /target/root/
+    - curtin in-target --target=/target -- bash -xv /root/ona/configure.sh

images/iso/autoinstall/nocloud-nodhcp/user-data

@@ -0,0 +1,54 @@
+#cloud-config
+# https://ubuntu.com/server/docs/install/autoinstall-reference
+autoinstall:
+  version: 1
+
+  interactive-sections:
+   - network
+
+  storage:
+    layout:
+      name: lvm
+
+  network:
+    version: 2
+    ethernets:
+      enp0s3:
+        dhcp4: true
+
+  identity:
+    realname: 'Ubuntu ONA User'
+    username: ona-ubuntu
+    password: '$y$j9T$vTPss2y8DIwI64Qhspa2u1$0WErBalivR6wnQ82NIMmjzQDEEr7m6vfsESbVLzgwa1'
+    hostname: ona-default
+
+  locale: en_US.UTF-8
+  timezone: UTC
+
+  ssh:
+    install-server: true
+
+  # iptables-persistent settings
+  early-commands:
+    - echo 'iptables-persistent iptables-persistent/autosave_v6 boolean false' | debconf-set-selections
+    - echo 'iptables-persistent iptables-persistent/autosave_v4 boolean false' | debconf-set-selections
+
+  packages:
+    - apt-transport-https
+    - iptables-persistent
+    - ipset
+    - libjansson4
+    - libltdl7
+    - liblzo2-2
+    - libnet1
+    - libyaml-0-2
+    - nano
+    - ntp
+    - ntpdate
+    - snmp
+    - tcpdump
+    - net-tools
+
+  late-commands:
+    - cp -r /cdrom/ona/ /target/root/
+    - curtin in-target --target=/target -- bash -xv /root/ona/configure.sh

images/iso/build_iso.sh

@@ -21,9 +21,9 @@
 #    wrong.
 #
 
-RELEASE="${RELEASE:-20.04.1}"
+RELEASE="${RELEASE:-22.04.4}"
 ARCH="${ARCH:-amd64}"
-VARIANT="${VARIANT:-legacy}"
+VARIANT="${VARIANT:-subiquity}"
 
 
 DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
@@ -50,42 +50,102 @@ ubuntu_name="ubuntu-${RELEASE}-server-${ARCH}.iso"
 ona_name="ona-${RELEASE}-server-${ARCH}.iso"
 ubuntu_url="${url:-$($DIR/build_iso_helper $RELEASE $VARIANT)}"
 test -n "$ubuntu_url" || fatal "failed getting Ubuntu ISO download URL"
-ona_service_url="https://s3.amazonaws.com/onstatic/ona-service/master/ona-service_UbuntuXenial_amd64.deb"
+
+ONA_URL="https://s3.amazonaws.com/onstatic/ona-service/master/"
+if [ -n "$PUBLIC_ONA" ]; then
+  ONA_URL="https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/"
+fi
+
+ona_service_url="${ONA_URL}ona-service_UbuntuXenial_amd64.deb"
 netsa_pkg_url="https://assets-production.obsrvbl.com/ona-packages/netsa/v0.1.27/netsa-pkg.deb"
 
 shift $(($OPTIND-1))
 
 test $EUID -ne 0 && sudo="sudo"
-which mkisofs 1> /dev/null || fatal "missing mkisofs: $sudo apt-get install genisoimage"
-which isohybrid 1> /dev/null || fatal "missing isohybrid: $sudo apt-get install syslinux-utils"
 
 [[ -d "$DIR" ]] || fatal  # invalid directory
-[[ -d "$DIR"/working && $(ls -A "$DIR"/working) ]] && fatal  # working directory exists and is not empty
 [[ -d "$DIR"/working ]] || mkdir "$DIR"/working # working directory does not exist, so create it
+
+major_version=$(echo "$RELEASE" | cut -d '.' -f 1)
+
+# Check if the major version number is greater than 20
+if [ "$major_version" -gt 20 ]; then
+  which xorriso 1> /dev/null || fatal "missing xorriso: $sudo apt-get install xorriso -y"
+  NEW_FORMAT=true
+  BOOT_CAT="/boot.catalog"
+  EFI='/boot/grub/i386-pc/eltorito.img'
+  ELTORITO='/boot/grub/i386-pc/eltorito.img'
+else
+  which mkisofs 1> /dev/null || fatal "missing mkisofs: $sudo apt-get install genisoimage"
+  which isohybrid 1> /dev/null || fatal "missing isohybrid: $sudo apt-get install syslinux-utils"
+  BOOT_CAT="isolinux/boot.cat"
+  EFI="isolinux/isolinux.bin"
+  ELTORITO="boot/grub/efi.img"
+fi
+
 (
   set -e
+  if [ ! -e "$ubuntu_name" ]; then
+    curl -L -o ${ubuntu_name} "${ubuntu_url}"
+  fi
+
   cd "$DIR"/working
-  curl -L -o ${ubuntu_name} "${ubuntu_url}"
   curl -L -o netsa-pkg.deb "${netsa_pkg_url}"
   curl -L -o ona-service.deb "${ona_service_url}"
+  # local is root dir in ISO
   mkdir cdrom local
-  $sudo mount -o loop --read-only "${ubuntu_name}" cdrom
+
+  $sudo mount -o loop --read-only "../${ubuntu_name}" cdrom
   rsync -av --quiet cdrom/ local
-  $sudo cp ../preseed/* local/preseed/
+
   $sudo cp -r ../ona local
   $sudo cp netsa-pkg.deb local/ona/netsa-pkg.deb
   $sudo cp ona-service.deb local/ona/ona-service.deb
-  $sudo cp ../isolinux/txt.cfg local/isolinux/txt.cfg
-  $sudo cp ../isolinux/grub.cfg local/boot/grub/grub.cfg
-  $sudo mkisofs -quiet -r -V "SWC Sensor Install CD" \
-          -cache-inodes \
-          -J -l -b isolinux/isolinux.bin \
-          -c isolinux/boot.cat -no-emul-boot \
-          -boot-load-size 4 -boot-info-table \
-          -eltorito-alt-boot -e boot/grub/efi.img -no-emul-boot \
-          -o "../${ona_name}" local
+
+  echo "New format: $NEW_FORMAT "
+  if [ -n "$NEW_FORMAT" ]; then
+    # copy autoinstall folders for grub
+    $sudo cp -r ../autoinstall/nocloud-dhcp  local/
+    $sudo cp -r ../autoinstall/nocloud-nodhcp  local/
+    $sudo cp ../isolinux/grub-new-format.cfg local/boot/grub/grub.cfg
+  else
+    $sudo cp ../preseed/* local/preseed/
+    $sudo cp ../isolinux/txt.cfg local/isolinux/txt.cfg
+    $sudo cp ../isolinux/grub.cfg local/boot/grub/grub.cfg
+  fi
+
+  if [ -n "$NEW_FORMAT" ]; then
+    xorriso -as mkisofs -r  -V 'SWC Sensor Install CD' \
+      -o "../${ona_name}"\
+      --grub2-mbr --interval:local_fs:0s-15s:zero_mbrpt,zero_gpt:"../${ubuntu_name}" \
+      -partition_offset 16 \
+      --mbr-force-bootable \
+      -append_partition 2 0xef \
+      --interval:local_fs:4099440d-4109507d::"../${ubuntu_name}" \
+      -appended_part_as_gpt \
+      -c "${BOOT_CAT}" \
+      -b "${ELTORITO}" \
+      -no-emul-boot -boot-load-size 4 -boot-info-table \
+      --grub2-boot-info \
+      -eltorito-alt-boot \
+      -e '--interval:appended_partition_2:::' \
+      -no-emul-boot \
+      local
+  else
+    $sudo mkisofs -quiet -r -V "SWC Sensor Install CD" \
+      -cache-inodes \
+      -J -l -b "${BOOT_CAT}" \
+      -c "${EFI}" -no-emul-boot \
+      -joliet-long \
+      -boot-load-size 4 -boot-info-table \
+      -eltorito-alt-boot -e "${ELTORITO}" -no-emul-boot \
+      -o "../${ona_name}" local
+
+    isohybrid "../${ona_name}"
+  fi
+
   $sudo umount cdrom
   $sudo chown $USER:$USER "../${ona_name}"
-  isohybrid "../${ona_name}"
+  $sudo rm -rf "$DIR"/working
 )
-$sudo rm -rf "$DIR"/working
+ 

images/iso/build_iso_helper

@@ -17,6 +17,9 @@ $ ./build_iso_helper --doctests
 
 Real results as of February 2022:
 
+>>> get_iso_url('22.04', subiquity=True)
+'https://releases.ubuntu.com/20.04/ubuntu-20.04.3-live-server-amd64.iso'
+
 >>> get_iso_url('20.04.3', subiquity=True)
 'https://releases.ubuntu.com/20.04/ubuntu-20.04.3-live-server-amd64.iso'
 
@@ -53,9 +56,10 @@ ValueError: no release found
 Traceback (most recent call last):
 ValueError: no Subiquity release for this version
 """
+
 from argparse import ArgumentParser, Action, SUPPRESS
 from doctest import DocTestSuite
-from unittest import TestCase, TextTestRunner, makeSuite
+from unittest import TestCase, TextTestRunner, TestLoader
 from unittest.mock import patch
 from urllib.request import Request, build_opener, HTTPRedirectHandler
 from urllib.error import URLError, HTTPError
@@ -120,11 +124,14 @@ def get_iso_url(version, subiquity: bool, arch='amd64'):
             urlpath = f'https://releases.ubuntu.com/{v.short}/'
             isofile = f'ubuntu-{v.long_if_patch}-live-server-{arch}.iso'
         else:
-            urlpath = (
-                'https://cdimage.ubuntu.com/ubuntu-legacy-server/releases'
-                f'/{v.short}/release/'
-            )
-            isofile = f'ubuntu-{v.long_if_patch}-legacy-server-{arch}.iso'
+            if v.major >= 22:
+                raise ValueError(f'No legacy ISO for version {v.short}')
+            else:
+              urlpath = (
+                  'https://cdimage.ubuntu.com/ubuntu-legacy-server/releases'
+                  f'/{v.short}/release/'
+              )
+              isofile = f'ubuntu-{v.long_if_patch}-legacy-server-{arch}.iso'
     elif v.major >= 18:
         if subiquity:
             urlpath = f'https://releases.ubuntu.com/{v.short}/'
@@ -181,7 +188,7 @@ class Tests(TestCase):
 
         def fake_head_status_code(url):
             if url in (
-                f'{cd_legacy}/20.04.1/release/ubuntu-20.04.1-legacy-server-amd64.iso',
+                f'{cd_legacy}/20.04/release/ubuntu-20.04.1-legacy-server-amd64.iso',
                 f'{releases}/20.04/ubuntu-20.04.1-live-server-amd64.iso',
                 f'{old}/20.04.0/ubuntu-20.04-live-server-amd64.iso',
                 'https://www.google.com',
@@ -195,9 +202,10 @@ class Tests(TestCase):
                 get_iso_url('20.04.1', True),
                 f'{releases}/20.04/ubuntu-20.04.1-live-server-amd64.iso',
             )
+
             self.assertEqual(
                 get_iso_url('20.04.1', False),
-                f'{cd_legacy}/20.04.1/release/ubuntu-20.04.1-legacy-server-amd64.iso',
+                f'{cd_legacy}/20.04/release/ubuntu-20.04.1-legacy-server-amd64.iso',
             )
             self.assertEqual(
                 get_iso_url('20.04.0', True),
@@ -210,7 +218,7 @@ class Tests(TestCase):
 class _TestAction(Action):
     def __init__(self, option_strings, dest, default=SUPPRESS, help=None):
         if dest == 'unittests':
-            self._suite = makeSuite(Tests)
+            self._suite = TestLoader().loadTestsFromTestCase(Tests)
         elif dest == 'doctests':
             self._suite = DocTestSuite()
         else:

images/iso/isolinux/grub-new-format.cfg

@@ -0,0 +1,53 @@
+if loadfont /boot/grub/font.pf2 ; then
+    set gfxmode=auto
+    insmod efi_gop
+    insmod efi_uga
+    insmod gfxterm
+    terminal_output gfxterm
+fi
+
+set menu_color_normal=white/black
+set menu_color_highlight=black/light-gray
+
+set timeout=30
+
+loadfont unicode
+
+menuentry "Install ONA (Static IP)" {
+    set gfxpayload=keep
+    linux   /casper/vmlinuz debug autoinstall ds=nocloud\;s=/cdrom/nocloud-nodhcp/ ---
+    initrd  /casper/initrd
+}
+
+menuentry "Install ONA (DHCP)" {
+    set gfxpayload=keep
+    linux   /casper/vmlinuz debug autoinstall ds=nocloud\;s=/cdrom/nocloud-dhcp/ ---
+    initrd  /casper/initrd
+}
+
+menuentry "Install ONA (Static IP), HWE kernel" {
+    set gfxpayload=keep
+    linux   /casper/hwe-vmlinuz debug autoinstall ds=nocloud\;s=/cdrom/nocloud-nodhcp/ ---
+    initrd  /casper/hwe-initrd
+}
+
+menuentry "Install ONA (DHCP), HWE kernel" {
+    set gfxpayload=keep
+    linux   /casper/hwe-vmlinuz debug autoinstall ds=nocloud\;s=/cdrom/nocloud-dhcp/ ---
+    initrd  /casper/hwe-initrd
+}
+
+grub_platform
+if [ "$grub_platform" = "efi" ]; then
+    menuentry 'Boot from next volume' {
+        exit 1
+    }
+    menuentry 'UEFI Firmware Settings' {
+    zo    fwsetup
+    }
+else
+    menuentry 'Test memory' {
+        linux16 /boot/memtest86+.bin
+    }
+fi
+