Merge pull request #48 from random-archer/dev-ci-booter

ci: qemu support

.azure.yml

@@ -27,23 +27,29 @@ jobs:
     displayName: python version
     inputs:
       versionSpec: 3.8
+#
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: tool/azure/home
+      targetFolder: $(HOME)
+      contents: '**'
+      overWrite: true
+    displayName: copy config
 #
   - script: tool/azure/setup.sh
     displayName: install deps
 #
-# FIXME
-#  - template: tool/azure/steps-cache.yml
-#    parameters: # change to reset cache
-#      cache_version: V7
+  - template: tool/azure/steps-cache.yml
+    parameters: # change to reset cache
+      cache_version: V15
 #
-#  - bash: env|sort|grep CACHE
-#    displayName: review caches
+  - bash: env|sort|grep CACHE
+    displayName: review caches
 #
-# FIXME
   - template: tool/azure/steps-image.yml
     parameters: # conditional cached build
       image_path: arch/base
-#     has_active: or(eq(variables.HAS_CACHE_ARCHIVE, 'false'),eq(variables.HAS_CACHE_ARCH_BASE, 'false'))
+      has_active: or(eq(variables.HAS_CACHE_ARCHIVE, 'false'),eq(variables.HAS_CACHE_ARCH_BASE, 'false'))
 #
   - template: tool/azure/steps-image.yml
     parameters:

.gitignore

@@ -8,3 +8,6 @@
 
 # image boot folder mount
 boot
+
+# image boot folder extract
+data

Makefile

@@ -10,6 +10,6 @@ install:
 	install -vDm 755 src/mkinitcpio-install.sh $(DESTDIR)$(PREFIX)/lib/initcpio/install/systemd-tool
 	install -vDm 755 src/initrd-build.sh -t $(DESTDIR)$(PREFIX)/lib/mkinitcpio-systemd-tool
 	install -vDm 755 src/initrd-shell.sh -t $(DESTDIR)$(PREFIX)/lib/mkinitcpio-systemd-tool
-	install -vDm 644 src/*.{path,service} -t $(DESTDIR)$(SYSTEMD_SYSTEM_PATH)/systemd/system
+	install -vDm 644 src/*.{path,service,target} -t $(DESTDIR)$(SYSTEMD_SYSTEM_PATH)/systemd/system
 	install -vDm 644 LICENSE.md -t $(DESTDIR)$(PREFIX)/share/licenses/mkinitcpio-systemd-tool
 	install -vDm 644 README.md -t $(DESTDIR)$(PREFIX)/share/doc/mkinitcpio-systemd-tool

src/initrd-emergency.service

@@ -2,10 +2,10 @@
 
 # Override Emergency Service
 
+# TODO does not work
+
 # the default reaction to several failed cryptsetup attempts is to isolate the emergency service
 # which results in loss of network and forced local interactive console - no good for ssh mode 
-# so for remote ssh/cryptsetup scenario we instead restart the boot sequence after a delay
-# and also optionally start the provided debug shell
 
 [Unit]
 Description=Initrd Emergency Service
@@ -15,21 +15,35 @@ DefaultDependencies=no
 Conflicts=shutdown.target
 Conflicts=rescue.service
 Before=shutdown.target
+Before=rescue.service
 Wants=initrd-debug-shell.service
 
 [Service]
-ExecStartPre=/bin/echo "Initrd Emergency Service"
-ExecStartPre=/bin/sh -c 'for step in 1 2 3; do echo -n "$step "; sleep 1; done; echo ".";'
-ExecStartPre=/usr/bin/systemctl daemon-reload
-ExecStartPre=/usr/bin/systemctl --job-mode=fail --no-block default
-ExecStart=/bin/true
 Type=idle
-StandardInput=null
-StandardOutput=journal+console
-StandardError=journal+console
-TTYPath=/dev/console
+# startup banner
+ExecStartPre=/bin/echo "### panic shell ###"
+# emergency console
+Environment=TERM=linux
+ExecStart=/bin/sh
+#
+Restart=on-failure
+RestartSec=3s
+#StandardInput=tty
+#StandardOutput=journal+console
+#StandardError=journal+console
+#TTYPath=/dev/console
+
+StandardInput=tty-force
+StandardOutput=inherit
+StandardError=inherit
+KillMode=process
+IgnoreSIGPIPE=no
+SendSIGHUP=yes
+
+[Install]
+
+# not used
 
 [X-SystemdTool]
 
-# override /usr/lib/systemd/system/emergency.service
-#InitrdPath=/etc/systemd/system/emergency.service source=/etc/systemd/system/initrd-emergency.service 
+# not used

src/initrd-emergency.target

@@ -0,0 +1,28 @@
+# This file is part of https://github.com/random-archer/mkinitcpio-systemd-tool
+
+# Override Emergency Target
+
+# TODO does not work
+
+# the default reaction to several failed cryptsetup attempts is to isolate the emergency service
+# which results in loss of network and forced local interactive console - no good for ssh mode 
+
+[Unit]
+
+Description=Initrd Emergency State
+Documentation=https://github.com/random-archer/mkinitcpio-systemd-tool/blob/master/README.md
+ConditionPathExists=/etc/initrd-release
+
+AllowIsolate=yes
+
+After=initrd-emergency.service
+Requires=initrd-emergency.service
+
+[Install]
+# hack to force unit install
+WantedBy=abrakadabra.target
+
+[X-SystemdTool]
+
+# replace default emergency target
+InitrdPath=/etc/systemd/system/emergency.target source=/usr/lib/systemd/system/initrd-emergency.target

tool/azure/home/.nspawn/config.ini

@@ -0,0 +1,9 @@
+#
+# https://github.com/random-python/nspawn
+#
+
+# location of program resources
+[storage]
+
+# base directory for all resources
+root = /home/vsts/nspawn

tool/azure/setup.sh

@@ -6,6 +6,15 @@ set -e
 # prepare azure environment
 #
 
+echo "### ubuntu refresh"
+sudo apt-get -y update
+
+#echo "### setup qemu"
+sudo apt-get -y install qemu-system-x86 cpu-checker
+
+#echo "### report qemu/kvm"
+sudo kvm-ok || true
+
 echo "### setup systemd"
 sudo apt-get -y install attr pigz systemd-container
 

tool/azure/steps-cache.yml

@@ -14,14 +14,14 @@ steps:
 - task: CacheBeta@1
   inputs:
     key: nspawn|archive|${{ parameters.cache_version }}
-    path: /var/lib/nspawn/archive/archive.archlinux.org
+    path: $(HOME)/nspawn/archive
     cacheHitVar: HAS_CACHE_ARCHIVE
   displayName: cache @ archive
 #
 - task: CacheBeta@1
   inputs:
     key: nspawn|arch-base|${{ parameters.cache_version }}
-    path: /var/lib/nspawn/archive/localhost/var/lib/nspawn_systemd_tool/arch-base
+    path: $(HOME)/nspawn/systemd_tool/arch-base
     cacheHitVar: HAS_CACHE_ARCH_BASE
   displayName: cache @ arch-base
-#
+#

tool/image/arch/base/build.py

@@ -17,6 +17,7 @@
 from arkon_config import image_base_url
 from arkon_config import project_repo
 from arkon_config import project_boot
+from arkon_config import project_data
 
 version_path = TOOL.date_path(build_epoch)
 version_dots = TOOL.date_dots(build_epoch)
@@ -63,6 +64,8 @@
    # developer support
    "mc "
    "htop "
+   "xterm "
+   "strace "
    # provide host sshd keys
    "openssh "
    # build/install deps
@@ -72,13 +75,13 @@
    # core package deps
    "linux "
    "mkinitcpio "
-   # dropbear service
+   # initrd-dropbear.service
    "dropbear "
-   # tinysshd service
+   # initrd-tinysshd.service
    "busybox "
    "tinyssh "
    "tinyssh-convert "
-   # cryptsetup service
+   # initrd-cryptsetup.service
    "cryptsetup "
 )
 
@@ -95,5 +98,8 @@
 # expose boot dir
 WITH(Bind=f"{project_boot}/:/repo/boot/")
 
+# expose data dir
+WITH(Bind=f"{project_data}/:/repo/data/")
+
 # publish image
 PUSH()

tool/image/arch/base/unsetup.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+
+#
+# terminate machine
+#
+
+import os
+import sys
+import time
+
+this_dir = os.path.dirname(os.path.abspath(__file__))
+
+command = f"{this_dir}/setup.py --action desure"
+
+os.system(command)

tool/image/arch/base/verify.py

@@ -18,4 +18,4 @@
 
 machine = Machine(machine_base, this_dir)
 
-machine.produce_boot_result()
+machine.perform_make_boot()

tool/image/readme.md

@@ -17,3 +17,7 @@ individual test image:
 operate currently active machines:
 * invoke `machinectl` to see the list
 * invoke `machinectl shell test-cryptsetup` to enter container
+
+### qemu tests require
+
+https://www.archlinux.org/packages/?name=qemu

tool/image/test/build.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 #
-# build in batch
+# build in batch for manual testing before commit
 #
 
 import os
@@ -13,11 +13,21 @@
     "cryptsetup",
     "dropbear",
     "tinysshd",
+    "unitada",
 ]
 
 for image in image_list:
-    print(f"build: {image}")
-    image_root = f"{this_dir}/{image}"
-    os.system(f"{image_root}/build.py")
-    os.system(f"{image_root}/setup.py")
-    os.system(f"{image_root}/verify.py")
+    print(f"@@@ build: {image}")
+    image_base = f"{this_dir}/{image}"
+    command_list = [
+        (f"{image_base}/build.py", 1),
+        (f"{image_base}/setup.py", 1),
+        (f"{image_base}/verify.py", 1),
+    ]
+    for command, settle_time in command_list:
+        print(f"@@@ command: {command}")
+        result = os.system(command)
+        assert result == 0, f"failure: {command}"
+        time.sleep(settle_time)
+
+print(f"@@@ finish")

tool/image/test/cryptsetup/build.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 #
-# build basic archux image
+# build cryptsetup image
 #
 
 from nspawn.build import *

tool/image/test/cryptsetup/etc/mkinitcpio.conf

@@ -2,10 +2,4 @@
 # https://wiki.archlinux.org/index.php/Mkinitcpio
 #
 
-MODULES=""
-
-BINARIES=""
-
-FILES=""
-
-HOOKS="base autodetect modconf block filesystems keyboard systemd systemd-tool" 
+HOOKS=(base autodetect modconf block filesystems keyboard systemd systemd-tool)

tool/image/test/cryptsetup/etc/systemd/system/initrd-cryptsetup.path.d/override.conf

@@ -1,6 +1,5 @@
 
 [X-SystemdTool]
 
-# disable default password agents, AGAIN
-InitrdPath=/usr/lib/systemd/system/systemd-ask-password-console.path replace=yes create=yes
-InitrdPath=/usr/lib/systemd/system/systemd-ask-password-wall.path    replace=yes create=yes
+# inject busybox enviro
+InitrdPath=/etc/profile source=/etc/systemd/system/profile.sh

tool/image/test/cryptsetup/etc/systemd/system/initrd-debug-progs.service.d/override.conf

@@ -0,0 +1,17 @@
+
+[X-SystemdTool]
+
+# debug support
+InitrdBinary=/usr/bin/strace
+
+# manual crypto mount
+InitrdBinary=/usr/bin/cryptsetup
+
+# dependency reporter
+InitrdBinary=/usr/bin/systemd-analyze 
+
+# serial console resizer
+InitrdBinary=/usr/bin/resize
+
+# qemu guest drivers
+InitrdCall=add_all_modules /virtio/

tool/image/test/cryptsetup/etc/systemd/system/profile.sh

@@ -0,0 +1,5 @@
+#
+# busybox/ash enviro
+#
+
+export TERM=xterm

tool/image/test/cryptsetup/test_base/usr/lib/systemd/system/initrd-cryptsetup.path

@@ -44,6 +44,5 @@ InitrdPath=/usr/lib/systemd/system/systemd-ask-password-wall.path      replace=y
 
 [X-SystemdTool]
 
-# disable default password agents, AGAIN
-InitrdPath=/usr/lib/systemd/system/systemd-ask-password-console.path replace=yes create=yes
-InitrdPath=/usr/lib/systemd/system/systemd-ask-password-wall.path    replace=yes create=yes
+# inject busybox enviro
+InitrdPath=/etc/profile source=/etc/systemd/system/profile.sh

tool/image/test/cryptsetup/unsetup.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python
+
+#
+# terminate machine
+#
+
+import os
+import sys
+import time
+
+this_dir = os.path.dirname(os.path.abspath(__file__))
+
+command = f"{this_dir}/setup.py --action desure"
+
+os.system(command)