From a3daccdd843e2b8e04cb660bf71b1baba8673f6e Mon Sep 17 00:00:00 2001 From: Steven Presti Date: Tue, 7 Nov 2023 15:38:44 -0500 Subject: [PATCH] mantle/kola: copy basic tests into formal kola workflow `cmd-kola` has non-conformed basic kola tests. These tests do not benefit from the supporting functionality around kola test's such as denylisting or other functionality. reduce `cmd-kola` and conform the basic kola tests. fixes https://github.com/coreos/coreos-assembler/issues/3418 co-authored-by: AdamOBrien --- mantle/kola/harness.go | 5 +- mantle/kola/register/register.go | 3 + mantle/kola/tests/coretest/core.go | 106 +++++++++++++++++++++++++++-- src/cmd-kola | 33 +-------- 4 files changed, 111 insertions(+), 36 deletions(-) diff --git a/mantle/kola/harness.go b/mantle/kola/harness.go index 2dc564e6d1..9d41d418df 100644 --- a/mantle/kola/harness.go +++ b/mantle/kola/harness.go @@ -614,7 +614,9 @@ func filterTests(tests map[string]*register.Test, patterns []string, pltfrm stri continue } } - + if t.Nvme != QEMUOptions.Nvme { + continue + } // Check native tests for arch-specific and distro-specfic exclusion for k, NativeFuncWrap := range t.NativeFuncs { _, excluded := isAllowed(Options.Distribution, nil, NativeFuncWrap.Exclusions) @@ -1031,6 +1033,7 @@ type externalTestMeta struct { AllowConfigWarnings bool `json:"allowConfigWarnings" yaml:"allowConfigWarnings"` NoInstanceCreds bool `json:"noInstanceCreds" yaml:"noInstanceCreds"` Description string `json:"description" yaml:"description"` + Nvme bool `json:"nvme" yaml:"nvme"` } // metadataFromTestBinary extracts JSON-in-comment like: diff --git a/mantle/kola/register/register.go b/mantle/kola/register/register.go index 65bedb205c..10afba77c8 100644 --- a/mantle/kola/register/register.go +++ b/mantle/kola/register/register.go @@ -112,6 +112,9 @@ type Test struct { // Conflicts is non-empty iff nonexclusive is true // Contains the tests that conflict with this particular test Conflicts []string + + // If true, this test will be run on NVMe storage + Nvme bool } // Registered tests that run as part of `kola run` live here. Mapping of names diff --git a/mantle/kola/tests/coretest/core.go b/mantle/kola/tests/coretest/core.go index 2c46ac5689..60a060b16f 100644 --- a/mantle/kola/tests/coretest/core.go +++ b/mantle/kola/tests/coretest/core.go @@ -19,6 +19,8 @@ import ( const ( DockerTimeout = time.Second * 60 PortTimeout = time.Second * 3 + uefi = "uefi" + uefiSecure = "uefi-secure" ) // RHCOS services we expect disabled/inactive @@ -38,11 +40,31 @@ var offServices = []string{ } func init() { + register.RegisterTest(®ister.Test{ - Name: "basic", - Description: "Verify basic functionalities like SSH, systemd services, useradd, etc.", - Run: LocalTests, - ClusterSize: 1, + Name: "basic.bios", + Description: "Verify basic functionalities like SSH, systemd services, useradd, etc.", + ExcludeFirmwares: []string{uefi, uefiSecure}, + Run: LocalTests, + ClusterSize: 1, + NativeFuncs: map[string]register.NativeFuncWrap{ + "PortSSH": register.CreateNativeFuncWrap(TestPortSsh), + "DbusPerms": register.CreateNativeFuncWrap(TestDbusPerms), + "ServicesActive": register.CreateNativeFuncWrap(TestServicesActive), + "ReadOnly": register.CreateNativeFuncWrap(TestReadOnlyFs), + "Useradd": register.CreateNativeFuncWrap(TestUseradd), + "MachineID": register.CreateNativeFuncWrap(TestMachineID), + "RHCOSGrowpart": register.CreateNativeFuncWrap(TestRHCOSGrowfs, []string{"fcos"}...), + "FCOSGrowpart": register.CreateNativeFuncWrap(TestFCOSGrowfs, []string{"rhcos"}...), + }, + }) + + register.RegisterTest(®ister.Test{ + Name: "basic.uefi", + Description: "Verify basic functionalities like SSH, systemd services, useradd, etc, with uefi enabled", + ExcludeFirmwares: []string{uefiSecure}, + Run: LocalTests, + ClusterSize: 1, NativeFuncs: map[string]register.NativeFuncWrap{ "PortSSH": register.CreateNativeFuncWrap(TestPortSsh), "DbusPerms": register.CreateNativeFuncWrap(TestDbusPerms), @@ -54,6 +76,82 @@ func init() { "FCOSGrowpart": register.CreateNativeFuncWrap(TestFCOSGrowfs, []string{"rhcos"}...), }, }) + + register.RegisterTest(®ister.Test{ + Name: "basic.uefi.secure", + Description: "Verify basic functionalities like SSH, systemd services, useradd, etc, with uefi enabled", + ExcludeFirmwares: []string{uefi}, + Run: LocalTests, + ClusterSize: 1, + NativeFuncs: map[string]register.NativeFuncWrap{ + "PortSSH": register.CreateNativeFuncWrap(TestPortSsh), + "DbusPerms": register.CreateNativeFuncWrap(TestDbusPerms), + "ServicesActive": register.CreateNativeFuncWrap(TestServicesActive), + "ReadOnly": register.CreateNativeFuncWrap(TestReadOnlyFs), + "Useradd": register.CreateNativeFuncWrap(TestUseradd), + "MachineID": register.CreateNativeFuncWrap(TestMachineID), + "RHCOSGrowpart": register.CreateNativeFuncWrap(TestRHCOSGrowfs, []string{"fcos"}...), + "FCOSGrowpart": register.CreateNativeFuncWrap(TestFCOSGrowfs, []string{"rhcos"}...), + }, + }) + + register.RegisterTest(®ister.Test{ + Name: "basic.nvme.bios", + Description: "Verify basic functionalities like SSH, systemd services, useradd, etc, with nvme enabled", + ExcludeFirmwares: []string{uefi, uefiSecure}, + Nvme: true, + Run: LocalTests, + ClusterSize: 1, + NativeFuncs: map[string]register.NativeFuncWrap{ + "PortSSH": register.CreateNativeFuncWrap(TestPortSsh), + "DbusPerms": register.CreateNativeFuncWrap(TestDbusPerms), + "ServicesActive": register.CreateNativeFuncWrap(TestServicesActive), + "ReadOnly": register.CreateNativeFuncWrap(TestReadOnlyFs), + "Useradd": register.CreateNativeFuncWrap(TestUseradd), + "MachineID": register.CreateNativeFuncWrap(TestMachineID), + "RHCOSGrowpart": register.CreateNativeFuncWrap(TestRHCOSGrowfs, []string{"fcos"}...), + "FCOSGrowpart": register.CreateNativeFuncWrap(TestFCOSGrowfs, []string{"rhcos"}...), + }, + }) + + register.RegisterTest(®ister.Test{ + Name: "basic.nvme.uefi", + Description: "Verify basic functionalities like SSH, systemd services, useradd, etc, with nvme and uefi enabled", + ExcludeFirmwares: []string{uefiSecure}, + Nvme: true, + Run: LocalTests, + ClusterSize: 1, + NativeFuncs: map[string]register.NativeFuncWrap{ + "PortSSH": register.CreateNativeFuncWrap(TestPortSsh), + "DbusPerms": register.CreateNativeFuncWrap(TestDbusPerms), + "ServicesActive": register.CreateNativeFuncWrap(TestServicesActive), + "ReadOnly": register.CreateNativeFuncWrap(TestReadOnlyFs), + "Useradd": register.CreateNativeFuncWrap(TestUseradd), + "MachineID": register.CreateNativeFuncWrap(TestMachineID), + "RHCOSGrowpart": register.CreateNativeFuncWrap(TestRHCOSGrowfs, []string{"fcos"}...), + "FCOSGrowpart": register.CreateNativeFuncWrap(TestFCOSGrowfs, []string{"rhcos"}...), + }, + }) + + register.RegisterTest(®ister.Test{ + Name: "basic.nvme.uefi.secure", + Description: "Verify basic functionalities like SSH, systemd services, useradd, etc, with nvme and uefi-secure enabled", + ExcludeFirmwares: []string{uefi}, + Nvme: true, + Run: LocalTests, + ClusterSize: 1, + NativeFuncs: map[string]register.NativeFuncWrap{ + "PortSSH": register.CreateNativeFuncWrap(TestPortSsh), + "DbusPerms": register.CreateNativeFuncWrap(TestDbusPerms), + "ServicesActive": register.CreateNativeFuncWrap(TestServicesActive), + "ReadOnly": register.CreateNativeFuncWrap(TestReadOnlyFs), + "Useradd": register.CreateNativeFuncWrap(TestUseradd), + "MachineID": register.CreateNativeFuncWrap(TestMachineID), + "RHCOSGrowpart": register.CreateNativeFuncWrap(TestRHCOSGrowfs, []string{"fcos"}...), + "FCOSGrowpart": register.CreateNativeFuncWrap(TestFCOSGrowfs, []string{"rhcos"}...), + }, + }) + // TODO: Enable DockerPing/DockerEcho once fixed // TODO: Only enable PodmanPing on non qemu. Needs: // https://github.com/coreos/mantle/issues/1132 diff --git a/src/cmd-kola b/src/cmd-kola index 2fe2a20dcf..f0c489e95c 100755 --- a/src/cmd-kola +++ b/src/cmd-kola @@ -10,8 +10,6 @@ import sys # Just test these boot to start with. In the future we should at least # do ostree upgrades with uefi etc. But we don't really need the *full* # suite...if podman somehow broke with nvme or uefi I'd be amazed and impressed. -BASIC_SCENARIOS = ["nvme=true", "firmware=uefi"] -BASIC_SCENARIOS_SECURE_BOOT = ["firmware=uefi-secure"] arch = platform.machine() cosa_dir = os.path.dirname(os.path.abspath(__file__)) @@ -24,9 +22,7 @@ basearch = cmdlib.get_basearch() # Parse args and dispatch parser = argparse.ArgumentParser() parser.add_argument("--build", help="Build ID") -parser.add_argument("--basic-qemu-scenarios", help="Run the basic test across uefi-secure,nvme etc.", action='store_true') parser.add_argument("--output-dir", help="Output directory") -parser.add_argument("--skip-secure-boot", help="Use with '--basic-qemu-scenarios' to skip the Secure Boot tests", action='store_true') parser.add_argument("--upgrades", help="Run upgrade tests", action='store_true') parser.add_argument("subargs", help="Remaining arguments for kola", nargs='*', default=[]) @@ -58,33 +54,8 @@ subargs = args.subargs or [default_cmd] kolaargs.extend(subargs) kolaargs.extend(unknown_args) -if args.basic_qemu_scenarios: - if arch == "x86_64": - os.mkdir(outputdir) # Create the toplevel output dir - for scenario in BASIC_SCENARIOS: - kolaargs.extend(['--output-dir', - os.path.join(outputdir, scenario.replace('=', '-'))]) - subargs = kolaargs + ['--qemu-' + scenario, 'basic'] - print(subprocess.list2cmdline(subargs), flush=True) - subprocess.check_call(subargs) - if not args.skip_secure_boot: - for scenario in BASIC_SCENARIOS_SECURE_BOOT: - kolaargs.extend(['--output-dir', - os.path.join(outputdir, scenario.replace('=', '-'))]) - # See https://issues.redhat.com/browse/COS-2000 - there's - # some bug with shim/grub2 that fails with secure boot on < ~1300MiB of RAM. - # But we're not going to block on that; real world OCP worker nodes are at least 16GiB etc. - subargs = kolaargs + ['--qemu-' + scenario, 'basic'] + ["--qemu-memory", "1536"] - print(subprocess.list2cmdline(subargs), flush=True) - subprocess.check_call(subargs) - else: - # Basic qemu scenarios using nvme and uefi - # are not supported on multi-arch - kolaargs.extend(['--output-dir', outputdir]) - subargs = kolaargs + ['basic'] - print(subprocess.list2cmdline(subargs), flush=True) - subprocess.check_call(subargs) -elif args.upgrades: + +if args.upgrades: kolaargs.extend(['--output-dir', outputdir]) if '--qemu-image-dir' not in unknown_args: os.makedirs('tmp/kola-qemu-cache', exist_ok=True)