From 2ba8878e6ed33b89599c501d7986265d6d48d4f2 Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Thu, 5 Sep 2024 13:15:01 -0600 Subject: [PATCH 1/2] test: add deprecation support to verify_clean_boot (#5671) Also verify that cloud-init status has a valid return code. --- tests/integration_tests/util.py | 152 +++++++++++++++++++++++++------- 1 file changed, 122 insertions(+), 30 deletions(-) diff --git a/tests/integration_tests/util.py b/tests/integration_tests/util.py index 8da16cb7bec..7c8055830d2 100644 --- a/tests/integration_tests/util.py +++ b/tests/integration_tests/util.py @@ -64,16 +64,20 @@ def _format_found(header: str, items: list) -> str: def verify_clean_boot( instance: "IntegrationInstance", + *, + ignore_deprecations: Optional[Union[List[str], bool]] = None, ignore_warnings: Optional[Union[List[str], bool]] = None, ignore_errors: Optional[Union[List[str], bool]] = None, ignore_tracebacks: Optional[Union[List[str], bool]] = None, + require_deprecations: Optional[list] = None, require_warnings: Optional[list] = None, require_errors: Optional[list] = None, ): - """raise assertions if the client experienced unexpected warnings or errors + """Raise exception if the client experienced unexpected conditions. - Fail when a required error isn't found. - Expected warnings and errors are defined in this function. + Fail when a required error, warning, or deprecation isn't found. + + Platform-specific conditions are defined in this function. This function is similar to verify_clean_log, hence the similar name. @@ -84,13 +88,17 @@ def verify_clean_boot( - less resource intensive (no log copying required) - nice error formatting - instance: test instance - ignored_warnings: list of expected warnings to ignore, - or true to ignore all - ignored_errors: list of expected errors to ignore, or true to ignore all - require_warnings: Optional[list] = None, - require_errors: Optional[list] = None, - fail_when_expected_not_found: optional list of expected errors + :param instance: test instance + :param ignored_deprecations: list of deprecation messages to ignore, or + true to ignore all + :param ignored_warnings: list of expected warnings to ignore, or true to + ignore all + :param ignored_errors: list of expected errors to ignore, or true to ignore + all + :param require_deprecations: list of expected deprecation messages to + require + :param require_warnings: list of expected warning messages to require + :param require_errors: list of expected error messages to require """ def append_or_create_list( @@ -150,9 +158,11 @@ def append_or_create_list( _verify_clean_boot( instance, + ignore_deprecations=ignore_deprecations, ignore_warnings=ignore_warnings, ignore_errors=ignore_errors, ignore_tracebacks=ignore_tracebacks, + require_deprecations=require_deprecations, require_warnings=require_warnings, require_errors=require_errors, ) @@ -160,21 +170,27 @@ def append_or_create_list( def _verify_clean_boot( instance: "IntegrationInstance", + ignore_deprecations: Optional[Union[List[str], bool]] = None, ignore_warnings: Optional[Union[List[str], bool]] = None, ignore_errors: Optional[Union[List[str], bool]] = None, ignore_tracebacks: Optional[Union[List[str], bool]] = None, + require_deprecations: Optional[list] = None, require_warnings: Optional[list] = None, require_errors: Optional[list] = None, ): + ignore_deprecations = ignore_deprecations or [] ignore_errors = ignore_errors or [] ignore_warnings = ignore_warnings or [] + require_deprecations = require_deprecations or [] require_errors = require_errors or [] require_warnings = require_warnings or [] status = json.loads(instance.execute("cloud-init status --format=json")) + unexpected_deprecations = set() unexpected_errors = set() unexpected_warnings = set() + required_deprecations_found = set() required_warnings_found = set() required_errors_found = set() @@ -211,16 +227,42 @@ def _verify_clean_boot( else: unexpected_warnings.add(current_warning) + # check for unexpected deprecations + for current_deprecation in status["recoverable_errors"].get( + "DEPRECATED", [] + ): + + # check for required deprecations + for expected in require_deprecations: + if expected in current_deprecation: + required_deprecations_found.add(expected) + + if ignore_deprecations is True: + continue + # check for unexpected deprecations + for expected in [*ignore_deprecations, *require_deprecations]: + if expected in current_deprecation: + break + else: + unexpected_deprecations.add(current_deprecation) + required_errors_not_found = set(require_errors) - required_errors_found required_warnings_not_found = ( set(require_warnings) - required_warnings_found ) + required_deprecations_not_found = ( + set(require_deprecations) - required_deprecations_found + ) + # ordered from most severe to least severe + # this allows the first message printed to be the most significant errors = [ *unexpected_errors, *required_errors_not_found, *unexpected_warnings, *required_warnings_not_found, + *unexpected_deprecations, + *required_deprecations_not_found, ] if errors: message = "" @@ -243,29 +285,79 @@ def _verify_clean_boot( message += _format_found( "Required warnings not found", list(required_warnings_not_found) ) + message += _format_found( + "Required deprecations not found", + list(required_deprecations_not_found), + ) assert not errors, message - if ignore_tracebacks is True: - return # assert no unexpected Tracebacks - expected_traceback_count = 0 - traceback_count = int( - instance.execute( - "grep --count Traceback /var/log/cloud-init.log" - ).stdout.strip() - ) - if ignore_tracebacks: - for expected_traceback in ignore_tracebacks: - expected_traceback_count += int( - instance.execute( - f"grep --count '{expected_traceback}'" - " /var/log/cloud-init.log" - ).stdout.strip() - ) - assert expected_traceback_count == traceback_count, ( - f"{traceback_count - expected_traceback_count} unexpected traceback(s)" - " found in /var/log/cloud-init.log" - ) + if ignore_tracebacks is not True: + expected_traceback_count = 0 + traceback_count = int( + instance.execute( + "grep --count Traceback /var/log/cloud-init.log" + ).stdout.strip() + ) + if ignore_tracebacks: + for expected_traceback in ignore_tracebacks: + expected_traceback_count += int( + instance.execute( + f"grep --count '{expected_traceback}'" + " /var/log/cloud-init.log" + ).stdout.strip() + ) + assert expected_traceback_count == traceback_count, ( + f"{traceback_count - expected_traceback_count} unexpected " + "traceback(s) found in /var/log/cloud-init.log" + ) + + # check that the return code of cloud-init status is expected + if not any( + [ + ignore_deprecations, + ignore_warnings, + ignore_errors, + ignore_tracebacks, + require_deprecations, + require_warnings, + require_errors, + ] + ): + # make sure that return code is 0 when all was good + out = instance.execute("cloud-init status --long") + assert out.ok, ( + "Unexpected non-zero return code from " + f"`cloud-init status`: {out.return_code}\nstdout: " + f"{out.stdout}\nstderr: {out.stderr}" + ) + elif any( + [ + ignore_deprecations, + ignore_warnings, + ignore_errors, + ignore_tracebacks, + ] + ): + # Ignore is optional, so we can't be sure whether we will got a return + # code of 0 or 2. This makes this function more useful when writing + # tests that run on series with different behavior. In this case we + # cannot be sure whether it will return 0 or 2, but we never want to + # see a return code of 1, so assert that it isn't 1. + out = instance.execute("cloud-init status --long") + assert 1 != out.return_code, ( + f"Unexpected return code from `cloud-init status`. Expected rc=0 " + f"or rc=2, received rc={out.return_code}\nstdout: " + f"{out.stdout}\nstderr: {out.stderr}" + ) + else: + # we know that we should have a return code of 2 + out = instance.execute("cloud-init status --long") + assert 2 == out.return_code, ( + f"Unexpected return code from `cloud-init status`. " + f"Expected rc=2, received rc={out.return_code}\nstdout: " + f"{out.stdout}\nstderr: {out.stderr}" + ) schema = instance.execute("cloud-init schema --system --annotate") assert schema.ok, ( f"Schema validation failed\nstdout:{schema.stdout}" From 75e9ecf24dce5c92c937713623242ec893453d7d Mon Sep 17 00:00:00 2001 From: Brett Holman Date: Thu, 5 Sep 2024 15:31:00 -0600 Subject: [PATCH 2/2] test: add verify_clean_boot() calls alongside verify_clean_log() (#5671) --- tests/integration_tests/bugs/test_gh632.py | 3 +- tests/integration_tests/bugs/test_gh868.py | 3 +- .../integration_tests/bugs/test_lp1813396.py | 2 ++ .../integration_tests/bugs/test_lp1886531.py | 3 +- .../integration_tests/bugs/test_lp1898997.py | 3 +- tests/integration_tests/cmd/test_schema.py | 17 +++++++---- .../datasources/test_lxd_discovery.py | 7 ++++- .../datasources/test_nocloud.py | 15 ++++++---- .../datasources/test_none.py | 4 ++- .../datasources/test_oci_networking.py | 3 +- .../datasources/test_tmp_noexec.py | 3 +- .../integration_tests/modules/test_ansible.py | 3 ++ .../modules/test_apt_functionality.py | 2 ++ .../modules/test_boothook.py | 3 +- .../modules/test_ca_certs.py | 7 ++++- .../modules/test_combined.py | 24 ++++++++------- .../modules/test_disk_setup.py | 5 +++- .../integration_tests/modules/test_hotplug.py | 5 ++++ .../modules/test_jinja_templating.py | 2 ++ tests/integration_tests/modules/test_lxd.py | 5 +++- .../test_package_update_upgrade_install.py | 3 +- .../integration_tests/modules/test_puppet.py | 3 +- .../modules/test_ubuntu_drivers.py | 3 +- .../modules/test_ubuntu_pro.py | 18 ++++++++---- .../modules/test_version_change.py | 29 +++++++++++-------- tests/integration_tests/net/test_dhcp.py | 5 +++- .../reporting/test_webhook_reporting.py | 7 ++++- tests/integration_tests/test_ds_identify.py | 7 ++++- tests/integration_tests/test_networking.py | 4 ++- tests/integration_tests/test_paths.py | 3 +- tests/integration_tests/test_upgrade.py | 1 + 31 files changed, 142 insertions(+), 60 deletions(-) diff --git a/tests/integration_tests/bugs/test_gh632.py b/tests/integration_tests/bugs/test_gh632.py index bd26e6b39d2..74150bc4eaa 100644 --- a/tests/integration_tests/bugs/test_gh632.py +++ b/tests/integration_tests/bugs/test_gh632.py @@ -8,7 +8,7 @@ from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import verify_clean_boot, verify_clean_log # With some datasource hacking, we can run this on a NoCloud instance @@ -30,6 +30,7 @@ def test_datasource_rbx_no_stacktrace(client: IntegrationInstance): log = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log) + verify_clean_boot(client) assert "Failed to load metadata and userdata" not in log assert ( "Getting data from /dev/null | wc -l" ).stdout diff --git a/tests/integration_tests/bugs/test_lp1886531.py b/tests/integration_tests/bugs/test_lp1886531.py index d170a133d35..b1c512e30c2 100644 --- a/tests/integration_tests/bugs/test_lp1886531.py +++ b/tests/integration_tests/bugs/test_lp1886531.py @@ -12,7 +12,7 @@ import pytest -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import verify_clean_boot, verify_clean_log USER_DATA = """\ #cloud-config @@ -26,3 +26,4 @@ class TestLp1886531: def test_lp1886531(self, client): log_content = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log_content) + verify_clean_boot(client) diff --git a/tests/integration_tests/bugs/test_lp1898997.py b/tests/integration_tests/bugs/test_lp1898997.py index d183223b9ac..5baff84b1e9 100644 --- a/tests/integration_tests/bugs/test_lp1898997.py +++ b/tests/integration_tests/bugs/test_lp1898997.py @@ -15,7 +15,7 @@ from tests.integration_tests import random_mac_address from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.releases import CURRENT_RELEASE, FOCAL -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import verify_clean_boot, verify_clean_log MAC_ADDRESS = random_mac_address() @@ -74,6 +74,7 @@ def test_ovs_member_interfaces_not_excluded(self, client): # Confirm that the network configuration was applied successfully verify_clean_log(cloudinit_output) + verify_clean_boot(client) # Confirm that the applied network config created the OVS bridge assert "ovs-br" in client.execute("ip addr") diff --git a/tests/integration_tests/cmd/test_schema.py b/tests/integration_tests/cmd/test_schema.py index 4654d7eec92..f03b3377586 100644 --- a/tests/integration_tests/cmd/test_schema.py +++ b/tests/integration_tests/cmd/test_schema.py @@ -9,6 +9,7 @@ from tests.integration_tests.releases import CURRENT_RELEASE, MANTIC from tests.integration_tests.util import ( get_feature_flag_value, + verify_clean_boot, verify_clean_log, ) @@ -70,16 +71,20 @@ def test_clean_log(self, class_client: IntegrationInstance): version_boundary = get_feature_flag_value( class_client, "DEPRECATION_INFO_BOUNDARY" ) + boundary_message = "Deprecated cloud-config provided:" + messages = [ + "apt_reboot_if_required: Deprecated ", + "apt_update: Deprecated in version", + "apt_upgrade: Deprecated in version", + ] # the deprecation_version is 22.2 in schema for apt_* keys in # user-data. Pass 22.2 in against the client's version_boundary. if lifecycle.should_log_deprecation("22.2", version_boundary): - log_level = "DEPRECATED" + messages += boundary_message + verify_clean_boot(class_client, require_deprecations=messages) else: - log_level = "INFO" - assert f"{log_level}]: Deprecated cloud-config provided:" in log - assert "apt_reboot_if_required: Deprecated " in log - assert "apt_update: Deprecated in version" in log - assert "apt_upgrade: Deprecated in version" in log + verify_clean_boot(class_client, require_deprecations=messages) + assert boundary_message in log def test_network_config_schema_validation( self, class_client: IntegrationInstance diff --git a/tests/integration_tests/datasources/test_lxd_discovery.py b/tests/integration_tests/datasources/test_lxd_discovery.py index 4f9074345ff..43a4ca78318 100644 --- a/tests/integration_tests/datasources/test_lxd_discovery.py +++ b/tests/integration_tests/datasources/test_lxd_discovery.py @@ -6,7 +6,11 @@ from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU -from tests.integration_tests.util import lxd_has_nocloud, verify_clean_log +from tests.integration_tests.util import ( + lxd_has_nocloud, + verify_clean_boot, + verify_clean_log, +) def _customize_environment(client: IntegrationInstance): @@ -75,6 +79,7 @@ def test_lxd_datasource_discovery(client: IntegrationInstance): } == netplan_cfg log = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log) + verify_clean_boot(client) result = client.execute("cloud-id") if result.stdout != "lxd": raise AssertionError( diff --git a/tests/integration_tests/datasources/test_nocloud.py b/tests/integration_tests/datasources/test_nocloud.py index f6659d45d17..2b2c7a25acb 100644 --- a/tests/integration_tests/datasources/test_nocloud.py +++ b/tests/integration_tests/datasources/test_nocloud.py @@ -198,15 +198,18 @@ def test_smbios_seed_network(self, client: IntegrationInstance): version_boundary = get_feature_flag_value( client, "DEPRECATION_INFO_BOUNDARY" ) + message = ( + "The 'nocloud-net' datasource name is " + 'deprecated" /var/log/cloud-init.log' + ) # nocloud-net deprecated in version 24.1 if lifecycle.should_log_deprecation("24.1", version_boundary): - log_level = "DEPRECATED" + verify_clean_boot(client, require_deprecations=[message]) else: - log_level = "INFO" - client.execute( - rf"grep \"{log_level}]: The 'nocloud-net' datasource name is" - ' deprecated" /var/log/cloud-init.log' - ).ok + client.execute( + r"grep \"INFO]: The 'nocloud-net' datasource name is" + ' deprecated" /var/log/cloud-init.log' + ).ok @pytest.mark.skipif(PLATFORM != "lxd_vm", reason="Modifies grub config") diff --git a/tests/integration_tests/datasources/test_none.py b/tests/integration_tests/datasources/test_none.py index d79c30404d8..8d12842767d 100644 --- a/tests/integration_tests/datasources/test_none.py +++ b/tests/integration_tests/datasources/test_none.py @@ -3,7 +3,7 @@ import json from tests.integration_tests.instances import IntegrationInstance -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import verify_clean_boot, verify_clean_log DS_NONE_BASE_CFG = """\ datasource_list: [None] @@ -26,6 +26,7 @@ def test_datasource_none_discovery(client: IntegrationInstance): """ log = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log) + verify_clean_boot(client) # Limit datasource detection to DataSourceNone. client.write_to_file( "/etc/cloud/cloud.cfg.d/99-force-dsnone.cfg", DS_NONE_BASE_CFG @@ -65,4 +66,5 @@ def test_datasource_none_discovery(client: IntegrationInstance): ) log = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log) + verify_clean_boot(client, require_warnings=expected_warnings) assert client.execute("test -f /var/tmp/success-with-datasource-none").ok diff --git a/tests/integration_tests/datasources/test_oci_networking.py b/tests/integration_tests/datasources/test_oci_networking.py index 802e8ec7d51..9b807927668 100644 --- a/tests/integration_tests/datasources/test_oci_networking.py +++ b/tests/integration_tests/datasources/test_oci_networking.py @@ -7,7 +7,7 @@ from tests.integration_tests.clouds import IntegrationCloud from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import verify_clean_boot, verify_clean_log DS_CFG = """\ datasource: @@ -52,6 +52,7 @@ def test_oci_networking_iscsi_instance(client: IntegrationInstance, tmpdir): log = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log) + verify_clean_boot(client) assert ( "opc/v2/vnics/" not in log diff --git a/tests/integration_tests/datasources/test_tmp_noexec.py b/tests/integration_tests/datasources/test_tmp_noexec.py index 5aa8537d420..e45a77115c0 100644 --- a/tests/integration_tests/datasources/test_tmp_noexec.py +++ b/tests/integration_tests/datasources/test_tmp_noexec.py @@ -2,7 +2,7 @@ from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import verify_clean_boot, verify_clean_log def customize_client(client: IntegrationInstance): @@ -30,3 +30,4 @@ def test_dhcp_tmp_noexec(client: IntegrationInstance): not in log ) verify_clean_log(log) + verify_clean_boot(client) diff --git a/tests/integration_tests/modules/test_ansible.py b/tests/integration_tests/modules/test_ansible.py index ba71bc47ab6..974f86fb4fb 100644 --- a/tests/integration_tests/modules/test_ansible.py +++ b/tests/integration_tests/modules/test_ansible.py @@ -5,6 +5,7 @@ from tests.integration_tests.releases import CURRENT_RELEASE, FOCAL from tests.integration_tests.util import ( push_and_enable_systemd_unit, + verify_clean_boot, verify_clean_log, ) @@ -267,6 +268,7 @@ def _test_ansible_pull_from_local_server(my_client): my_client.restart() log = my_client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log) + verify_clean_boot(my_client) output_log = my_client.read_from_file("/var/log/cloud-init-output.log") assert "ok=3" in output_log assert "SUCCESS: config-ansible ran successfully" in log @@ -320,6 +322,7 @@ def test_ansible_pull_distro(client): def test_ansible_controller(client): log = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log) + verify_clean_boot(client) content_ansible = client.execute( "lxc exec lxd-container-00 -- cat /home/ansible/ansible.txt" ) diff --git a/tests/integration_tests/modules/test_apt_functionality.py b/tests/integration_tests/modules/test_apt_functionality.py index 2af9e590ce0..26c4d9523b6 100644 --- a/tests/integration_tests/modules/test_apt_functionality.py +++ b/tests/integration_tests/modules/test_apt_functionality.py @@ -13,6 +13,7 @@ from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU from tests.integration_tests.util import ( get_feature_flag_value, + verify_clean_boot, verify_clean_log, ) @@ -489,4 +490,5 @@ def test_install_missing_deps(setup_image, session_cloud: IntegrationCloud): ) as minimal_client: log = minimal_client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log) + verify_clean_boot(minimal_client) assert re.search(RE_GPG_SW_PROPERTIES_INSTALLED, log) diff --git a/tests/integration_tests/modules/test_boothook.py b/tests/integration_tests/modules/test_boothook.py index e2a289b4f29..e660ab9f70a 100644 --- a/tests/integration_tests/modules/test_boothook.py +++ b/tests/integration_tests/modules/test_boothook.py @@ -4,7 +4,7 @@ import pytest from tests.integration_tests.instances import IntegrationInstance -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import verify_clean_boot, verify_clean_log USER_DATA = """\ #cloud-boothook @@ -25,6 +25,7 @@ def test_boothook_header_runs_part_per_instance(client: IntegrationInstance): RE_BOOTHOOK = f"BOOTHOOK: {instance_id}: is called every boot" log = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log) + verify_clean_boot(client) output = client.read_from_file("/boothook.txt") assert 1 == len(re.findall(RE_BOOTHOOK, output)) client.restart() diff --git a/tests/integration_tests/modules/test_ca_certs.py b/tests/integration_tests/modules/test_ca_certs.py index 0b84c9b9fe4..ef8881644a8 100644 --- a/tests/integration_tests/modules/test_ca_certs.py +++ b/tests/integration_tests/modules/test_ca_certs.py @@ -14,7 +14,11 @@ from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.releases import IS_UBUNTU -from tests.integration_tests.util import get_inactive_modules, verify_clean_log +from tests.integration_tests.util import ( + get_inactive_modules, + verify_clean_boot, + verify_clean_log, +) CERT_CONTENT = """\ -----BEGIN CERTIFICATE----- @@ -106,6 +110,7 @@ def test_clean_log(self, class_client: IntegrationInstance): """ log = class_client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log, ignore_deprecations=False) + verify_clean_boot(class_client) expected_inactive = { "apt_pipelining", diff --git a/tests/integration_tests/modules/test_combined.py b/tests/integration_tests/modules/test_combined.py index 2d8b51ee362..759c9528e62 100644 --- a/tests/integration_tests/modules/test_combined.py +++ b/tests/integration_tests/modules/test_combined.py @@ -27,6 +27,7 @@ get_feature_flag_value, get_inactive_modules, lxd_has_nocloud, + verify_clean_boot, verify_clean_log, verify_ordered_items_in_text, ) @@ -137,23 +138,26 @@ def test_deprecated_message(self, class_client: IntegrationInstance): version_boundary = get_feature_flag_value( class_client, "DEPRECATION_INFO_BOUNDARY" ) + deprecated_messages = ["users.1.sudo: Changed in version 22.2."] + boundary_message = ( + "The value of 'false' in user craig's 'sudo'" + " config is deprecated" + ) # the changed_version is 22.2 in schema for user.sudo key in # user-data. Pass 22.2 in against the client's version_boundary. if lifecycle.should_log_deprecation("22.2", version_boundary): - log_level = "DEPRECATED" - deprecation_count = 2 + deprecated_messages.append(boundary_message) + verify_clean_boot( + class_client, require_deprecations=deprecated_messages + ) else: # Expect the distros deprecated call to be redacted. # jsonschema still emits deprecation log due to changed_version # instead of deprecated_version - log_level = "INFO" - deprecation_count = 1 - - assert ( - f"[{log_level}]: The value of 'false' in user craig's 'sudo'" - " config is deprecated" in log - ) - assert deprecation_count == log.count("DEPRECATE") + verify_clean_boot( + class_client, require_deprecations=deprecated_messages + ) + assert f"[INFO]: {boundary_message}" in log def test_ntp_with_apt(self, class_client: IntegrationInstance): """LP #1628337. diff --git a/tests/integration_tests/modules/test_disk_setup.py b/tests/integration_tests/modules/test_disk_setup.py index 97be1abe2f2..27a70d32f4b 100644 --- a/tests/integration_tests/modules/test_disk_setup.py +++ b/tests/integration_tests/modules/test_disk_setup.py @@ -9,7 +9,7 @@ from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.releases import CURRENT_RELEASE, FOCAL, IS_UBUNTU -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import verify_clean_boot, verify_clean_log DISK_PATH = "/tmp/test_disk_setup_{}".format(uuid4()) @@ -69,6 +69,7 @@ def test_device_alias(self, create_disk, client: IntegrationInstance): assert "changed my_alias.1 => /dev/sdb1" in log assert "changed my_alias.2 => /dev/sdb2" in log verify_clean_log(log) + verify_clean_boot(client) lsblk = json.loads(client.execute("lsblk --json")) sdb = [x for x in lsblk["blockdevices"] if x["name"] == "sdb"][0] @@ -143,6 +144,7 @@ class TestPartProbeAvailability: def _verify_first_disk_setup(self, client, log): verify_clean_log(log) + verify_clean_boot(client) lsblk = json.loads(client.execute("lsblk --json")) sdb = [x for x in lsblk["blockdevices"] if x["name"] == "sdb"][0] assert len(sdb["children"]) == 2 @@ -200,6 +202,7 @@ def test_disk_setup_when_mounted( # Assert new setup works as expected verify_clean_log(log) + verify_clean_boot(client) lsblk = json.loads(client.execute("lsblk --json")) sdb = [x for x in lsblk["blockdevices"] if x["name"] == "sdb"][0] diff --git a/tests/integration_tests/modules/test_hotplug.py b/tests/integration_tests/modules/test_hotplug.py index c088240de1a..a41b2313cde 100644 --- a/tests/integration_tests/modules/test_hotplug.py +++ b/tests/integration_tests/modules/test_hotplug.py @@ -16,6 +16,7 @@ ) from tests.integration_tests.util import ( push_and_enable_systemd_unit, + verify_clean_boot, verify_clean_log, wait_for_cloud_init, ) @@ -256,6 +257,7 @@ def test_multi_nic_hotplug(setup_image, session_cloud: IntegrationCloud): log_content = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log_content) + verify_clean_boot(client) ips_after_add = _get_ip_addr(client) @@ -297,6 +299,7 @@ def test_multi_nic_hotplug(setup_image, session_cloud: IntegrationCloud): log_content = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log_content) + verify_clean_boot(client) @pytest.mark.skipif(CURRENT_RELEASE <= FOCAL, reason="See LP: #2055397") @@ -317,6 +320,7 @@ def test_multi_nic_hotplug_vpc(setup_image, session_cloud: IntegrationCloud): _wait_till_hotplug_complete(client) log_content = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log_content) + verify_clean_boot(client) netplan_cfg = client.read_from_file("/etc/netplan/50-cloud-init.yaml") config = yaml.safe_load(netplan_cfg) @@ -359,6 +363,7 @@ def test_multi_nic_hotplug_vpc(setup_image, session_cloud: IntegrationCloud): log_content = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log_content) + verify_clean_boot(client) @pytest.mark.skipif(PLATFORM != "ec2", reason="test is ec2 specific") diff --git a/tests/integration_tests/modules/test_jinja_templating.py b/tests/integration_tests/modules/test_jinja_templating.py index 93810c76153..59c0de358c0 100644 --- a/tests/integration_tests/modules/test_jinja_templating.py +++ b/tests/integration_tests/modules/test_jinja_templating.py @@ -4,6 +4,7 @@ from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.util import ( + verify_clean_boot, verify_clean_log, verify_ordered_items_in_text, ) @@ -72,6 +73,7 @@ def test_substitution_in_etc_cloud(client: IntegrationInstance): log = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log) + verify_clean_boot(client) # Ensure /etc/cloud/cloud.cfg template works as expected hostname = client.execute("hostname").stdout.strip() diff --git a/tests/integration_tests/modules/test_lxd.py b/tests/integration_tests/modules/test_lxd.py index a4ff5906a23..44814e67cc6 100644 --- a/tests/integration_tests/modules/test_lxd.py +++ b/tests/integration_tests/modules/test_lxd.py @@ -13,7 +13,7 @@ from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.releases import CURRENT_RELEASE, FOCAL -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import verify_clean_boot, verify_clean_log BRIDGE_USER_DATA = """\ #cloud-config @@ -166,6 +166,7 @@ def test_bridge(self, class_client): """Check that the given bridge is configured""" cloud_init_log = class_client.read_from_file("/var/log/cloud-init.log") verify_clean_log(cloud_init_log) + verify_clean_boot(class_client) # The bridge should exist assert class_client.execute("ip addr show lxdbr0").ok @@ -178,6 +179,7 @@ def test_bridge(self, class_client): def validate_storage(validate_client, pkg_name, command): log = validate_client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log, ignore_deprecations=False) + verify_clean_boot(validate_client) return log @@ -289,6 +291,7 @@ def test_basic_preseed(client): preseed_cfg = yaml.safe_load(preseed_cfg) cloud_init_log = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(cloud_init_log) + verify_clean_boot(client) validate_preseed_profiles(client, preseed_cfg) validate_preseed_storage_pools(client, preseed_cfg) validate_preseed_projects(client, preseed_cfg) diff --git a/tests/integration_tests/modules/test_package_update_upgrade_install.py b/tests/integration_tests/modules/test_package_update_upgrade_install.py index 7da54054263..7e84104f40b 100644 --- a/tests/integration_tests/modules/test_package_update_upgrade_install.py +++ b/tests/integration_tests/modules/test_package_update_upgrade_install.py @@ -12,7 +12,7 @@ from tests.integration_tests.clouds import IntegrationCloud from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import verify_clean_boot, verify_clean_log USER_DATA = """\ #cloud-config @@ -146,6 +146,7 @@ def test_versioned_packages_are_installed(session_cloud: IntegrationCloud): user_data=VERSIONED_USER_DATA.format(pkg_version=pkg_version) ) as client: verify_clean_log(client.read_from_file("/var/log/cloud-init.log")) + verify_clean_boot(client) assert f"hello {pkg_version}" == client.execute( "dpkg-query -W hello" ), ( diff --git a/tests/integration_tests/modules/test_puppet.py b/tests/integration_tests/modules/test_puppet.py index 9598b8ec971..640ffe1fe20 100644 --- a/tests/integration_tests/modules/test_puppet.py +++ b/tests/integration_tests/modules/test_puppet.py @@ -3,7 +3,7 @@ import pytest from tests.integration_tests.instances import IntegrationInstance -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import verify_clean_boot, verify_clean_log SERVICE_DATA = """\ #cloud-config @@ -18,6 +18,7 @@ def test_puppet_service(client: IntegrationInstance): """Basic test that puppet gets installed and runs.""" log = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log) + verify_clean_boot(client) puppet_ok = client.execute("systemctl is-active puppet.service").ok puppet_agent_ok = client.execute( "systemctl is-active puppet-agent.service" diff --git a/tests/integration_tests/modules/test_ubuntu_drivers.py b/tests/integration_tests/modules/test_ubuntu_drivers.py index 66649c17f05..ffa19ac1282 100644 --- a/tests/integration_tests/modules/test_ubuntu_drivers.py +++ b/tests/integration_tests/modules/test_ubuntu_drivers.py @@ -4,7 +4,7 @@ from tests.integration_tests.clouds import IntegrationCloud from tests.integration_tests.integration_settings import PLATFORM -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import verify_clean_boot, verify_clean_log USER_DATA = """\ #cloud-config @@ -24,6 +24,7 @@ def test_ubuntu_drivers_installed(session_cloud: IntegrationCloud): ) as client: log = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log) + verify_clean_boot(client) assert 1 == log.count( "Installing and activating NVIDIA drivers " "(nvidia/license-accepted=True, version=latest)" diff --git a/tests/integration_tests/modules/test_ubuntu_pro.py b/tests/integration_tests/modules/test_ubuntu_pro.py index 0f0cb944aec..c9fe6af0757 100644 --- a/tests/integration_tests/modules/test_ubuntu_pro.py +++ b/tests/integration_tests/modules/test_ubuntu_pro.py @@ -22,6 +22,7 @@ ) from tests.integration_tests.util import ( get_feature_flag_value, + verify_clean_boot, verify_clean_log, ) @@ -142,21 +143,22 @@ def test_valid_token(self, client: IntegrationInstance): version_boundary = get_feature_flag_value( client, "DEPRECATION_INFO_BOUNDARY" ) + boundary_message = ( + "Module has been renamed from cc_ubuntu_advantage " + "to cc_ubuntu_pro /var/log/cloud-init.log" + ) # ubuntu_advantage key is deprecated in version 24.1 if lifecycle.should_log_deprecation("24.1", version_boundary): - log_level = "DEPRECATED" + verify_clean_boot(client, require_deprecations=[boundary_message]) else: - log_level = "INFO" - client.execute( - rf"grep \"{log_level}]: Module has been renamed from" - " cc_ubuntu_advantage to cc_ubuntu_pro /var/log/cloud-init.log" - ).ok + client.execute(rf"grep \"INFO]: {boundary_message}").ok assert is_attached(client) @pytest.mark.user_data(ATTACH.format(token=CLOUD_INIT_UA_TOKEN)) def test_idempotency(self, client: IntegrationInstance): log = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log) + verify_clean_boot(client) assert is_attached(client) # Clean reboot to change instance-id and trigger cc_ua in next boot @@ -165,6 +167,7 @@ def test_idempotency(self, client: IntegrationInstance): log = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log) + verify_clean_boot(client) assert is_attached(client) # Assert service-already-enabled handling for esm-infra. @@ -178,6 +181,7 @@ def test_idempotency(self, client: IntegrationInstance): client.restart() log = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log) + verify_clean_boot(client) assert "Service `esm-infra` already enabled" in log @@ -209,6 +213,7 @@ def maybe_install_cloud_init(session_cloud: IntegrationCloud): ) as client: log = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log) + verify_clean_boot(client) assert not is_attached( client @@ -247,6 +252,7 @@ def test_custom_services(self, session_cloud: IntegrationCloud): ) as client: log = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log) + verify_clean_boot(client) assert_ua_service_noop(client) assert is_attached(client) services_status = get_services_status(client) diff --git a/tests/integration_tests/modules/test_version_change.py b/tests/integration_tests/modules/test_version_change.py index 9df436b8f6e..30e2f71501f 100644 --- a/tests/integration_tests/modules/test_version_change.py +++ b/tests/integration_tests/modules/test_version_change.py @@ -4,28 +4,33 @@ from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM -from tests.integration_tests.util import ASSETS_DIR, verify_clean_log +from tests.integration_tests.util import ( + ASSETS_DIR, + verify_clean_boot, + verify_clean_log, +) PICKLE_PATH = Path("/var/lib/cloud/instance/obj.pkl") TEST_PICKLE = ASSETS_DIR / "test_version_change.pkl" -def _assert_no_pickle_problems(log): +def _assert_no_pickle_problems(log, client): assert "Failed loading pickled blob" not in log verify_clean_log(log) + verify_clean_boot(client) def test_reboot_without_version_change(client: IntegrationInstance): log = client.read_from_file("/var/log/cloud-init.log") assert "Python version change detected" not in log assert "Cache compatibility status is currently unknown." not in log - _assert_no_pickle_problems(log) + _assert_no_pickle_problems(log, client) client.restart() log = client.read_from_file("/var/log/cloud-init.log") assert "Python version change detected" not in log assert "Could not determine Python version used to write cache" not in log - _assert_no_pickle_problems(log) + _assert_no_pickle_problems(log, client) # Now ensure that loading a bad pickle gives us problems client.push_file(TEST_PICKLE, PICKLE_PATH) @@ -58,7 +63,7 @@ def test_cache_purged_on_version_change(client: IntegrationInstance): client.restart() log = client.read_from_file("/var/log/cloud-init.log") assert "Python version change detected. Purging cache" in log - _assert_no_pickle_problems(log) + _assert_no_pickle_problems(log, client) def test_log_message_on_missing_version_file(client: IntegrationInstance): @@ -68,10 +73,10 @@ def test_log_message_on_missing_version_file(client: IntegrationInstance): client.execute("rm /var/log/cloud-init.log") client.restart() log = client.read_from_file("/var/log/cloud-init.log") - if "no cache found" not in log: - # We don't expect the python version file to exist if we have no - # pre-existing cache - assert ( - "Writing python-version file. " - "Cache compatibility status is currently unknown." in log - ) + assert "no cache found" not in log + # We don't expect the python version file to exist if we have no + # pre-existing cache + assert ( + "Writing python-version file. " + "Cache compatibility status is currently unknown." in log + ) diff --git a/tests/integration_tests/net/test_dhcp.py b/tests/integration_tests/net/test_dhcp.py index c4349e5b596..39c9427975e 100644 --- a/tests/integration_tests/net/test_dhcp.py +++ b/tests/integration_tests/net/test_dhcp.py @@ -4,7 +4,7 @@ from tests.integration_tests.integration_settings import PLATFORM from tests.integration_tests.releases import CURRENT_RELEASE, IS_UBUNTU, NOBLE -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import verify_clean_boot, verify_clean_log @pytest.mark.skipif(not IS_UBUNTU, reason="ubuntu-specific tests") @@ -23,6 +23,7 @@ def test_old_ubuntu_uses_isc_dhclient_by_default(self, client): log = client.read_from_file("/var/log/cloud-init.log") assert "DHCP client selected: dhclient" in log verify_clean_log(log) + verify_clean_boot(client) @pytest.mark.xfail( reason=( @@ -41,6 +42,7 @@ def test_noble_and_newer_uses_dhcpcd_by_default(self, client): ", DHCP is still running" not in log ), "cloud-init leaked a dhcp daemon that is still running" verify_clean_log(log) + verify_clean_boot(client) @pytest.mark.skipif( CURRENT_RELEASE < NOBLE, @@ -89,3 +91,4 @@ def test_noble_and_newer_force_client(self, client, dhcp_client, package): ), "Failed to get unknown option 245" assert "'unknown-245'" in log, "Failed to get unknown option 245" verify_clean_log(log) + verify_clean_boot(client) diff --git a/tests/integration_tests/reporting/test_webhook_reporting.py b/tests/integration_tests/reporting/test_webhook_reporting.py index b806f71f0a6..ba5336efa11 100644 --- a/tests/integration_tests/reporting/test_webhook_reporting.py +++ b/tests/integration_tests/reporting/test_webhook_reporting.py @@ -7,7 +7,11 @@ import pytest from tests.integration_tests.instances import IntegrationInstance -from tests.integration_tests.util import ASSETS_DIR, verify_clean_log +from tests.integration_tests.util import ( + ASSETS_DIR, + verify_clean_boot, + verify_clean_log, +) URL = "http://127.0.0.1:55555" @@ -47,6 +51,7 @@ def test_webhook_reporting(client: IntegrationInstance): "cloud-init status --wait" ) verify_clean_log(client.read_from_file("/var/log/cloud-init.log")) + verify_clean_boot(client) server_output = client.read_from_file( "/var/tmp/echo_server_output" diff --git a/tests/integration_tests/test_ds_identify.py b/tests/integration_tests/test_ds_identify.py index 59d3edd778f..8d5a313130c 100644 --- a/tests/integration_tests/test_ds_identify.py +++ b/tests/integration_tests/test_ds_identify.py @@ -2,7 +2,11 @@ from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.integration_settings import PLATFORM -from tests.integration_tests.util import verify_clean_log, wait_for_cloud_init +from tests.integration_tests.util import ( + verify_clean_boot, + verify_clean_log, + wait_for_cloud_init, +) DATASOURCE_LIST_FILE = "/etc/cloud/cloud.cfg.d/90_dpkg.cfg" MAP_PLATFORM_TO_DATASOURCE = { @@ -26,6 +30,7 @@ def test_ds_identify(client: IntegrationInstance): client.restart() wait_for_cloud_init(client) verify_clean_log(client.execute("cat /var/log/cloud-init.log")) + verify_clean_boot(client) assert client.execute("cloud-init status --wait") datasource = MAP_PLATFORM_TO_DATASOURCE.get(PLATFORM, PLATFORM) diff --git a/tests/integration_tests/test_networking.py b/tests/integration_tests/test_networking.py index bf238957250..11cff2662aa 100644 --- a/tests/integration_tests/test_networking.py +++ b/tests/integration_tests/test_networking.py @@ -18,7 +18,7 @@ MANTIC, NOBLE, ) -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import verify_clean_boot, verify_clean_log # Older Ubuntu series didn't read cloud-init.* config keys LXD_NETWORK_CONFIG_KEY = ( @@ -339,6 +339,7 @@ def test_ec2_multi_nic_reboot(setup_image, session_cloud: IntegrationCloud): log_content = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log_content) + verify_clean_boot(client) # SSH over primary and secondary NIC works for ip in public_ips: @@ -459,6 +460,7 @@ def test_ec2_multi_network_cards(setup_image, session_cloud: IntegrationCloud): log_content = client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log_content) + verify_clean_boot(client) # SSH over secondary NICs works subp("nc -w 5 -zv " + allocation_1["PublicIp"] + " 22", shell=True) diff --git a/tests/integration_tests/test_paths.py b/tests/integration_tests/test_paths.py index 31f3497fc36..d9608da49d0 100644 --- a/tests/integration_tests/test_paths.py +++ b/tests/integration_tests/test_paths.py @@ -10,7 +10,7 @@ ) from tests.integration_tests.instances import IntegrationInstance from tests.integration_tests.releases import CURRENT_RELEASE, FOCAL -from tests.integration_tests.util import verify_clean_log +from tests.integration_tests.util import verify_clean_boot, verify_clean_log DEFAULT_CLOUD_DIR = "/var/lib/cloud" NEW_CLOUD_DIR = "/new-cloud-dir" @@ -39,6 +39,7 @@ class TestHonorCloudDir: def verify_log_and_files(self, custom_client): log_content = custom_client.read_from_file("/var/log/cloud-init.log") verify_clean_log(log_content) + verify_clean_boot(custom_client) assert NEW_CLOUD_DIR in log_content assert DEFAULT_CLOUD_DIR not in log_content assert custom_client.execute(f"test ! -d {DEFAULT_CLOUD_DIR}").ok diff --git a/tests/integration_tests/test_upgrade.py b/tests/integration_tests/test_upgrade.py index 0a53eabb50e..4b17ccdd09f 100644 --- a/tests/integration_tests/test_upgrade.py +++ b/tests/integration_tests/test_upgrade.py @@ -198,3 +198,4 @@ def test_subsequent_boot_of_upgraded_package(session_cloud: IntegrationCloud): log = instance.read_from_file("/var/log/cloud-init.log") verify_clean_log(log) assert instance.execute("cloud-init status --wait --long").ok + verify_clean_boot(instance)