From 2dbb6429e97bc7c515bbe742fbf6b080e96fe0bf Mon Sep 17 00:00:00 2001 From: Chris Peterson Date: Fri, 26 Jul 2024 09:21:35 -0700 Subject: [PATCH 01/13] validation: module doc and arg parser doc update --- scripts/validate-autoinstall-user-data.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/scripts/validate-autoinstall-user-data.py b/scripts/validate-autoinstall-user-data.py index 7713b1396..c5c5febfb 100755 --- a/scripts/validate-autoinstall-user-data.py +++ b/scripts/validate-autoinstall-user-data.py @@ -14,13 +14,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -""" Entry-point to validate autoinstall-user-data against schema. +"""Validate autoinstall-user-data against the autoinstall schema. + By default, we are expecting the autoinstall user-data to be wrapped in a cloud -config format: +config format. Example: -#cloud-config -autoinstall: - + #cloud-config + autoinstall: + To validate the user-data directly, you can pass the --no-expect-cloudconfig switch. @@ -36,7 +37,11 @@ def main() -> None: """ Entry point. """ - parser = argparse.ArgumentParser() + parser = argparse.ArgumentParser( + prog="validate-autoinstall-user-data", + description=__doc__, + formatter_class=argparse.RawTextHelpFormatter, + ) parser.add_argument("--json-schema", help="Path to the JSON schema", From 14ac6e05553da7c29605c13fef2c8fe9d1fa9e4a Mon Sep 17 00:00:00 2001 From: Chris Peterson Date: Fri, 26 Jul 2024 09:25:14 -0700 Subject: [PATCH 02/13] validation: format with black --- scripts/validate-autoinstall-user-data.py | 58 ++++++++++++++--------- 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/scripts/validate-autoinstall-user-data.py b/scripts/validate-autoinstall-user-data.py index c5c5febfb..e776ab55a 100755 --- a/scripts/validate-autoinstall-user-data.py +++ b/scripts/validate-autoinstall-user-data.py @@ -36,25 +36,33 @@ def main() -> None: - """ Entry point. """ + """Entry point.""" parser = argparse.ArgumentParser( - prog="validate-autoinstall-user-data", - description=__doc__, - formatter_class=argparse.RawTextHelpFormatter, - ) - - parser.add_argument("--json-schema", - help="Path to the JSON schema", - type=argparse.FileType("r"), - default="autoinstall-schema.json") - parser.add_argument("input", nargs="?", - help="Path to the user data instead of stdin", - type=argparse.FileType("r"), - default="-") - parser.add_argument("--no-expect-cloudconfig", dest="expect-cloudconfig", - action="store_false", - help="Assume the data is not wrapped in cloud-config.", - default=True) + prog="validate-autoinstall-user-data", + description=__doc__, + formatter_class=argparse.RawTextHelpFormatter, + ) + + parser.add_argument( + "--json-schema", + help="Path to the JSON schema", + type=argparse.FileType("r"), + default="autoinstall-schema.json", + ) + parser.add_argument( + "input", + nargs="?", + help="Path to the user data instead of stdin", + type=argparse.FileType("r"), + default="-", + ) + parser.add_argument( + "--no-expect-cloudconfig", + dest="expect-cloudconfig", + action="store_false", + help="Assume the data is not wrapped in cloud-config.", + default=True, + ) args = vars(parser.parse_args()) @@ -62,8 +70,12 @@ def main() -> None: if args["expect-cloudconfig"]: assert user_data.readline() == "#cloud-config\n" - def get_autoinstall_data(data): return data["autoinstall"] + + def get_autoinstall_data(data): + return data["autoinstall"] + else: + def get_autoinstall_data(data): try: cfg = data["autoinstall"] @@ -71,14 +83,15 @@ def get_autoinstall_data(data): cfg = data return cfg - # Verify autoinstall doc link is in the file stream_pos: int = user_data.tell() data: str = user_data.read() - link: str = "https://canonical-subiquity.readthedocs-hosted.com/en/latest/reference/autoinstall-reference.html" # noqa: E501 + link: str = ( + "https://canonical-subiquity.readthedocs-hosted.com/en/latest/reference/autoinstall-reference.html" # noqa: E501 + ) assert link in data @@ -87,8 +100,7 @@ def get_autoinstall_data(data): data = yaml.safe_load(user_data) - jsonschema.validate(get_autoinstall_data(data), - json.load(args["json_schema"])) + jsonschema.validate(get_autoinstall_data(data), json.load(args["json_schema"])) if __name__ == "__main__": From 38eeb009022d7348b5f338d4b8c9cec4dd215205 Mon Sep 17 00:00:00 2001 From: Chris Peterson Date: Fri, 26 Jul 2024 09:27:15 -0700 Subject: [PATCH 03/13] validation: refactor argument parsing --- scripts/validate-autoinstall-user-data.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/scripts/validate-autoinstall-user-data.py b/scripts/validate-autoinstall-user-data.py index e776ab55a..716c1d00d 100755 --- a/scripts/validate-autoinstall-user-data.py +++ b/scripts/validate-autoinstall-user-data.py @@ -30,13 +30,15 @@ import argparse import io import json +from argparse import Namespace import jsonschema import yaml -def main() -> None: - """Entry point.""" +def parse_args() -> Namespace: + """Parse argparse arguments.""" + parser = argparse.ArgumentParser( prog="validate-autoinstall-user-data", description=__doc__, @@ -64,7 +66,13 @@ def main() -> None: default=True, ) - args = vars(parser.parse_args()) + return parser.parse_args() + + +def main() -> None: + """Entry point.""" + + args = vars(parse_args) user_data: io.TextIOWrapper = args["input"] From 2f27c9ddd2d654c36d7cf50b418198e51c66f354 Mon Sep 17 00:00:00 2001 From: Chris Peterson Date: Fri, 26 Jul 2024 09:34:41 -0700 Subject: [PATCH 04/13] validation: refactor link verification --- scripts/validate-autoinstall-user-data.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/scripts/validate-autoinstall-user-data.py b/scripts/validate-autoinstall-user-data.py index 716c1d00d..24c4df8eb 100755 --- a/scripts/validate-autoinstall-user-data.py +++ b/scripts/validate-autoinstall-user-data.py @@ -35,6 +35,16 @@ import jsonschema import yaml +DOC_LINK: str = ( + "https://canonical-subiquity.readthedocs-hosted.com/en/latest/reference/autoinstall-reference.html" # noqa: E501 +) + + +def verify_link(data: str) -> bool: + """Verify the autoinstall doc link is in the generated user-data.""" + + return DOC_LINK in data + def parse_args() -> Namespace: """Parse argparse arguments.""" @@ -72,9 +82,9 @@ def parse_args() -> Namespace: def main() -> None: """Entry point.""" - args = vars(parse_args) + args: Namespace = parse_args() - user_data: io.TextIOWrapper = args["input"] + user_data: io.TextIOWrapper = args.input if args["expect-cloudconfig"]: assert user_data.readline() == "#cloud-config\n" @@ -97,11 +107,7 @@ def get_autoinstall_data(data): data: str = user_data.read() - link: str = ( - "https://canonical-subiquity.readthedocs-hosted.com/en/latest/reference/autoinstall-reference.html" # noqa: E501 - ) - - assert link in data + assert verify_link(data) # Verify autoinstall schema user_data.seek(stream_pos) From 255d662799537ddd20fa295b43a23832a3546d3b Mon Sep 17 00:00:00 2001 From: Chris Peterson Date: Fri, 26 Jul 2024 10:02:32 -0700 Subject: [PATCH 05/13] validation: refactor parsing user data --- scripts/validate-autoinstall-user-data.py | 72 +++++++++++++++-------- 1 file changed, 46 insertions(+), 26 deletions(-) diff --git a/scripts/validate-autoinstall-user-data.py b/scripts/validate-autoinstall-user-data.py index 24c4df8eb..38c51d677 100755 --- a/scripts/validate-autoinstall-user-data.py +++ b/scripts/validate-autoinstall-user-data.py @@ -28,9 +28,9 @@ """ import argparse -import io import json from argparse import Namespace +from typing import Any import jsonschema import yaml @@ -46,6 +46,44 @@ def verify_link(data: str) -> bool: return DOC_LINK in data +def parse_cloud_config(data: str) -> dict[str, Any]: + """Parse cloud-config and extra autoinstall data.""" + + # "#cloud-config" header is required for cloud-config data + first_line: str = data.splitlines()[0] + if not first_line == "#cloud-config": + raise AssertionError( + ( + "Expected data to be wrapped in cloud-config " + "but first line is not '#cloud-config'. Try " + "passing --no-expect-cloudconfig." + ) + ) + + cc_data: dict[str, Any] = yaml.safe_load(data) + + # "autoinstall" top-level keyword is required in cloud-config delivery case + if "autoinstall" not in cc_data: + raise AssertionError( + ( + "Expected data to be wrapped in cloud-config " + "but could not find top level 'autoinstall' " + "key." + ) + ) + else: + return cc_data["autoinstall"] + + +def parse_autoinstall(user_data: str, expect_cloudconfig: bool) -> dict[str, Any]: + """Parse stringified user_data and extract autoinstall data.""" + + if expect_cloudconfig: + return parse_cloud_config(user_data) + else: + return yaml.safe_load(user_data) + + def parse_args() -> Namespace: """Parse argparse arguments.""" @@ -70,7 +108,7 @@ def parse_args() -> Namespace: ) parser.add_argument( "--no-expect-cloudconfig", - dest="expect-cloudconfig", + dest="expect_cloudconfig", action="store_false", help="Assume the data is not wrapped in cloud-config.", default=True, @@ -84,37 +122,19 @@ def main() -> None: args: Namespace = parse_args() - user_data: io.TextIOWrapper = args.input - - if args["expect-cloudconfig"]: - assert user_data.readline() == "#cloud-config\n" - - def get_autoinstall_data(data): - return data["autoinstall"] - - else: - - def get_autoinstall_data(data): - try: - cfg = data["autoinstall"] - except KeyError: - cfg = data - return cfg + str_user_data: str = args.input.read() # Verify autoinstall doc link is in the file - stream_pos: int = user_data.tell() - - data: str = user_data.read() - - assert verify_link(data) + assert verify_link(str_user_data) # Verify autoinstall schema - user_data.seek(stream_pos) - data = yaml.safe_load(user_data) + ai_user_data: dict[str, Any] = parse_autoinstall( + str_user_data, args.expect_cloudconfig + ) - jsonschema.validate(get_autoinstall_data(data), json.load(args["json_schema"])) + jsonschema.validate(ai_user_data, json.load(args.json_schema)) if __name__ == "__main__": From 0f6602671ea5b28d95d1c259d1abaaaefb61c0e0 Mon Sep 17 00:00:00 2001 From: Chris Peterson Date: Fri, 26 Jul 2024 10:11:49 -0700 Subject: [PATCH 06/13] validation: improve messaging --- scripts/validate-autoinstall-user-data.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/scripts/validate-autoinstall-user-data.py b/scripts/validate-autoinstall-user-data.py index 38c51d677..b5f9fd4fe 100755 --- a/scripts/validate-autoinstall-user-data.py +++ b/scripts/validate-autoinstall-user-data.py @@ -40,6 +40,10 @@ ) +SUCCESS_MSG: str = "Success: The provided autoinstall config validated successfully" +FAILURE_MSG: str = "Failure: The provided autoinstall config failed validation" + + def verify_link(data: str) -> bool: """Verify the autoinstall doc link is in the generated user-data.""" @@ -126,15 +130,21 @@ def main() -> None: # Verify autoinstall doc link is in the file - assert verify_link(str_user_data) + assert verify_link(str_user_data), "Documentation link missing from user data" # Verify autoinstall schema - ai_user_data: dict[str, Any] = parse_autoinstall( - str_user_data, args.expect_cloudconfig - ) + try: + + ai_user_data: dict[str, Any] = parse_autoinstall( + str_user_data, args.expect_cloudconfig + ) + except Exception as exc: + print(f"FAILURE: {exc}") + return 1 jsonschema.validate(ai_user_data, json.load(args.json_schema)) + print(SUCCESS_MSG) if __name__ == "__main__": From fcd26237951e2c659bbd577efbb95b56f0b349a9 Mon Sep 17 00:00:00 2001 From: Chris Peterson Date: Fri, 26 Jul 2024 10:19:56 -0700 Subject: [PATCH 07/13] validation: CI specific flag In the future we want to use dry-run and subiquity internals to do more robust validation of autoinstall user-data. Today the CI isn't ready for this and we should rely on old behavior to not regress CI results. This effectively moves current behavior behind the --legacy flag. --- scripts/runtests.sh | 2 +- scripts/validate-autoinstall-user-data.py | 33 ++++++++++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/scripts/runtests.sh b/scripts/runtests.sh index 46dbac2c1..21929f7e3 100755 --- a/scripts/runtests.sh +++ b/scripts/runtests.sh @@ -49,7 +49,7 @@ validate () { answers-core-desktop|answers-uc24) ;; *) - python3 scripts/validate-autoinstall-user-data.py < $tmpdir/var/log/installer/autoinstall-user-data + python3 scripts/validate-autoinstall-user-data.py --legacy < $tmpdir/var/log/installer/autoinstall-user-data # After the lunar release and the introduction of mirror testing, it # came to our attention that new Ubuntu installations have the security # repository configured with the primary mirror URL (i.e., diff --git a/scripts/validate-autoinstall-user-data.py b/scripts/validate-autoinstall-user-data.py index b5f9fd4fe..688e20425 100755 --- a/scripts/validate-autoinstall-user-data.py +++ b/scripts/validate-autoinstall-user-data.py @@ -28,6 +28,7 @@ """ import argparse +import io import json from argparse import Namespace from typing import Any @@ -88,6 +89,12 @@ def parse_autoinstall(user_data: str, expect_cloudconfig: bool) -> dict[str, Any return yaml.safe_load(user_data) +def legacy_verify(ai_data: dict[str, Any], json_schema: io.TextIOWrapper) -> None: + """Legacy verification method for use in CI""" + + jsonschema.validate(ai_data, json.load(json_schema)) + + def parse_args() -> Namespace: """Parse argparse arguments.""" @@ -97,12 +104,6 @@ def parse_args() -> Namespace: formatter_class=argparse.RawTextHelpFormatter, ) - parser.add_argument( - "--json-schema", - help="Path to the JSON schema", - type=argparse.FileType("r"), - default="autoinstall-schema.json", - ) parser.add_argument( "input", nargs="?", @@ -118,6 +119,22 @@ def parse_args() -> Namespace: default=True, ) + # Hidden validation path we use in CI until the new validation method + # is ready. i.e. continue to validate based on the json schema directly. + parser.add_argument( + "--json-schema", + help=argparse.SUPPRESS, + type=argparse.FileType("r"), + default="autoinstall-schema.json", + ) + + parser.add_argument( + "--legacy", + action="store_true", + help=argparse.SUPPRESS, + default=False, + ) + return parser.parse_args() @@ -143,7 +160,9 @@ def main() -> None: print(f"FAILURE: {exc}") return 1 - jsonschema.validate(ai_user_data, json.load(args.json_schema)) + if args.legacy: + legacy_verify(ai_user_data, args.json_schema) + print(SUCCESS_MSG) From 3181d3ac750c6e6b4d361a281a489506d9eabacd Mon Sep 17 00:00:00 2001 From: Chris Peterson Date: Fri, 26 Jul 2024 10:23:22 -0700 Subject: [PATCH 08/13] validation: link checking is CI only With moving to make the validation script more user facing, we don't need users to have the documentation link in their autoinstall file. Add a hidden flag, --check-link, to be used in CI to validate rendered autoinstall config has the documentation link. --- scripts/runtests.sh | 2 +- scripts/validate-autoinstall-user-data.py | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/scripts/runtests.sh b/scripts/runtests.sh index 21929f7e3..f20a95d5a 100755 --- a/scripts/runtests.sh +++ b/scripts/runtests.sh @@ -49,7 +49,7 @@ validate () { answers-core-desktop|answers-uc24) ;; *) - python3 scripts/validate-autoinstall-user-data.py --legacy < $tmpdir/var/log/installer/autoinstall-user-data + python3 scripts/validate-autoinstall-user-data.py --legacy --check-link < $tmpdir/var/log/installer/autoinstall-user-data # After the lunar release and the introduction of mirror testing, it # came to our attention that new Ubuntu installations have the security # repository configured with the primary mirror URL (i.e., diff --git a/scripts/validate-autoinstall-user-data.py b/scripts/validate-autoinstall-user-data.py index 688e20425..b2c611d63 100755 --- a/scripts/validate-autoinstall-user-data.py +++ b/scripts/validate-autoinstall-user-data.py @@ -135,6 +135,17 @@ def parse_args() -> Namespace: default=False, ) + # An option we use in CI to make sure Subiquity will insert a link to + # the documentation in the auto-generated autoinstall file post-install. + # There's not need for users to check this. + parser.add_argument( + "--check-link", + dest="check_link", + action="store_true", + help=argparse.SUPPRESS, + default=False, + ) + return parser.parse_args() @@ -147,7 +158,9 @@ def main() -> None: # Verify autoinstall doc link is in the file - assert verify_link(str_user_data), "Documentation link missing from user data" + if args.check_link: + + assert verify_link(str_user_data), "Documentation link missing from user data" # Verify autoinstall schema From 8451ae66761ba532b9e2b3398aaf6a8f50f8b40d Mon Sep 17 00:00:00 2001 From: Chris Peterson Date: Fri, 26 Jul 2024 10:33:11 -0700 Subject: [PATCH 09/13] validation: legacy: support top-level autoinstall --- scripts/validate-autoinstall-user-data.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/validate-autoinstall-user-data.py b/scripts/validate-autoinstall-user-data.py index b2c611d63..533a545e8 100755 --- a/scripts/validate-autoinstall-user-data.py +++ b/scripts/validate-autoinstall-user-data.py @@ -92,7 +92,13 @@ def parse_autoinstall(user_data: str, expect_cloudconfig: bool) -> dict[str, Any def legacy_verify(ai_data: dict[str, Any], json_schema: io.TextIOWrapper) -> None: """Legacy verification method for use in CI""" - jsonschema.validate(ai_data, json.load(json_schema)) + # support top-level "autoinstall" in regular autoinstall user data + if "autoinstall" in ai_data: + data: dict[str, Any] = ai_data["autoinstall"] + else: + data: dict[str, Any] = ai_data + + jsonschema.validate(data, json.load(json_schema)) def parse_args() -> Namespace: From d38f0071232a4f4c3343db27c224908d1e7c5397 Mon Sep 17 00:00:00 2001 From: Chris Peterson Date: Tue, 30 Jul 2024 23:36:20 -0700 Subject: [PATCH 10/13] validate: use dry-run server for validation ./scripts/validate-autoinstall-user-data is used by the integration tests to verify the rendered user data validates against the combined JSON schema, but we have introduced run-time checks for more things than can be caught by simple JSON validation (e.g. warns/errors on unknown keys or strict top-level key checking for supporting a top-level "autoinstall" keyword in the non-cloud-config delivery scenario). Now the validation logic relies on the server validation logic directly to perform pre-validation of the the supplied autoinstall configuration. --- scripts/validate-autoinstall-user-data.py | 85 +++++++++++++++++++++-- 1 file changed, 80 insertions(+), 5 deletions(-) diff --git a/scripts/validate-autoinstall-user-data.py b/scripts/validate-autoinstall-user-data.py index 533a545e8..b540a3403 100755 --- a/scripts/validate-autoinstall-user-data.py +++ b/scripts/validate-autoinstall-user-data.py @@ -28,14 +28,36 @@ """ import argparse +import asyncio import io import json -from argparse import Namespace +import sys +import tempfile +from argparse import ArgumentParser, Namespace +from pathlib import Path from typing import Any import jsonschema import yaml +# Python path trickery so we can import subiquity code and still call this +# script without using the makefile. Eventually we should ship this a +# program in the subiquity snap, so users don't even have to checkout the +# source code but that will also require work to make sure Subiquity is +# safe to install on regular systems. +scripts_dir = sys.path[0] +subiquity_root = Path(scripts_dir) / ".." +curtin_root = subiquity_root / "curtin" +probert_root = subiquity_root / "probert" + +sys.path.insert(0, str(subiquity_root)) +sys.path.insert(1, str(curtin_root)) +sys.path.insert(2, str(probert_root)) + +from subiquity.cmd.server import make_server_args_parser # noqa: E402 +from subiquity.server.dryrun import DRConfig # noqa: E402 +from subiquity.server.server import SubiquityServer # noqa: E402 + DOC_LINK: str = ( "https://canonical-subiquity.readthedocs-hosted.com/en/latest/reference/autoinstall-reference.html" # noqa: E501 ) @@ -101,6 +123,58 @@ def legacy_verify(ai_data: dict[str, Any], json_schema: io.TextIOWrapper) -> Non jsonschema.validate(data, json.load(json_schema)) +async def make_app() -> SubiquityServer: + parser: ArgumentParser = make_server_args_parser() + opts, unknown = parser.parse_known_args(["--dry-run"]) + app: SubiquityServer = SubiquityServer(opts, "") + # This is needed because the ubuntu-pro server controller accesses dr_cfg + # in the initializer. + app.dr_cfg = DRConfig() + app.base_model = app.make_model() + app.controllers.load_all() + return app + + +async def verify_autoinstall( + app: SubiquityServer, + cfg_path: str, +) -> int: + """Verify autoinstall configuration using a SubiquityServer. + + Returns 0 if successfully validated. + Returns 1 if fails to validate. + """ + + # Tell the server where to load the autoinstall + app.autoinstall = cfg_path + + # Validation happens during load phases. Do both phases. + try: + app.load_autoinstall_config(only_early=True, context=None) + app.load_autoinstall_config(only_early=False, context=None) + except Exception as exc: + + print(exc) # Has the useful error message + + print(FAILURE_MSG) + return 1 + + print(SUCCESS_MSG) + return 0 + + +async def _async_main(ai_user_data: dict[str, Any], args: Namespace) -> int: + # Make a dry-run server + app: SubiquityServer = await make_app() + + with tempfile.TemporaryDirectory() as td: + path = Path(td) / "autoinstall.yaml" + yaml_as_text: str = yaml.dump(ai_user_data) + path.write_text(yaml_as_text) + + return await verify_autoinstall(app=app, cfg_path=path) + + def parse_args() -> Namespace: """Parse argparse arguments.""" @@ -155,7 +229,7 @@ def parse_args() -> Namespace: return parser.parse_args() -def main() -> None: +def main() -> int: """Entry point.""" args: Namespace = parse_args() @@ -171,7 +245,6 @@ def main() -> None: # Verify autoinstall schema try: - ai_user_data: dict[str, Any] = parse_autoinstall( str_user_data, args.expect_cloudconfig ) @@ -181,9 +254,11 @@ def main() -> None: if args.legacy: legacy_verify(ai_user_data, args.json_schema) + print(SUCCESS_MSG) + return 0 - print(SUCCESS_MSG) + return asyncio.run(_async_main(ai_user_data, args)) if __name__ == "__main__": - main() + sys.exit(main()) From 9aee3590ddddc1a29e86a9f528b76549e69d0401 Mon Sep 17 00:00:00 2001 From: Chris Peterson Date: Tue, 30 Jul 2024 23:42:25 -0700 Subject: [PATCH 11/13] validate: update logging + verbosity option --- scripts/validate-autoinstall-user-data.py | 37 ++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/scripts/validate-autoinstall-user-data.py b/scripts/validate-autoinstall-user-data.py index b540a3403..155cb4d06 100755 --- a/scripts/validate-autoinstall-user-data.py +++ b/scripts/validate-autoinstall-user-data.py @@ -33,6 +33,7 @@ import json import sys import tempfile +import traceback from argparse import ArgumentParser, Namespace from pathlib import Path from typing import Any @@ -138,6 +139,7 @@ async def make_app() -> SubiquityServer: async def verify_autoinstall( app: SubiquityServer, cfg_path: str, + verbosity: int = 0, ) -> int: """Verify autoinstall configuration using a SubiquityServer. @@ -148,6 +150,21 @@ async def verify_autoinstall( # Tell the server where to load the autoinstall app.autoinstall = cfg_path + # Suppress start and finish events unless verbosity >=2 + if verbosity < 2: + for el in app.event_listeners: + el.report_start_event = lambda x, y: None + el.report_finish_event = lambda x, y, z: None + # Suppress info events unless verbosity >=1 + if verbosity < 1: + for el in app.event_listeners: + el.report_info_event = lambda x, y: None + + # Make sure all events are printed (we could fail during read, which + # would happen before we setup the reporting controller) + app.controllers.Reporting.config = {"builtin": {"type": "print"}} + app.controllers.Reporting.start() + # Validation happens during load phases. Do both phases. try: app.load_autoinstall_config(only_early=True, context=None) @@ -156,6 +173,10 @@ async def verify_autoinstall( print(exc) # Has the useful error message + # Print the full traceback if verbosity > 2 + if verbosity > 2: + traceback.print_exception(exc) + print(FAILURE_MSG) return 1 @@ -172,7 +193,11 @@ async def _async_main(ai_user_data: dict[str, Any], args: Namespace) -> int: yaml_as_text: str = yaml.dump(ai_user_data) path.write_text(yaml_as_text) - return await verify_autoinstall(app=app, cfg_path=path) + return await verify_autoinstall( + app=app, + cfg_path=path, + verbosity=args.verbosity, + ) def parse_args() -> Namespace: @@ -225,6 +250,16 @@ def parse_args() -> Namespace: help=argparse.SUPPRESS, default=False, ) + parser.add_argument( + "-v", + "--verbosity", + action="count", + help=( + "Increase output verbosity. Use -v for more info, -vv for " + "detailed output, and -vvv for fully detailed output." + ), + default=0, + ) return parser.parse_args() From 04f06be12e96b33bc37ca0f360363ba05b8e1d9c Mon Sep 17 00:00:00 2001 From: Chris Peterson Date: Sat, 18 May 2024 22:20:36 +0200 Subject: [PATCH 12/13] docs: new how-to pre-validate autoinstall config --- doc/.wordlist.txt | 3 + doc/howto/autoinstall-validation.rst | 249 ++++++++++++++++++++++++ doc/howto/index.rst | 1 + doc/reference/autoinstall-reference.rst | 2 + doc/reference/autoinstall-schema.rst | 26 ++- doc/tutorial/providing-autoinstall.rst | 6 +- 6 files changed, 279 insertions(+), 8 deletions(-) create mode 100644 doc/howto/autoinstall-validation.rst diff --git a/doc/.wordlist.txt b/doc/.wordlist.txt index 8a0ec8a14..1dbfebde8 100644 --- a/doc/.wordlist.txt +++ b/doc/.wordlist.txt @@ -28,6 +28,7 @@ YAML addons balancer boolean +datasource dropdown favicon installable @@ -36,6 +37,8 @@ namespaces observability reST reStructuredText +stdin subdirectories subfolders subtree +validator diff --git a/doc/howto/autoinstall-validation.rst b/doc/howto/autoinstall-validation.rst new file mode 100644 index 000000000..f8a4db001 --- /dev/null +++ b/doc/howto/autoinstall-validation.rst @@ -0,0 +1,249 @@ +.. _autoinstall_validation: + +Autoinstall Validation +===================================== + +The following how-to guide demonstrates how to perform pre-validation of a autoinstall config. + +Autoinstall config is validated against a :doc:`JSON schema <../reference/autoinstall-schema>` during runtime before it is applied. This check ensures existence of required keys and their data types, but does not guarantee the validity of the data provided (e.g., a bad :ref:`match directive `). Additionally, the following validation script is unable to replicate some of the cloud-config based :ref:`delivery checks `. There are some basic checks performed to catch simple delivery-related errors, which you can read more about in the examples section, but the focus of the validation is on the Autoinstall configuration *after* it has been delivered to the installer. + +.. note:: + See the cloud-init documentation for `how to validate your cloud-config`_. + + +Pre-validating the Autoinstall configuration +-------------------------------------------- + +You can validate autoinstall config prior to install time by using the `validate-autoinstall-user-data script `_ in the Subiquity GitHub repository. + +Getting Started +^^^^^^^^^^^^^^^ + +Running the validation script requires downloading the Subiquity source code and installing the development dependencies. First, clone the Subiquity repository and ``cd`` into the root of the repository: + +.. code:: none + + git clone https://github.com/canonical/subiquity.git && cd subiquity/ + +Then the required dependencies can be installed by running: + +.. code:: none + + make install_deps + + +Now you can invoke the validation script with: + +.. code:: none + + ./scripts/validate-autoinstall-user-data + + +or you can feed the configuration data via stdin: + + +.. code:: none + + # a trivial example + cat | ./scripts/validate-autoinstall-user-data + +.. warning:: + + Never run the validation script as ``sudo``. + +Finally, after running the validation script it will report the result of the validation attempt: + +.. code:: none + + $ ./scripts/validate-autoinstall-user-data.py + Success: The provided autoinstall config validated successfully + +You can also use the exit codes to determine the result: 0 (success) or 1 (failure). + + +Choice of Delivery Method +^^^^^^^^^^^^^^^^^^^^^^^^^ + +By default the validation script will expect your autoinstall configuration to be passed via cloud-config and expects a valid cloud-config file containing an ``autoinstall`` section: + +.. code:: none + + #cloud-config + + # some cloud-init directives + + autoinstall: + # autoinstall directives + +This allows you to use the script directly on your cloud-config data. The validation script will extract the autoinstall configuration from the provided cloud-config data and perform the validation on the extracted autoinstall section directly. + + +If you want to validate autoinstall configurations which will be delivered via the installation media, like the following example: + +.. code:: none + + autoinstall: + # autoinstall directives + +then this can be signalled by passing the ``--no-expect-cloudconfig`` flag. Both formats in this delivery method, with or without a top-level ``autoinstall`` keyword, are supported in this mode. + + +------------ + +Examples +-------- + +Common mistake #1 +^^^^^^^^^^^^^^^^^ + +If a top level ``autoinstall`` keyword is not found in the provided cloud-config during runtime then the installer will miss the autoinstall config and present an interactive session. To prevent occurrences of this issue, the validation script will report a failure if the provided cloud-config does not contain an autoinstall section. *This does not indicate a crash at runtime*, as you can definitely provide cloud-config without autoinstall, but it is a useful result for checking a common formatting mistake. + +.. tabs:: + + .. tab:: Validation output + + + Validating cloud-config which is missing the ``autoinstall`` keyword: + + .. code:: none + + $ ./scripts/validate-autoinstall-user-data.py + AssertionError: Expected data to be wrapped in cloud-config but could not find top level 'autoinstall' key. + Failure: The provided autoinstall config did not validate successfully + + .. tab:: Faulty config + + As an example, the following cloud-config contains an autoinstall section but has misspelled the ``autoinstall`` keyword: + + .. code:: none + + #cloud-config + autoinstll: + # autoinstall directives + + +Common Mistake #2 +^^^^^^^^^^^^^^^^^ + +Another common mistake is to forget the ``#cloud-config`` header in the cloud-config file, which will result in the installer "missing" the autoinstall configuration. + +.. tabs:: + + .. tab:: Validation output + + The validator will fail the provided cloud-config data if it does not contain the right header: + + + .. code:: none + + $ ./scripts/validate-autoinstall-user-data.py + AssertionError: Expected data to be wrapped in cloud-config but first line is not '#cloud-config'. Try passing --no-expect-cloudconfig. + Failure: The provided autoinstall config did not validate successfully + + + .. tab:: Faulty config + + Missing the ``#cloud-config`` header will mean the file is not read by cloud-init: + + .. code:: none + + autoinstall: + # autoinstall directives + + +Again, this is not indicative of a real runtime error that would appear. Instead, this case would result in having the installer presenting a fully interactive install where a partially or fully automated installation was desired instead. + +Common Mistake #3 +^^^^^^^^^^^^^^^^^ + +Another possible mistake is to think that the autoinstall config on the installation media is a cloud-config datasource (it is not): + +.. tabs:: + + .. tab:: Validation output + + When providing the autoinstall configuration using the top-level ``autoinstall`` keyword format, the installer will verify there are no other top-level keys: + + .. code:: none + + $ ./scripts/validate-autoinstall-user-data.py --no-expect-cloudconfig + error: subiquity/load_autoinstall_config/read_config: autoinstall.yaml is not a valid cloud config datasource. + No other keys may be present alongside 'autoinstall' at the top level. + Malformed autoinstall in 'top-level keys' section + Failure: The provided autoinstall config did not validate successfully + + .. tab:: Faulty config + + The following config contains cloud-config directives when it is not expected to contain any: + + .. code:: none + + #cloud-config + + # some cloud-config directives + + autoinstall: + # autoinstall directives + + + +Debugging errors +^^^^^^^^^^^^^^^^ + +By default, the validation script has low verbosity output: + +.. code:: none + + Malformed autoinstall in 'version or interactive-sections' section + Failure: The provided autoinstall config did not validate successfully + +However, you can increase the output level by successively passing the ``-v`` flag. At maximum verbosity, the validation script will report errors the same way they are reported at runtime. This is great for inspecting issues in cases where the short error message isn't yet specific enough to be useful and can be used to inspect specific JSON schema validation errors. + + +.. code:: none + + $ ./scripts/validate-autoinstall-user-data.py autoinstall.yaml -vvv + start: subiquity/load_autoinstall_config: + start: subiquity/load_autoinstall_config/read_config: + finish: subiquity/load_autoinstall_config/read_config: SUCCESS: + start: subiquity/Reporting/load_autoinstall_data: + finish: subiquity/Reporting/load_autoinstall_data: SUCCESS: + start: subiquity/Error/load_autoinstall_data: + finish: subiquity/Error/load_autoinstall_data: SUCCESS: + start: subiquity/core_validation: + finish: subiquity/core_validation: FAIL: Malformed autoinstall in 'version or interactive-sections' section + finish: subiquity/load_autoinstall_config: FAIL: Malformed autoinstall in 'version or interactive-sections' section + Malformed autoinstall in 'version or interactive-sections' section + Traceback (most recent call last): + File ".../subiquity/scripts/../subiquity/server/server.py", line 654, in validate_autoinstall + jsonschema.validate(self.autoinstall_config, self.base_schema) + File "/usr/lib/python3/dist-packages/jsonschema/validators.py", line 1080, in validate + raise error + jsonschema.exceptions.ValidationError: '*' is not of type 'array' + + Failed validating 'type' in schema['properties']['interactive-sections']: + {'items': {'type': 'string'}, 'type': 'array'} + + On instance['interactive-sections']: + '*' + + The above exception was the direct cause of the following exception: + + Traceback (most recent call last): + File ".../subiquity/./scripts/validate-autoinstall-user-data.py", line 186, in verify_autoinstall + app.load_autoinstall_config(only_early=True, context=None) + File ".../subiquity/scripts/../subiquitycore/context.py", line 159, in decorated_sync + return meth(self, **kw) + ^^^^^^^^^^^^^^^^ + File ".../subiquity/scripts/../subiquity/server/server.py", line 734, in load_autoinstall_config + self.validate_autoinstall() + File ".../subiquity/scripts/../subiquity/server/server.py", line 663, in validate_autoinstall + raise new_exception from original_exception + subiquity.server.autoinstall.AutoinstallValidationError: Malformed autoinstall in 'version or interactive-sections' section + Failure: The provided autoinstall config did not validate successfully + +In this case, the above output shows that ``interactive-sections`` section failed to validate against the JSON schema because the type provided was a ``string`` and not an ``array`` of ``string`` s. + +.. LINKS + +.. _how to validate your cloud-config: https://cloudinit.readthedocs.io/en/latest/howto/debug_user_data.html diff --git a/doc/howto/index.rst b/doc/howto/index.rst index 0dc988692..8087d1f54 100644 --- a/doc/howto/index.rst +++ b/doc/howto/index.rst @@ -19,6 +19,7 @@ Getting started with autoinstall autoinstall-quickstart-s390x basic-server-installation configure-storage + autoinstall-validation Found a problem? ---------------- diff --git a/doc/reference/autoinstall-reference.rst b/doc/reference/autoinstall-reference.rst index 140d068fe..17037a872 100644 --- a/doc/reference/autoinstall-reference.rst +++ b/doc/reference/autoinstall-reference.rst @@ -623,6 +623,8 @@ An example storage section: The extensions to the curtin syntax allow for disk selection and partition or logical-volume sizing. +.. _disk_selection_extensions: + Disk selection extensions ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doc/reference/autoinstall-schema.rst b/doc/reference/autoinstall-schema.rst index ad4b62741..f99a7479a 100644 --- a/doc/reference/autoinstall-schema.rst +++ b/doc/reference/autoinstall-schema.rst @@ -3,14 +3,30 @@ Autoinstall schema ================== -The server installer validates the provided autoinstall configuration against a :ref:`JSON schema`. +The server installer validates the provided autoinstall configuration against a :ref:`JSON schema`. The end of this reference manual presents the schema as a single document which could be used to manually pre-validate an autoinstall configuration, however the actual runtime validation process is more involved than a simple JSON schema validation. See the provided :doc:`pre-validation script <../howto/autoinstall-validation>` for how to perform autoinstall pre-validation. + +.. _how_the_delivery_is_verified: + +How the delivery is verified +---------------------------- + +To ensure expected runtime behaviour after delivering the autoinstall config, the installer performs some sanity checks to ensure one delivery method is not confused for another. + +cloud-config +^^^^^^^^^^^^ + +When passing autoinstall via cloud-config, the installer will inspect the cloud-config data for any autoinstall-specific keywords outside of the top-level ``autoinstall`` keyword in the config and throw an error if any are encountered. If there are no misplaced keys, the data within the ``autoinstall`` section is passed to the installer. + + +Installation Media +^^^^^^^^^^^^^^^^^^ + +When passing autoinstall via the installation media and using the top-level ``autoinstall`` keyword format, the installer will inspect the passed autoinstall file to guarantee that there are no other top-level keys. This check guarantees that the autoinstall config is not mistaken for a cloud-config datasource. How the configuration is validated ---------------------------------- -This reference manual presents the schema as a single document. Use it pre-validate your configuration. - -At run time, the configuration is not validated against this document. Instead, configuration sections are loaded and validated in this order: +After the configuration has been delivered to the installer successfully, the configuration sections are loaded and validated in this order: 1. The reporting section is loaded, validated and applied. 2. The error commands are loaded and validated. @@ -35,7 +51,7 @@ Regeneration To regenerate the schema, run ``make schema`` in the root directory of the `Subiquity source repository`_. -.. LINKS +.. LINKS .. _JSON schema: https://json-schema.org/ .. _Subiquity source repository: https://github.com/canonical/subiquity diff --git a/doc/tutorial/providing-autoinstall.rst b/doc/tutorial/providing-autoinstall.rst index 6e4c5cfac..9c2ad44a5 100644 --- a/doc/tutorial/providing-autoinstall.rst +++ b/doc/tutorial/providing-autoinstall.rst @@ -21,7 +21,7 @@ The suggested way of providing autoinstall configuration to the Ubuntu installer The autoinstall configuration is provided via cloud-init configuration, which is almost endlessly flexible. In most scenarios the easiest way will be to provide user data via the :external+cloud-init:ref:`datasource_nocloud` data source. -When providing autoinstall via cloud-init, the autoinstall configuration is provided as :external+cloud-init:ref:`user_data_formats-cloud_config`. This means it requires a :code:`#cloud-config` header. The autoinstall directives are placed under a top level :code:`autoinstall:` key: +When providing autoinstall via cloud-init, the autoinstall configuration is provided as :external+cloud-init:ref:`user_data_formats-cloud_config`. This means the file requires a :code:`#cloud-config` header and the autoinstall directives are placed under a top level :code:`autoinstall:` key: .. code-block:: yaml @@ -47,7 +47,7 @@ The autoinstall configuration provided in this way is passed to the Ubuntu insta version: 1 .... -Starting in 24.04 (Noble), to be consistent with the cloud-config based format, a top-level :code:`autoinstall:` keyword is allowed: +Starting in 24.04 (Noble), to be consistent with the cloud-config based format, a single top-level :code:`autoinstall:` keyword is allowed: .. code-block:: yaml @@ -69,7 +69,7 @@ Alternatively, you can pass the location of the autoinstall file on the kernel c Order of precedence for autoinstall locations ---------------------------------------------- +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Because there are many ways to specify the autoinstall file, it may happen that multiple locations are specified at the same time. Subiquity searches for the autoinstall file in the following order and uses the first existing one: From 79217f4887b0617c4f454bdfa8abe1b95ff70ddb Mon Sep 17 00:00:00 2001 From: Chris Peterson Date: Mon, 12 Aug 2024 16:42:56 -0700 Subject: [PATCH 13/13] doc: validator limitations section --- doc/howto/autoinstall-validation.rst | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/doc/howto/autoinstall-validation.rst b/doc/howto/autoinstall-validation.rst index f8a4db001..7a2067e43 100644 --- a/doc/howto/autoinstall-validation.rst +++ b/doc/howto/autoinstall-validation.rst @@ -5,11 +5,7 @@ Autoinstall Validation The following how-to guide demonstrates how to perform pre-validation of a autoinstall config. -Autoinstall config is validated against a :doc:`JSON schema <../reference/autoinstall-schema>` during runtime before it is applied. This check ensures existence of required keys and their data types, but does not guarantee the validity of the data provided (e.g., a bad :ref:`match directive `). Additionally, the following validation script is unable to replicate some of the cloud-config based :ref:`delivery checks `. There are some basic checks performed to catch simple delivery-related errors, which you can read more about in the examples section, but the focus of the validation is on the Autoinstall configuration *after* it has been delivered to the installer. - -.. note:: - See the cloud-init documentation for `how to validate your cloud-config`_. - +Autoinstall config is validated against a :doc:`JSON schema <../reference/autoinstall-schema>` during runtime before it is applied. This check ensures existence of required keys and their data types, but does not guarantee total validity of the data provided (see the :ref:`Validator Limitations` section for more details). Pre-validating the Autoinstall configuration -------------------------------------------- @@ -87,6 +83,25 @@ If you want to validate autoinstall configurations which will be delivered via t then this can be signalled by passing the ``--no-expect-cloudconfig`` flag. Both formats in this delivery method, with or without a top-level ``autoinstall`` keyword, are supported in this mode. +.. _validator-limitations: + +Validator Limitations +--------------------- + +The autoinstall validator currently has the following limitations: + +1. The validator makes an assumption about the target installation media that may not necessarily be true about the actual installation media. It assumes that (1) the installation target is ubuntu-server and (2) the only valid install source is :code:`synthesized`. Some cases where this would cause the validator fail otherwise correct autoinstall configurations: + + a. Missing both an :code:`identity` and :code:`user-data` section for a Desktop target, where these sections are fully optional. + b. A :code:`source` section which specifies any :code:`id` other than :code:`synthesized`, where the :code:`id` may really match a valid source on the target ISO. + +2. Validity of the data provided in each section is not guaranteed as some sections cannot be reasonably validated outside of the installation runtime environment (e.g., a bad :ref:`match directive `). + +3. The validator is unable to replicate some of the cloud-config based :ref:`delivery checks `. There are some basic checks performed to catch simple delivery-related errors, which you can read more about in the examples section, but the focus of the validation is on the Autoinstall configuration *after* it has been delivered to the installer. + +.. note:: + See the cloud-init documentation for `how to validate your cloud-config`_. + ------------