From 8854179d682f0229957edb209a03dd59e2910c7b Mon Sep 17 00:00:00 2001 From: Cat Red Date: Thu, 25 Apr 2024 06:22:56 -0400 Subject: [PATCH] feature(schema): add networkv2 schema (#4892) Adds networkv2 schema definition file, removes skipping of schema validation for networkv2, and adds unit tests for same. Co-authored-by: Alberto Contreras --- cloudinit/config/schema.py | 56 ++- .../schemas/schema-network-config-v2.json | 435 ++++++++++++++++++ tests/unittests/config/test_schema.py | 135 +++++- 3 files changed, 591 insertions(+), 35 deletions(-) create mode 100644 cloudinit/config/schemas/schema-network-config-v2.json diff --git a/cloudinit/config/schema.py b/cloudinit/config/schema.py index eb7e43856ee..abfab98a1d5 100644 --- a/cloudinit/config/schema.py +++ b/cloudinit/config/schema.py @@ -66,6 +66,7 @@ # 3. Add the new version definition to versions.schema.cloud-config.json USERDATA_SCHEMA_FILE = "schema-cloud-config-v1.json" NETWORK_CONFIG_V1_SCHEMA_FILE = "schema-network-config-v1.json" +NETWORK_CONFIG_V2_SCHEMA_FILE = "schema-network-config-v2.json" _YAML_MAP = {True: "true", False: "false", None: "null"} SCHEMA_DOC_TMPL = """ @@ -160,6 +161,8 @@ class SchemaType(Enum): CLOUD_CONFIG = "cloud-config" NETWORK_CONFIG = "network-config" + NETWORK_CONFIG_V1 = "network-config-v1" + NETWORK_CONFIG_V2 = "network-config-v2" # Placeholders for versioned schema and schema file locations. @@ -169,8 +172,14 @@ class SchemaType(Enum): "latest": USERDATA_SCHEMA_FILE, }, SchemaType.NETWORK_CONFIG: { + "latest": NETWORK_CONFIG_V2_SCHEMA_FILE, + }, + SchemaType.NETWORK_CONFIG_V1: { "latest": NETWORK_CONFIG_V1_SCHEMA_FILE, }, + SchemaType.NETWORK_CONFIG_V2: { + "latest": NETWORK_CONFIG_V2_SCHEMA_FILE, + }, } @@ -699,7 +708,8 @@ def validate_cloudconfig_schema( for the cloud config module (config.cc_*). If None, validate against global schema. @param schema_type: Optional SchemaType. - One of: SchemaType.CLOUD_CONFIG or SchemaType.NETWORK_CONFIG. + One of: SchemaType.CLOUD_CONFIG or SchemaType.NETWORK_CONFIG_V1 or + SchemaType.NETWORK_CONFIG_V2 Default: SchemaType.CLOUD_CONFIG @param strict: Boolean, when True raise SchemaValidationErrors instead of logging warnings. @@ -714,18 +724,22 @@ def validate_cloudconfig_schema( against the provided schema. @raises: RuntimeError when provided config sourced from YAML is not a dict. @raises: ValueError on invalid schema_type not in CLOUD_CONFIG or - NETWORK_CONFIG + NETWORK_CONFIG_V1 or NETWORK_CONFIG_V2 """ if schema_type == SchemaType.NETWORK_CONFIG: - if network_schema_version(config) == 2: - if netplan_validate_network_schema( - network_config=config, strict=strict, log_details=log_details - ): - # Schema was validated by netplan - return True - # network-config schema version 2 but no netplan. - # TODO(add JSON schema definition for network version 2) - return False + network_version = network_schema_version(config) + if network_version == 2: + schema_type = SchemaType.NETWORK_CONFIG_V2 + elif network_version == 1: + schema_type = SchemaType.NETWORK_CONFIG_V1 + schema = get_schema(schema_type) + + if schema_type == SchemaType.NETWORK_CONFIG_V2: + if netplan_validate_network_schema( + network_config=config, strict=strict, log_details=log_details + ): + # Schema was validated by netplan + return True if schema is None: schema = get_schema(schema_type) @@ -1121,22 +1135,22 @@ def validate_cloudconfig_file( return False network_version = network_schema_version(cloudconfig) if network_version == 2: + schema_type = SchemaType.NETWORK_CONFIG_V2 if netplan_validate_network_schema( network_config=cloudconfig, strict=True, annotate=annotate ): return True # schema validation performed by netplan - if network_version != 1: - # Validation requires JSON schema definition in - # cloudinit/config/schemas/schema-network-config-v1.json - print( - "Skipping network-config schema validation." - " No network schema for version:" - f" {network_schema_version(cloudconfig)}" - ) - return False + elif network_version == 1: + schema_type = SchemaType.NETWORK_CONFIG_V1 + # refresh schema since NETWORK_CONFIG defaults to V2 + schema = get_schema(schema_type) try: if not validate_cloudconfig_schema( - cloudconfig, schema=schema, strict=True, log_deprecations=False + cloudconfig, + schema=schema, + schema_type=schema_type, + strict=True, + log_deprecations=False, ): print( f"Skipping {schema_type.value} schema validation." diff --git a/cloudinit/config/schemas/schema-network-config-v2.json b/cloudinit/config/schemas/schema-network-config-v2.json new file mode 100644 index 00000000000..133146a0da4 --- /dev/null +++ b/cloudinit/config/schemas/schema-network-config-v2.json @@ -0,0 +1,435 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$defs": { + "renderer": { + "type": "string", + "description": "Use the given networking backend for this definition. Default is networkd.", + "enum": [ + "networkd", + "NetworkManager" + ] + }, + "dhcp-overrides": { + "type": "object", + "description": "DHCP behaviour overrides. Overrides will only have an effect if the corresponding DHCP type is enabled.", + "additionalProperties": false, + "properties": { + "hostname": { + "type": "string", + "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." + }, + "route-metric": { + "type": "integer", + "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." + }, + "send-hostname": { + "type": "boolean", + "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." + }, + "use-dns": { + "type": "boolean" + }, + "use-domains": { + "type": "string" + }, + "use-hostname": { + "type": "boolean" + }, + "use-mtu": { + "type": "boolean", + "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." + }, + "use-ntp": { + "type": "boolean" + }, + "use-routes": { + "type": "boolean", + "description": "Unsupported for dhcp6-overrides when used with the networkd renderer." + } + } + }, + "gateway": { + "type": "string", + "description": "Deprecated, see Netplan#default-routes. Set default gateway for IPv4/6, for manual address configuration. This requires setting addresses too. Gateway IPs must be in a form recognised by inet_pton(3)." + }, + "mapping": { + "type": "object", + "properties": { + "renderer": { + "$ref": "#/$defs/renderer" + }, + "dhcp4": { + "type": ["boolean", "string"], + "description": "Enable DHCP for IPv4. Off by default.", + "enum": ["yes", "no", true, false] + }, + "dhcp6": { + "type": ["boolean", "string"], + "description": "Enable DHCP for IPv6. Off by default.", + "enum": ["yes", "no", true, false] + }, + "dhcp4-overrides": { + "$ref": "#/$defs/dhcp-overrides" + }, + "dhcp6-overrides": { + "$ref": "#/$defs/dhcp-overrides" + }, + "addresses": { + "type": "array", + "description": "Add static addresses to the interface in addition to the ones received through DHCP or RA. Each sequence entry is in CIDR notation, i.e., of the form addr/prefixlen. addr is an IPv4 or IPv6 address as recognised by inet_pton(3) and prefixlen the number of bits of the subnet.", + "items": { + "type": "string" + } + }, + "gateway4": { + "$ref": "#/$defs/gateway" + }, + "gateway6": { + "$ref": "#/$defs/gateway" + }, + "mtu": { + "type": "integer", + "description": "The MTU key represents a device’s Maximum Transmission Unit, the largest size packet or frame, specified in octets (eight-bit bytes), that can be sent in a packet- or frame-based network. Specifying mtu is optional." + }, + "nameservers": { + "type": "object", + "additionalProperties": false, + "description": "Set DNS servers and search domains, for manual address configuration. There are two supported fields: addresses: is a list of IPv4 or IPv6 addresses similar to gateway*, and search: is a list of search domains.", + "properties": { + "search": { + "type": "array", + "items": { + "type": "string" + } + }, + "addresses": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "routes": { + "type": "array", + "description": "", + "items": { + "type": "object", + "required": [ + "to" + ], + "additionalProperties": false, + "properties": { + "to": { + "type": "string" + }, + "via": { + "type": "string" + }, + "metric": { + "type": "integer" + } + } + } + } + } + }, + "mapping_physical": { + "allOf": [ + { + "$ref": "#/$defs/mapping" + } + ], + "properties": { + "match": { + "type": "object", + "description": "This selects a subset of available physical devices by various hardware properties. The following configuration will then apply to all matching devices, as soon as they appear. All specified properties must match.", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Current interface name. Globs are supported, and the primary use case for matching on names, as selecting one fixed name can be more easily achieved with having no match: at all and just using the ID (see above). Note that currently only networkd supports globbing, NetworkManager does not." + }, + "macaddress": { + "type": "string", + "description": "Device’s MAC address in the form xx:xx:xx:xx:xx:xx. Globs are not allowed. Letters must be lowercase." + }, + "driver": { + "type": "string", + "description": "Kernel driver name, corresponding to the DRIVER udev property. Globs are supported. Matching on driver is only supported with networkd." + } + } + }, + "set-name": { + "type": "string", + "description": "When matching on unique properties such as path or MAC, or with additional assumptions such as ''there will only ever be one wifi device'', match rules can be written so that they only match one device. Then this property can be used to give that device a more specific/desirable/nicer name than the default from udev’s ifnames. Any additional device that satisfies the match rules will then fail to get renamed and keep the original kernel name (and dmesg will show an error)." + }, + "wakeonlan": { + "type": "boolean", + "description": "Enable wake on LAN. Off by default." + } + } + }, + "mapping_bond": { + "allOf": [ + { + "$ref": "#/$defs/mapping" + } + ], + "properties": { + "interfaces": { + "type": "array", + "description": "All devices matching this ID list will be added to the bond.", + "items": { + "type": "string" + } + }, + "parameters": { + "type": "object", + "description": "Customisation parameters for special bonding options. Time values are specified in seconds unless otherwise specified.", + "properties": { + "mode": { + "type": "string", + "description": "Set the bonding mode used for the interfaces. The default is balance-rr (round robin).", + "enum": [ + "balance-rr", + "active-backup", + "balance-xor", + "broadcast", + "802.3ad", + "balance-tlb", + "balance-alb" + ] + }, + "lacp-rate": { + "type": "string", + "description": "Set the rate at which LACPDUs are transmitted. This is only useful in 802.3ad mode. Possible values are slow (30 seconds, default), and fast (every second).", + "enum": [ + "fast", + "slow" + ] + }, + "mii-monitor-interval": { + "type": "string", + "description": "Specifies the interval for MII monitoring (verifying if an interface of the bond has carrier). The default is 0; which disables MII monitoring." + }, + "min-links": { + "type": "integer", + "description": "The minimum number of links up in a bond to consider the bond interface to be up." + }, + "transmit-hash-policy": { + "type": "string", + "description": "Specifies the transmit hash policy for the selection of slaves. This is only useful in balance-xor, 802.3ad and balance-tlb modes.", + "enum": [ + "layer2", + "layer3+4", + "layer2+3", + "encap2+3", + "encap3+4" + ] + }, + "ad-select": { + "type": "string", + "description": "Set the aggregation selection mode. This option is only used in 802.3ad mode.", + "enum": [ + "stable", + "bandwidth", + "count" + ] + }, + "all-slaves-active": { + "type": "boolean", + "description": "If the bond should drop duplicate frames received on inactive ports, set this option to false. If they should be delivered, set this option to true. The default value is false, and is the desirable behaviour in most situations." + }, + "arp-interval": { + "type": "integer", + "description": "Set the interval value for how frequently ARP link monitoring should happen. The default value is 0, which disables ARP monitoring." + }, + "arp-ip-targets": { + "type": "array", + "description": "IPs of other hosts on the link which should be sent ARP requests in order to validate that a slave is up. This option is only used when arp-interval is set to a value other than 0. At least one IP address must be given for ARP link monitoring to function. Only IPv4 addresses are supported. You can specify up to 16 IP addresses. The default value is an empty list.", + "items": { + "type": "string" + } + }, + "arp-validate": { + "type": "string", + "description": "Configure how ARP replies are to be validated when using ARP link monitoring.", + "enum": [ + "none", + "active", + "backup", + "all" + ] + }, + "arp-all-targets": { + "type": "string", + "description": "Specify whether to use any ARP IP target being up as sufficient for a slave to be considered up; or if all the targets must be up. This is only used for active-backup mode when arp-validate is enabled.", + "enum": [ + "any", + "all" + ] + }, + "up-delay": { + "type": "integer", + "description": "Specify the delay before enabling a link once the link is physically up. The default value is 0." + }, + "down-delay": { + "type": "integer", + "description": "Specify the delay before enabling a link once the link has been lost. The default value is 0." + }, + "fail-over-mac-policy": { + "type": "string", + "description": "Set whether to set all slaves to the same MAC address when adding them to the bond, or how else the system should handle MAC addresses.", + "enum": [ + "none", + "active", + "follow" + ] + }, + "gratuitous-arp": { + "type": "integer", + "description": "Specify how many ARP packets to send after failover. Once a link is up on a new slave, a notification is sent and possibly repeated if this value is set to a number greater than 1. The default value is 1 and valid values are between 1 and 255. This only affects active-backup mode." + }, + "packets-per-slave": { + "type": "integer", + "description": "In balance-rr mode, specifies the number of packets to transmit on a slave before switching to the next. When this value is set to 0, slaves are chosen at random. Allowable values are between 0 and 65535. The default value is 1. This setting is only used in balance-rr mode." + }, + "primary-reselect-policy": { + "type": "string", + "description": "Set the reselection policy for the primary slave. On failure of the active slave, the system will use this policy to decide how the new active slave will be chosen and how recovery will be handled.", + "enum": [ + "always", + "better", + "failure" + ] + }, + "learn-packet-interval": { + "type": "string", + "description": "Specify the interval between sending Learning packets to each slave. The value range is between 1 and 0x7fffffff. The default value is 1. This option only affects balance-tlb and balance-alb modes." + } + } + } + } + }, + "mapping_bridge": { + "allOf": [ + { + "$ref": "#/$defs/mapping" + } + ], + "properties": { + "interfaces": { + "type": "array", + "description": "All devices matching this ID list will be added to the bridge.", + "items": { + "type": "string" + } + }, + "parameters": { + "type": "object", + "description": "Customisation parameters for special bridging options. Time values are specified in seconds unless otherwise stated.", + "properties": { + "ageing-time": { + "type": "integer", + "description": "Set the period of time to keep a MAC address in the forwarding database after a packet is received." + }, + "priority": { + "type": "integer", + "description": "Set the priority value for the bridge. This value should be a number between 0 and 65535. Lower values mean higher priority. The bridge with the higher priority will be elected as the root bridge." + }, + "forward-delay": { + "type": "integer", + "description": "Specify the period of time the bridge will remain in Listening and Learning states before getting to the Forwarding state. This value should be set in seconds for the systemd backend, and in milliseconds for the NetworkManager backend." + }, + "hello-time": { + "type": "integer", + "description": "Specify the interval between two hello packets being sent out from the root and designated bridges. Hello packets communicate information about the network topology." + }, + "max-age": { + "type": "integer", + "description": "Set the maximum age of a hello packet. If the last hello packet is older than that value, the bridge will attempt to become the root bridge." + }, + "path-cost": { + "type": "integer", + "description": "Set the cost of a path on the bridge. Faster interfaces should have a lower cost. This allows a finer control on the network topology so that the fastest paths are available whenever possible." + }, + "stp": { + "type": "boolean", + "description": "Define whether the bridge should use Spanning Tree Protocol. The default value is “true”, which means that Spanning Tree should be used." + } + } + } + } + }, + "mapping_vlan": { + "allOf": [ + { + "$ref": "#/$defs/mapping" + } + ], + "properties": { + "id": { + "type": "integer", + "description": "VLAN ID, a number between 0 and 4094." + }, + "link": { + "type": "string", + "description": "ID of the underlying device definition on which this VLAN gets created." + } + } + }, + "network_config_version2": { + "type": "object", + "additionalProperties": false, + "required": [ + "version" + ], + "properties": { + "version": { + "type": "integer", + "enum": [ + 2 + ] + }, + "renderer": { + "$ref": "#/$defs/renderer" + }, + "ethernets": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/mapping_physical" + } + }, + "bonds": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/mapping_bond" + } + }, + "bridges": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/mapping_bridge" + } + }, + "vlans": { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/mapping_vlan" + } + } + } + } + }, + "type": "object", + "required": [ + "network" + ], + "properties": { + "network": { + "$ref": "#/$defs/network_config_version2" + } + }, + "additionalProperties": false +} diff --git a/tests/unittests/config/test_schema.py b/tests/unittests/config/test_schema.py index 7dfd6bf0b7d..f9d533e12e8 100644 --- a/tests/unittests/config/test_schema.py +++ b/tests/unittests/config/test_schema.py @@ -1890,8 +1890,7 @@ def test_main_prints_docs(self, _read_cfg_paths, capsys): b"network: {'version': 2, 'ethernets':" b" {'eth0': {'dhcp': true}}}" ), - "Skipping network-config schema validation. No network schema" - " for version: 2", + "Valid schema", ), ( "network-config", @@ -2043,11 +2042,11 @@ def test_main_processed_data_preference_over_raw_data( id="netv1_schema_validated", ), pytest.param( - "network:\n version: 2\n ethernets:\n eth0:\n dhcp4:true\n", - "Skipping network-config schema validation." - " No network schema for version: 2", + "network:\n version: 2\n ethernets:\n eth0:\n" + " dhcp4: true\n", + " Valid schema network-config", does_not_raise(), - id="netv2_validation_is_skipped", + id="netv2_schema_validated", ), pytest.param( "network: {}\n", @@ -2163,7 +2162,8 @@ def _get_meta_doc_examples(file_glob="cloud-config*.txt"): class TestSchemaDocExamples: schema = get_schema() - net_schema = get_schema(schema_type=SchemaType.NETWORK_CONFIG) + net_schema_v1 = get_schema(schema_type=SchemaType.NETWORK_CONFIG_V1) + net_schema_v2 = get_schema(schema_type=SchemaType.NETWORK_CONFIG_V2) @pytest.mark.parametrize("example_path", _get_meta_doc_examples()) @skipUnlessJsonSchema() @@ -2178,7 +2178,21 @@ def test_cloud_config_schema_doc_examples(self, example_path): def test_network_config_schema_v1_doc_examples(self, example_path): validate_cloudconfig_schema( config=yaml.safe_load(open(example_path)), - schema=self.net_schema, + schema=self.net_schema_v1, + schema_type=SchemaType.NETWORK_CONFIG_V1, + strict=True, + ) + + @pytest.mark.parametrize( + "example_path", + _get_meta_doc_examples(file_glob="network-config-v2*yaml"), + ) + @skipUnlessJsonSchema() + def test_network_config_schema_v2_doc_examples(self, example_path): + validate_cloudconfig_schema( + config=yaml.safe_load(open(example_path)), + schema=self.net_schema_v2, + schema_type=SchemaType.NETWORK_CONFIG_V2, strict=True, ) @@ -2239,16 +2253,98 @@ class TestNetworkSchema: net_schema = get_schema(schema_type=SchemaType.NETWORK_CONFIG) @pytest.mark.parametrize( - "src_config, expectation, log", + "src_config, schema_type_version, expectation, log", ( pytest.param( {"network": {"config": [], "version": 2}}, + SchemaType.NETWORK_CONFIG_V2, + pytest.raises( + SchemaValidationError, + match=re.escape( + "Additional properties are not allowed ('config' was " + "unexpected)" + ), + ), + "", + id="net_v2_invalid_config", + ), + pytest.param( + { + "network": { + "version": 2, + "ethernets": {"eno1": {"dhcp4": True}}, + } + }, + SchemaType.NETWORK_CONFIG_V2, does_not_raise(), - "Skipping netplan schema validation. No netplan available", - id="net_v2_skipped", + "", + id="net_v2_simple_example", + ), + pytest.param( + { + "network": { + "version": 2, + "ethernets": { + "id0": { + "match": { + "macaddress": "00:11:22:33:44:55", + }, + "wakeonlan": True, + "dhcp4": True, + "addresses": [ + "192.168.14.2/24", + "2001:1::1/64", + ], + "gateway4": "192.168.14.1", + "gateway6": "2001:1::2", + "nameservers": { + "search": ["foo.local", "bar.local"], + "addresses": ["8.8.8.8"], + }, + "routes": [ + { + "to": "192.0.2.0/24", + "via": "11.0.0.1", + "metric": 3, + }, + ], + }, + "lom": { + "match": {"driver": "ixgbe"}, + "set-name": "lom1", + "dhcp6": True, + }, + "switchports": { + "match": {"name": "enp2*"}, + "mtu": 1280, + }, + }, + "bonds": { + "bond0": {"interfaces": ["id0", "lom"]}, + }, + "bridges": { + "br0": { + "interfaces": ["wlp1s0", "switchports"], + "dhcp4": True, + }, + }, + "vlans": { + "en-intra": { + "id": 1, + "link": "id0", + "dhcp4": "yes", + }, + }, + } + }, + SchemaType.NETWORK_CONFIG_V2, + does_not_raise(), + "", + id="net_v2_complex_example", ), pytest.param( {"network": {"version": 1}}, + SchemaType.NETWORK_CONFIG_V1, pytest.raises( SchemaValidationError, match=re.escape("'config' is a required property"), @@ -2258,6 +2354,7 @@ class TestNetworkSchema: ), pytest.param( {"network": {"version": 1, "config": []}}, + SchemaType.NETWORK_CONFIG_V1, does_not_raise(), "", id="config_key_required", @@ -2269,6 +2366,7 @@ class TestNetworkSchema: "config": [{"name": "me", "type": "typo"}], } }, + SchemaType.NETWORK_CONFIG_V1, pytest.raises( SchemaValidationError, match=( @@ -2281,6 +2379,7 @@ class TestNetworkSchema: ), pytest.param( {"network": {"version": 1, "config": [{"type": "physical"}]}}, + SchemaType.NETWORK_CONFIG_V1, pytest.raises( SchemaValidationError, match=r"network.config.0: 'name' is a required property.*", @@ -2295,6 +2394,7 @@ class TestNetworkSchema: "config": [{"type": "physical", "name": "a"}], } }, + SchemaType.NETWORK_CONFIG_V1, does_not_raise(), "", id="physical_with_name_succeeds", @@ -2308,6 +2408,7 @@ class TestNetworkSchema: ], } }, + SchemaType.NETWORK_CONFIG_V1, pytest.raises( SchemaValidationError, match=r"Additional properties are not allowed.*", @@ -2322,6 +2423,7 @@ class TestNetworkSchema: "config": [VALID_PHYSICAL_CONFIG], } }, + SchemaType.NETWORK_CONFIG_V1, does_not_raise(), "", id="physical_with_all_known_properties", @@ -2333,6 +2435,7 @@ class TestNetworkSchema: "config": [VALID_BOND_CONFIG], } }, + SchemaType.NETWORK_CONFIG_V1, does_not_raise(), "", id="bond_with_all_known_properties", @@ -2347,18 +2450,22 @@ class TestNetworkSchema: ], } }, + SchemaType.NETWORK_CONFIG_V1, does_not_raise(), "", id="GH-4710_mtu_none_and_str_address", ), ), ) - def test_network_schema(self, src_config, expectation, log, caplog): + def test_network_schema( + self, src_config, schema_type_version, expectation, log, caplog + ): + net_schema = get_schema(schema_type=schema_type_version) with expectation: validate_cloudconfig_schema( config=src_config, - schema=self.net_schema, - schema_type=SchemaType.NETWORK_CONFIG, + schema=net_schema, + schema_type=schema_type_version, strict=True, ) if log: