Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Oracle netplan v2 migration #6024

Draft
wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

Oracle netplan v2 migration #6024

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
40f865d
super crappy initial impl of oracle netplan v2 migration
a-dubs Feb 10, 2025
06f36cf
all unit tests passing and basic impl done!
a-dubs Feb 13, 2025
19c0aaa
trying to fix linting
a-dubs Feb 17, 2025
9a08de7
removed v1 stuffs from oracle ds
a-dubs Feb 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
217 changes: 156 additions & 61 deletions cloudinit/sources/DataSourceOracle.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,25 @@
"""

import base64
import difflib
import ipaddress
import json
import logging
import time
from pprint import pformat
from typing import Any, Dict, List, NamedTuple, Optional, Tuple

import yaml

from cloudinit import atomic_helper, dmi, net, sources, util
from cloudinit.distros.networking import NetworkConfig
from cloudinit.net import (
cmdline,
ephemeral,
get_interfaces_by_mac,
is_netfail_master,
netplan,
network_state,
)
from cloudinit.url_helper import wait_for_url

Expand Down Expand Up @@ -149,7 +155,8 @@ def __init__(self, sys_cfg, *args, **kwargs):
]
)
self._network_config_source = KlibcOracleNetworkConfigSource()
self._network_config: dict = {"config": [], "version": 1}
self._network_config_v1: dict = {"config": [], "version": 1}
self._network_config: dict = {"ethernets": {}, "version": 2}

url_params = self.get_url_params()
self.url_max_wait = url_params.max_wait_seconds
Expand All @@ -167,9 +174,16 @@ def _unpickle(self, ci_pkl_version: int) -> None:
)
if not hasattr(self, "_network_config"):
self._network_config = {"config": [], "version": 1}
if not hasattr(self, "_network_config_v1"):
self._network_config_v1 = {"config": [], "version": 1}

def _has_network_config(self) -> bool:
return bool(self._network_config.get("config", []))
if self._network_config:
if self._network_config.get("version") == 1:
return bool(self._network_config.get("config"))
elif self._network_config.get("version") == 2:
return bool(self._network_config.get("ethernets"))
return False

@staticmethod
def ds_detect() -> bool:
Expand Down Expand Up @@ -276,9 +290,12 @@ def _is_iscsi_root(self) -> bool:
"""Return whether we are on a iscsi machine."""
return self._network_config_source.is_applicable()

def _get_iscsi_config(self) -> dict:
def _get_iscsi_config_v1(self) -> dict:
return self._network_config_source.render_config()

def _get_iscsi_config(self) -> dict:
return convert_v1_netplan_to_v2(self._get_iscsi_config_v1())

@property
def network_config(self):
"""Network config is read from initramfs provided files
Expand All @@ -295,6 +312,7 @@ def network_config(self):
set_primary = False
# this is v1
if self._is_iscsi_root():
self._network_config_v1 = self._get_iscsi_config_v1()
self._network_config = self._get_iscsi_config()
if not self._has_network_config():
LOG.warning(
Expand Down Expand Up @@ -380,71 +398,148 @@ def _add_network_config_from_opc_imds(self, set_primary: bool = False):
else:
network = ipaddress.ip_network(vnic_dict["subnetCidrBlock"])

if self._network_config["version"] == 1:
if is_primary:
if is_ipv6_only:
subnets = [{"type": "dhcp6"}]
else:
subnets = [{"type": "dhcp"}]
############################# V1 HERE #############################

if is_primary:
if is_ipv6_only:
subnets = [{"type": "dhcp6"}]
else:
subnets = []
if vnic_dict.get("privateIp"):
subnets.append(
{
"type": "static",
"address": (
f"{vnic_dict['privateIp']}/"
f"{network.prefixlen}"
),
}
)
if vnic_dict.get("ipv6Addresses"):
subnets.append(
{
"type": "static",
"address": (
f"{vnic_dict['ipv6Addresses'][0]}/"
f"{network.prefixlen}"
),
}
)
interface_config = {
"name": name,
"type": "physical",
"mac_address": mac_address,
"mtu": MTU,
"subnets": subnets,
}
self._network_config["config"].append(interface_config)
elif self._network_config["version"] == 2:
# Why does this elif exist???
# Are there plans to switch to v2?
interface_config = {
"mtu": MTU,
"match": {"macaddress": mac_address},
}
self._network_config["ethernets"][name] = interface_config

interface_config["dhcp6"] = is_primary and is_ipv6_only
interface_config["dhcp4"] = is_primary and not is_ipv6_only
if not is_primary:
interface_config["addresses"] = []
if vnic_dict.get("privateIp"):
interface_config["addresses"].append(
f"{vnic_dict['privateIp']}/{network.prefixlen}"
)
if vnic_dict.get("ipv6Addresses"):
interface_config["addresses"].append(
f"{vnic_dict['ipv6Addresses'][0]}/"
f"{network.prefixlen}"
)
self._network_config["ethernets"][name] = interface_config
subnets = [{"type": "dhcp"}]
else:
subnets = []
if vnic_dict.get("privateIp"):
subnets.append(
{
"type": "static",
"address": (
f"{vnic_dict['privateIp']}/"
f"{network.prefixlen}"
),
}
)
if vnic_dict.get("ipv6Addresses"):
subnets.append(
{
"type": "static",
"address": (
f"{vnic_dict['ipv6Addresses'][0]}/"
f"{network.prefixlen}"
),
}
)
interface_config = {
"name": name,
"type": "physical",
"mac_address": mac_address,
"mtu": MTU,
"subnets": subnets,
}
self._network_config_v1["config"].append(interface_config)

############################# V2 HERE #############################
# Why does this elif exist???
# Are there plans to switch to v2?
interface_config = {
"mtu": MTU,
"match": {"macaddress": mac_address},
"set-name": name,
}
self._network_config["ethernets"][name] = interface_config

if is_primary and is_ipv6_only:
interface_config["dhcp6"] = True
elif is_primary and not is_ipv6_only:
interface_config["dhcp4"] = True

if not is_primary:
interface_config["addresses"] = []
if vnic_dict.get("privateIp"):
interface_config["addresses"].append(
f"{vnic_dict['privateIp']}/{network.prefixlen}"
)
if vnic_dict.get("ipv6Addresses"):
interface_config["addresses"].append(
f"{vnic_dict['ipv6Addresses'][0]}/"
f"{network.prefixlen}"
)
self._network_config["ethernets"][name] = interface_config

LOG.debug("Comparing IMDS network configs between v1 and v2")
compare_v1_and_v2_config_entries(
self._network_config_v1, self._network_config
)


class DataSourceOracleNet(DataSourceOracle):
perform_dhcp_setup = False


def convert_v1_netplan_to_v2(network_config: dict) -> dict:
has_base_network_key = "network" in network_config
if has_base_network_key:
network_config = network_config["network"]
v1_network_state = network_state.parse_net_config_data(
network_config, renderer=netplan.Renderer
)
netplan_v2_yaml_string = netplan.Renderer()._render_content(
v1_network_state
)
netplan_v2_dict = yaml.safe_load(netplan_v2_yaml_string)
if has_base_network_key:
if "network" not in netplan_v2_dict:
netplan_v2_dict = {"network": netplan_v2_dict}
if not has_base_network_key:
if "network" in netplan_v2_dict:
netplan_v2_dict = netplan_v2_dict["network"]
return netplan_v2_dict


def move_version_to_beggining_of_nteplan_yaml_str(yaml_str: str) -> str:
version_line = [
line for line in yaml_str.split("\n") if "version:" in line
][0]
lines = [line for line in yaml_str.split("\n") if "version:" not in line]
lines.insert(1, version_line)
return "\n".join(lines)


def compare_v1_and_v2_config_entries(v1_config, v2_config):
v1_network_state = network_state.parse_net_config_data(
v1_config, renderer=netplan.Renderer
)
v2_network_state = network_state.parse_net_config_data(
v2_config, renderer=netplan.Renderer
)
v1_rendered_string = netplan.Renderer()._render_content(v1_network_state)
v2_rendered_string = netplan.Renderer()._render_content(v2_network_state)
v1_rendered_string = move_version_to_beggining_of_nteplan_yaml_str(
yaml_str=v1_rendered_string,
)
v2_rendered_string = move_version_to_beggining_of_nteplan_yaml_str(
yaml_str=v2_rendered_string,
)
LOG.debug("v1:\n%s\n\n%s", pformat(v1_config), v1_rendered_string)
LOG.debug("v2:\n%s\n\n%s", pformat(v2_config), v2_rendered_string)

if yaml.safe_load(v1_rendered_string) != yaml.safe_load(
v2_rendered_string
):
diff = difflib.unified_diff(
v1_rendered_string.splitlines(),
v2_rendered_string.splitlines(),
lineterm="",
)
LOG.warning(
"oracle datasource v1 and v2 network config entries do not match! "
"diff:\n%s",
"\n".join(diff),
)
return False

LOG.debug("oracle datasource v1 and v2 network config entries match!")
return True


def _is_ipv4_metadata_url(metadata_address: str):
if not metadata_address:
return False
Expand Down
46 changes: 46 additions & 0 deletions tests/integration_tests/datasources/test_oci_networking.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import logging
import re
from typing import Iterator, Set

Expand All @@ -9,6 +10,8 @@
from tests.integration_tests.integration_settings import PLATFORM
from tests.integration_tests.util import verify_clean_boot, verify_clean_log

logger = logging.getLogger(__name__)

DS_CFG = """\
datasource:
Oracle:
Expand Down Expand Up @@ -157,3 +160,46 @@ def test_oci_networking_system_cfg(client: IntegrationInstance, tmpdir):
netplan_cfg = yaml.safe_load(netplan_yaml)
expected_netplan_cfg = yaml.safe_load(SYSTEM_CFG)
assert expected_netplan_cfg == netplan_cfg


def _install_custom_cloudinit(client: IntegrationInstance, restart=True):
client.push_file(
"cloud-init_all.deb",
"/home/ubuntu/cloud-init.deb",
)
r1 = client.execute("sudo apt remove cloud-init --assume-yes -y")
assert r1.return_code == 0
r2 = client.execute("sudo apt install -y /home/ubuntu/cloud-init.deb")
assert r2.return_code == 0
r3 = client.execute("cloud-init --version")
logger.info(r3.stdout)
assert r3.return_code == 0
if restart:
client.execute("cloud-init clean --logs")
logger.info("Restarting instance")
client.instance.restart()


# function that looks for the v1 vs v2 logs
def test_v1_and_v2_network_config_match(
client_with_secondary_vnic: IntegrationInstance, tmpdir
):
client = client_with_secondary_vnic
_install_custom_cloudinit(client, restart=False)
# make it so that that the instance will configure secondary nics
customize_environment(client, tmpdir, configure_secondary_nics=True)

# pull the log file locally for easier debugging
client.pull_file("/var/log/cloud-init.log", "cpc-6431-cloud-init.log")

log = client.read_from_file("/var/log/cloud-init.log")
verify_clean_log(log)
verify_clean_boot(client)

# assert that the v1 and v2 network config entries match
# compare_v1_and_v2_config_entries(v1_config, v2_config)

assert "Comparing IMDS network configs between v1 and v2" in log

# and that the debug log is present
assert "oracle datasource v1 and v2 network config entries match!" in log
Loading
Loading