VPC Subnet Routing [1/2] -- RPW and System Routers (#5777)

This PR wires up all the backing machinery for VPC subnet routing, and
automatically resolves and pushes updated rules to sleds using an RPW.
This allows instances in all subnets of a VPC to talk with one another
-- assuming no firewall rules have been configured otherwise. At a high
level, this works by a few changes:
* During the VPC create saga, we now push two rules explicitly to the
system router -- default routes from `(0.0.0.0/0, ::/0) ->
inetgw:outbound`.
* Any CRUD operation on a VPC subnet will reconcile the set of VPC
subnet routes within the system router to have one entry per subnet.
This takes the form `subnet:{name} -> subnet:{name}` for each subnet,
which are later resolved to both v4 and v6 entries.
* Ports are created using route information known to sled-agent -- this
defaults to an empty route set for instances/probes, and an internet
gateway rule for services to enable early NTP sync.
* Routes are sync'd with sleds using a new background task. Broadly,
this asks each sled for the set of VPCs and subnets it has ports on, and
a version for the current route set installed in each. The background
task will use this information to determine which routes must be
rebuilt, and will send updated versions out in response.

The most immediate consequence in this PR is that hosts within a subnet
-- on different VPCs -- will be able to talk with one another at last.
The user facing API (#2116) will be re-enabled in a concurrent PR --
#5823 -- as will NIC spoof detection hole-punching.

Depends on oxidecomputer/opte#490.

Closes #2232, Fixes #1336.

---

A few pieces will block tests passing & merge-readiness:
- [x] Creation of a `lab-2.0-opte-0.32` image.
- [x] Merge of oxidecomputer/maghemite#274 (and updating all the right
SHAs in this PR).

.github/buildomat/jobs/deploy.sh

@@ -2,7 +2,7 @@
 #:
 #: name = "helios / deploy"
 #: variety = "basic"
-#: target = "lab-2.0-opte-0.31"
+#: target = "lab-2.0-opte-0.32"
 #: output_rules = [
 #:  "%/var/svc/log/oxide-sled-agent:default.log*",
 #:  "%/zone/oxz_*/root/var/svc/log/oxide-*.log*",

Cargo.toml

@@ -333,8 +333,8 @@ macaddr = { version = "1.0.1", features = ["serde_std"] }
 maplit = "1.0.2"
 mockall = "0.12"
 newtype_derive = "0.1.6"
-mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "5630887d0373857f77cb264f84aa19bdec720ce3" }
-ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "5630887d0373857f77cb264f84aa19bdec720ce3" }
+mg-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" }
+ddm-admin-client = { git = "https://github.com/oxidecomputer/maghemite", rev = "3c3fa8482fe09a01da62fbd35efe124ea9cac9e7" }
 multimap = "0.10.0"
 nexus-auth = { path = "nexus/auth" }
 nexus-client = { path = "clients/nexus-client" }
@@ -354,7 +354,7 @@ omicron-certificates = { path = "certificates" }
 omicron-passwords = { path = "passwords" }
 omicron-workspace-hack = "0.1.0"
 oxlog = { path = "dev-tools/oxlog" }
-oxnet = { git = "https://github.com/oxidecomputer/oxnet", branch = "main" }
+oxnet = { git = "https://github.com/oxidecomputer/oxnet" }
 nexus-test-interface = { path = "nexus/test-interface" }
 nexus-test-utils-macros = { path = "nexus/test-utils-macros" }
 nexus-test-utils = { path = "nexus/test-utils" }
@@ -372,14 +372,14 @@ omicron-sled-agent = { path = "sled-agent" }
 omicron-test-utils = { path = "test-utils" }
 omicron-zone-package = "0.11.0"
 oxide-client = { path = "clients/oxide-client" }
-oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "417f74e94978c23f3892ac328c3387f3ecd9bb29", features = [ "api", "std" ] }
+oxide-vpc = { git = "https://github.com/oxidecomputer/opte", rev = "915975f6d1729db95619f752148974016912412f", features = [ "api", "std" ] }
 once_cell = "1.19.0"
 openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" }
 openapiv3 = "2.0.0"
 # must match samael's crate!
 openssl = "0.10"
 openssl-sys = "0.9"
-opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "417f74e94978c23f3892ac328c3387f3ecd9bb29" }
+opte-ioctl = { git = "https://github.com/oxidecomputer/opte", rev = "915975f6d1729db95619f752148974016912412f" }
 oso = "0.27"
 owo-colors = "4.0.0"
 oximeter = { path = "oximeter/oximeter" }

clients/ddm-admin-client/src/lib.rs

@@ -12,7 +12,7 @@
 pub use ddm_admin_client::types;
 pub use ddm_admin_client::Error;
 
-use ddm_admin_client::types::{Ipv6Prefix, TunnelOrigin};
+use ddm_admin_client::types::TunnelOrigin;
 use ddm_admin_client::Client as InnerClient;
 use either::Either;
 use omicron_common::address::Ipv6Subnet;
@@ -81,8 +81,7 @@ impl Client {
     pub fn advertise_prefix(&self, address: Ipv6Subnet<SLED_PREFIX>) {
         let me = self.clone();
         tokio::spawn(async move {
-            let prefix =
-                Ipv6Prefix { addr: address.net().prefix(), len: SLED_PREFIX };
+            let prefix = address.net();
             retry_notify(retry_policy_internal_service_aggressive(), || async {
                 info!(
                     me.log, "Sending prefix to ddmd for advertisement";
@@ -130,8 +129,8 @@ impl Client {
         let prefixes = self.inner.get_prefixes().await?.into_inner();
         Ok(prefixes.into_iter().flat_map(|(_, prefixes)| {
             prefixes.into_iter().flat_map(|prefix| {
-                let mut segments = prefix.destination.addr.segments();
-                if prefix.destination.len == BOOTSTRAP_MASK
+                let mut segments = prefix.destination.addr().segments();
+                if prefix.destination.width() == BOOTSTRAP_MASK
                     && segments[0] == BOOTSTRAP_PREFIX
                 {
                     Either::Left(interfaces.iter().map(move |interface| {

clients/sled-agent-client/src/lib.rs

@@ -51,6 +51,11 @@ progenitor::generate_api!(
         NetworkInterface = omicron_common::api::internal::shared::NetworkInterface,
         PortFec = omicron_common::api::internal::shared::PortFec,
         PortSpeed = omicron_common::api::internal::shared::PortSpeed,
+        RouterId = omicron_common::api::internal::shared::RouterId,
+        ResolvedVpcRoute = omicron_common::api::internal::shared::ResolvedVpcRoute,
+        ResolvedVpcRouteSet = omicron_common::api::internal::shared::ResolvedVpcRouteSet,
+        RouterTarget = omicron_common::api::internal::shared::RouterTarget,
+        RouterVersion = omicron_common::api::internal::shared::RouterVersion,
         SourceNatConfig = omicron_common::api::internal::shared::SourceNatConfig,
         SwitchLocation = omicron_common::api::external::SwitchLocation,
         TypedUuidForInstanceKind = omicron_uuid_kinds::InstanceUuid,

common/src/api/external/mod.rs

@@ -1331,6 +1331,9 @@ pub enum RouteTarget {
     #[display("inetgw:{0}")]
     /// Forward traffic to an internet gateway
     InternetGateway(Name),
+    #[display("drop")]
+    /// Drop matching traffic
+    Drop,
 }
 
 /// A `RouteDestination` is used to match traffic with a routing rule, on the

common/src/api/internal/shared.rs

@@ -6,13 +6,13 @@
 
 use crate::{
     address::NUM_SOURCE_NAT_PORTS,
-    api::external::{self, BfdMode, ImportExportPolicy, Name},
+    api::external::{self, BfdMode, ImportExportPolicy, Name, Vni},
 };
 use oxnet::{IpNet, Ipv4Net, Ipv6Net};
 use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use std::{
-    collections::HashMap,
+    collections::{HashMap, HashSet},
     fmt,
     net::{IpAddr, Ipv4Addr, Ipv6Addr},
     str::FromStr,
@@ -50,11 +50,11 @@ pub enum NetworkInterfaceKind {
 pub struct NetworkInterface {
     pub id: Uuid,
     pub kind: NetworkInterfaceKind,
-    pub name: external::Name,
+    pub name: Name,
     pub ip: IpAddr,
     pub mac: external::MacAddr,
     pub subnet: IpNet,
-    pub vni: external::Vni,
+    pub vni: Vni,
     pub primary: bool,
     pub slot: u8,
 }
@@ -624,6 +624,82 @@ impl TryFrom<&[ipnetwork::IpNetwork]> for IpAllowList {
     }
 }
 
+/// A VPC route resolved into a concrete target.
+#[derive(
+    Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash,
+)]
+pub struct ResolvedVpcRoute {
+    pub dest: IpNet,
+    pub target: RouterTarget,
+}
+
+/// The target for a given router entry.
+#[derive(
+    Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash,
+)]
+#[serde(tag = "type", rename_all = "snake_case", content = "value")]
+pub enum RouterTarget {
+    Drop,
+    InternetGateway,
+    Ip(IpAddr),
+    VpcSubnet(IpNet),
+}
+
+/// Information on the current parent router (and version) of a route set
+/// according to the control plane.
+#[derive(
+    Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash,
+)]
+pub struct RouterVersion {
+    pub router_id: Uuid,
+    pub version: u64,
+}
+
+impl RouterVersion {
+    /// Return whether a new route set should be applied over the current
+    /// values.
+    ///
+    /// This will occur when seeing a new version and a matching parent,
+    /// or a new parent router on the control plane.
+    pub fn is_replaced_by(&self, other: &Self) -> bool {
+        (self.router_id != other.router_id) || self.version < other.version
+    }
+}
+
+/// Identifier for a VPC and/or subnet.
+#[derive(
+    Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash,
+)]
+pub struct RouterId {
+    pub vni: Vni,
+    pub kind: RouterKind,
+}
+
+/// The scope of a set of VPC router rules.
+#[derive(
+    Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, Hash,
+)]
+#[serde(tag = "type", rename_all = "snake_case", content = "subnet")]
+pub enum RouterKind {
+    System,
+    Custom(IpNet),
+}
+
+/// Version information for routes on a given VPC subnet.
+#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
+pub struct ResolvedVpcRouteState {
+    pub id: RouterId,
+    pub version: Option<RouterVersion>,
+}
+
+/// An updated set of routes for a given VPC and/or subnet.
+#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq)]
+pub struct ResolvedVpcRouteSet {
+    pub id: RouterId,
+    pub version: Option<RouterVersion>,
+    pub routes: HashSet<ResolvedVpcRoute>,
+}
+
 #[cfg(test)]
 mod tests {
     use crate::api::internal::shared::AllowedSourceIps;

dev-tools/omdb/tests/env.out

@@ -131,6 +131,10 @@ task: "v2p_manager"
     manages opte v2p mappings for vpc networking
 
 
+task: "vpc_route_manager"
+    propagates updated VPC routes to all OPTE ports
+
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT
@@ -259,6 +263,10 @@ task: "v2p_manager"
     manages opte v2p mappings for vpc networking
 
 
+task: "vpc_route_manager"
+    propagates updated VPC routes to all OPTE ports
+
+
 ---------------------------------------------
 stderr:
 note: Nexus URL not specified.  Will pick one from DNS.
@@ -374,6 +382,10 @@ task: "v2p_manager"
     manages opte v2p mappings for vpc networking
 
 
+task: "vpc_route_manager"
+    propagates updated VPC routes to all OPTE ports
+
+
 ---------------------------------------------
 stderr:
 note: Nexus URL not specified.  Will pick one from DNS.

dev-tools/omdb/tests/successes.out

@@ -332,6 +332,10 @@ task: "v2p_manager"
     manages opte v2p mappings for vpc networking
 
 
+task: "vpc_route_manager"
+    propagates updated VPC routes to all OPTE ports
+
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT/
@@ -545,6 +549,13 @@ task: "v2p_manager"
     started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
 warning: unknown background task: "v2p_manager" (don't know how to interpret details: Object {})
 
+task: "vpc_route_manager"
+  configured period: every 30s
+  currently executing: no
+  last completed activation: <REDACTED ITERATIONS>, triggered by an explicit signal
+    started at <REDACTED     TIMESTAMP> (<REDACTED DURATION>s ago) and ran for <REDACTED DURATION>ms
+warning: unknown background task: "vpc_route_manager" (don't know how to interpret details: Object {})
+
 ---------------------------------------------
 stderr:
 note: using Nexus URL http://127.0.0.1:REDACTED_PORT/