diff --git a/.github/buildomat/jobs/tuf-repo.sh b/.github/buildomat/jobs/tuf-repo.sh index 59fc0fd7a7..14c2293f5b 100644 --- a/.github/buildomat/jobs/tuf-repo.sh +++ b/.github/buildomat/jobs/tuf-repo.sh @@ -278,8 +278,8 @@ EOF done } # usage: SERIES ROT_DIR ROT_VERSION BOARDS... -add_hubris_artifacts rot-staging-dev staging/dev cert-staging-dev-v1.0.5 "${ALL_BOARDS[@]}" -add_hubris_artifacts rot-prod-rel prod/rel cert-prod-rel-v1.0.5 "${ALL_BOARDS[@]}" +add_hubris_artifacts rot-staging-dev staging/dev cert-staging-dev-v1.0.6 "${ALL_BOARDS[@]}" +add_hubris_artifacts rot-prod-rel prod/rel cert-prod-rel-v1.0.6 "${ALL_BOARDS[@]}" for series in "${SERIES_LIST[@]}"; do /work/tufaceous assemble --no-generate-key /work/manifest-"$series".toml /work/repo-"$series".zip diff --git a/.github/workflows/hakari.yml b/.github/workflows/hakari.yml index 68bc323a07..1f55f2f255 100644 --- a/.github/workflows/hakari.yml +++ b/.github/workflows/hakari.yml @@ -24,7 +24,7 @@ jobs: with: toolchain: stable - name: Install cargo-hakari - uses: taiki-e/install-action@12af778b97addf4c562c75a0564dc7e7dc5339a5 # v2 + uses: taiki-e/install-action@19e9b549a48620cc50fcf6e6e866b8fb4eca1b01 # v2 with: tool: cargo-hakari - name: Check workspace-hack Cargo.toml is up-to-date diff --git a/nexus/blueprint-execution/src/lib.rs b/nexus/blueprint-execution/src/lib.rs index a13acdf265..d6f5f8fc31 100644 --- a/nexus/blueprint-execution/src/lib.rs +++ b/nexus/blueprint-execution/src/lib.rs @@ -21,6 +21,7 @@ use uuid::Uuid; mod dns; mod omicron_zones; +mod resource_allocation; struct Sled { id: Uuid, @@ -69,6 +70,14 @@ where "blueprint_id" => ?blueprint.id ); + resource_allocation::ensure_zone_resources_allocated( + &opctx, + datastore, + &blueprint.omicron_zones, + ) + .await + .map_err(|err| vec![err])?; + let sleds_by_id: BTreeMap = datastore .sled_list_all_batched(&opctx) .await @@ -82,9 +91,9 @@ where dns::deploy_dns( &opctx, - &datastore, + datastore, String::from(nexus_label), - &blueprint, + blueprint, &sleds_by_id, ) .await diff --git a/nexus/blueprint-execution/src/resource_allocation.rs b/nexus/blueprint-execution/src/resource_allocation.rs new file mode 100644 index 0000000000..7f3ebb9876 --- /dev/null +++ b/nexus/blueprint-execution/src/resource_allocation.rs @@ -0,0 +1,992 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Manges allocation of resources required for blueprint realization + +use anyhow::bail; +use anyhow::Context; +use nexus_db_model::IncompleteNetworkInterface; +use nexus_db_model::Name; +use nexus_db_model::SqlU16; +use nexus_db_model::VpcSubnet; +use nexus_db_queries::context::OpContext; +use nexus_db_queries::db::fixed_data::vpc_subnet::DNS_VPC_SUBNET; +use nexus_db_queries::db::fixed_data::vpc_subnet::NEXUS_VPC_SUBNET; +use nexus_db_queries::db::fixed_data::vpc_subnet::NTP_VPC_SUBNET; +use nexus_db_queries::db::DataStore; +use nexus_types::deployment::NetworkInterface; +use nexus_types::deployment::NetworkInterfaceKind; +use nexus_types::deployment::OmicronZoneType; +use nexus_types::deployment::OmicronZonesConfig; +use nexus_types::deployment::SourceNatConfig; +use omicron_common::api::external::IdentityMetadataCreateParams; +use slog::info; +use slog::warn; +use std::collections::BTreeMap; +use std::net::IpAddr; +use std::net::SocketAddr; +use uuid::Uuid; + +pub(crate) async fn ensure_zone_resources_allocated( + opctx: &OpContext, + datastore: &DataStore, + zones: &BTreeMap, +) -> anyhow::Result<()> { + let allocator = ResourceAllocator { opctx, datastore }; + + for config in zones.values() { + for z in &config.zones { + match &z.zone_type { + OmicronZoneType::Nexus { external_ip, nic, .. } => { + allocator + .ensure_nexus_external_networking_allocated( + z.id, + *external_ip, + nic, + ) + .await?; + } + OmicronZoneType::ExternalDns { dns_address, nic, .. } => { + allocator + .ensure_external_dns_external_networking_allocated( + z.id, + dns_address, + nic, + ) + .await?; + } + OmicronZoneType::BoundaryNtp { snat_cfg, nic, .. } => { + allocator + .ensure_boundary_ntp_external_networking_allocated( + z.id, snat_cfg, nic, + ) + .await?; + } + OmicronZoneType::InternalNtp { .. } + | OmicronZoneType::Clickhouse { .. } + | OmicronZoneType::ClickhouseKeeper { .. } + | OmicronZoneType::CockroachDb { .. } + | OmicronZoneType::Crucible { .. } + | OmicronZoneType::CruciblePantry { .. } + | OmicronZoneType::InternalDns { .. } + | OmicronZoneType::Oximeter { .. } => (), + } + } + } + + Ok(()) +} + +struct ResourceAllocator<'a> { + opctx: &'a OpContext, + datastore: &'a DataStore, +} + +impl<'a> ResourceAllocator<'a> { + // Helper function to determine whether a given external IP address is + // already allocated to a specific service zone. + async fn is_external_ip_already_allocated( + &self, + zone_type: &'static str, + zone_id: Uuid, + external_ip: IpAddr, + port_range: Option<(u16, u16)>, + ) -> anyhow::Result { + let allocated_ips = self + .datastore + .service_lookup_external_ips(self.opctx, zone_id) + .await + .with_context(|| { + format!( + "failed to look up external IPs for {zone_type} {zone_id}" + ) + })?; + + if !allocated_ips.is_empty() { + // All the service zones that want external IP addresses only expect + // to have a single IP. This service already has (at least) one: + // make sure this list includes the one we want, or return an error. + for allocated_ip in &allocated_ips { + if allocated_ip.ip.ip() == external_ip + && port_range + .map(|(first, last)| { + allocated_ip.first_port == SqlU16(first) + && allocated_ip.last_port == SqlU16(last) + }) + .unwrap_or(true) + { + info!( + self.opctx.log, "found already-allocated external IP"; + "zone_type" => zone_type, + "zone_id" => %zone_id, + "ip" => %external_ip, + ); + return Ok(true); + } + } + + warn!( + self.opctx.log, "zone has unexpected IPs allocated"; + "zone_type" => zone_type, + "zone_id" => %zone_id, + "want_ip" => %external_ip, + "allocated_ips" => ?allocated_ips, + ); + bail!( + "zone {zone_id} already has {} non-matching IP(s) allocated", + allocated_ips.len() + ); + } + + info!( + self.opctx.log, "external IP allocation required for zone"; + "zone_type" => zone_type, + "zone_id" => %zone_id, + "ip" => %external_ip, + ); + + Ok(false) + } + + // Helper function to determine whether a given NIC is already allocated to + // a specific service zone. + async fn is_nic_already_allocated( + &self, + zone_type: &'static str, + zone_id: Uuid, + nic: &NetworkInterface, + ) -> anyhow::Result { + let allocated_nics = self + .datastore + .service_list_network_interfaces(self.opctx, zone_id) + .await + .with_context(|| { + format!("failed to look up NICs for {zone_type} {zone_id}") + })?; + + if !allocated_nics.is_empty() { + // All the service zones that want NICs only expect to have a single + // one. Bail out here if this zone already has one or more allocated + // NICs but not the one we think it needs. + // + // This doesn't check the allocated NIC's subnet against our NICs, + // because that would require an extra DB lookup. We'll assume if + // these main properties are correct, the subnet is too. + for allocated_nic in &allocated_nics { + if allocated_nic.ip.ip() == nic.ip + && *allocated_nic.mac == nic.mac + && allocated_nic.slot == i16::from(nic.slot) + && allocated_nic.primary == nic.primary + { + info!( + self.opctx.log, "found already-allocated NIC"; + "zone_type" => zone_type, + "zone_id" => %zone_id, + "nic" => ?allocated_nic, + ); + return Ok(true); + } + } + + warn!( + self.opctx.log, "zone has unexpected NICs allocated"; + "zone_type" => zone_type, + "zone_id" => %zone_id, + "want_nic" => ?nic, + "allocated_nics" => ?allocated_nics, + ); + + bail!( + "zone {zone_id} already has {} non-matching NIC(s) allocated", + allocated_nics.len() + ); + } + + info!( + self.opctx.log, "NIC allocation required for zone"; + "zone_type" => zone_type, + "zone_id" => %zone_id, + "nid" => ?nic, + ); + + Ok(false) + } + + // Nexus and ExternalDns both use non-SNAT service IPs; this method is used + // to allocate external networking for both of them. + async fn ensure_external_service_ip( + &self, + zone_type: &'static str, + service_id: Uuid, + external_ip: IpAddr, + ip_name: &Name, + ) -> anyhow::Result<()> { + // Only attempt to allocate `external_ip` if it isn't already assigned + // to this zone. + // + // Checking for the existing of the external IP and then creating it + // if not found inserts a classic TOCTOU race: what if another Nexus + // is running concurrently, we both check and see that the IP is not + // allocated, then both attempt to create it? We believe this is + // okay: the loser of the race (i.e., the one whose create tries to + // commit second) will fail to allocate the IP, which will bubble + // out and prevent realization of the current blueprint. That's + // exactly what we want if two Nexuses try to realize the same + // blueprint at the same time. + if self + .is_external_ip_already_allocated( + zone_type, + service_id, + external_ip, + None, + ) + .await? + { + return Ok(()); + } + let ip_id = Uuid::new_v4(); + let description = zone_type; + self.datastore + .allocate_explicit_service_ip( + self.opctx, + ip_id, + ip_name, + description, + service_id, + external_ip, + ) + .await + .with_context(|| { + format!( + "failed to allocate IP to {zone_type} {service_id}: \ + {external_ip}" + ) + })?; + + info!( + self.opctx.log, "successfully allocated external IP"; + "zone_type" => zone_type, + "zone_id" => %service_id, + "ip" => %external_ip, + "ip_id" => %ip_id, + ); + + Ok(()) + } + + // BoundaryNtp uses a SNAT service IPs; this method is similar to + // `ensure_external_service_ip` but accounts for that. + async fn ensure_external_service_snat_ip( + &self, + zone_type: &'static str, + service_id: Uuid, + snat: &SourceNatConfig, + ) -> anyhow::Result<()> { + // Only attempt to allocate `external_ip` if it isn't already assigned + // to this zone. + // + // This is subject to the same kind of TOCTOU race as described for IP + // allocation in `ensure_external_service_ip`, and we believe it's okay + // for the same reasons as described there. + if self + .is_external_ip_already_allocated( + zone_type, + service_id, + snat.ip, + Some((snat.first_port, snat.last_port)), + ) + .await? + { + return Ok(()); + } + + let ip_id = Uuid::new_v4(); + self.datastore + .allocate_explicit_service_snat_ip( + self.opctx, + ip_id, + service_id, + snat.ip, + (snat.first_port, snat.last_port), + ) + .await + .with_context(|| { + format!( + "failed to allocate snat IP to {zone_type} {service_id}: \ + {snat:?}" + ) + })?; + + info!( + self.opctx.log, "successfully allocated external SNAT IP"; + "zone_type" => zone_type, + "zone_id" => %service_id, + "snat" => ?snat, + "ip_id" => %ip_id, + ); + + Ok(()) + } + + // All service zones with external connectivity get service vNICs. + async fn ensure_service_nic( + &self, + zone_type: &'static str, + service_id: Uuid, + nic: &NetworkInterface, + nic_subnet: &VpcSubnet, + ) -> anyhow::Result<()> { + // We don't pass `nic.kind` into the database below, but instead + // explicitly call `service_create_network_interface`. Ensure this is + // indeed a service NIC. + match &nic.kind { + NetworkInterfaceKind::Instance { .. } => { + bail!("invalid NIC kind (expected service, got instance)") + } + NetworkInterfaceKind::Service { .. } => (), + } + + // Only attempt to allocate `nic` if it isn't already assigned to this + // zone. + // + // This is subject to the same kind of TOCTOU race as described for IP + // allocation in `ensure_external_service_ip`, and we believe it's okay + // for the same reasons as described there. + if self.is_nic_already_allocated(zone_type, service_id, nic).await? { + return Ok(()); + } + let nic_arg = IncompleteNetworkInterface::new_service( + nic.id, + service_id, + nic_subnet.clone(), + IdentityMetadataCreateParams { + name: nic.name.clone(), + description: format!("{zone_type} service vNIC"), + }, + nic.ip, + nic.mac, + nic.slot, + ) + .with_context(|| { + format!( + "failed to convert NIC into IncompleteNetworkInterface: {nic:?}" + ) + })?; + let created_nic = self + .datastore + .service_create_network_interface(self.opctx, nic_arg) + .await + .map_err(|err| err.into_external()) + .with_context(|| { + format!( + "failed to allocate NIC to {zone_type} {service_id}: \ + {nic:?}" + ) + })?; + + // We don't pass all the properties of `nic` into the create request + // above. Double-check that the properties the DB assigned match + // what we expect. + // + // We do not check `nic.vni`, because it's not stored in the + // database. (All services are given the constant vni + // `Vni::SERVICES_VNI`.) + if created_nic.primary != nic.primary + || created_nic.slot != i16::from(nic.slot) + { + warn!( + self.opctx.log, "unexpected property on allocated NIC"; + "db_primary" => created_nic.primary, + "expected_primary" => nic.primary, + "db_slot" => created_nic.slot, + "expected_slot" => nic.slot, + ); + + // Now what? We've allocated a NIC in the database but it's + // incorrect. Should we try to delete it? That would be best + // effort (we could fail to delete, or we could crash between + // creation and deletion). + // + // We only expect services to have one NIC, so the only way it + // should be possible to get a different primary/slot value is + // if somehow this same service got a _different_ NIC allocated + // to it in the TOCTOU race window above. That should be + // impossible with the way we generate blueprints, so we'll just + // return a scary error here and expect to never see it. + bail!( + "database cleanup required: \ + unexpected NIC ({created_nic:?}) \ + allocated for {zone_type} {service_id}" + ); + } + + info!( + self.opctx.log, "successfully allocated service vNIC"; + "zone_type" => zone_type, + "zone_id" => %service_id, + "nic" => ?nic, + ); + + Ok(()) + } + + async fn ensure_nexus_external_networking_allocated( + &self, + zone_id: Uuid, + external_ip: IpAddr, + nic: &NetworkInterface, + ) -> anyhow::Result<()> { + self.ensure_external_service_ip( + "nexus", + zone_id, + external_ip, + &Name(nic.name.clone()), + ) + .await?; + self.ensure_service_nic("nexus", zone_id, nic, &NEXUS_VPC_SUBNET) + .await?; + Ok(()) + } + + async fn ensure_external_dns_external_networking_allocated( + &self, + zone_id: Uuid, + dns_address: &str, + nic: &NetworkInterface, + ) -> anyhow::Result<()> { + let dns_address = + dns_address.parse::().with_context(|| { + format!("failed to parse ExternalDns address {dns_address}") + })?; + self.ensure_external_service_ip( + "external_dns", + zone_id, + dns_address.ip(), + &Name(nic.name.clone()), + ) + .await?; + self.ensure_service_nic("external_dns", zone_id, nic, &DNS_VPC_SUBNET) + .await?; + Ok(()) + } + + async fn ensure_boundary_ntp_external_networking_allocated( + &self, + zone_id: Uuid, + snat: &SourceNatConfig, + nic: &NetworkInterface, + ) -> anyhow::Result<()> { + self.ensure_external_service_snat_ip("ntp", zone_id, snat).await?; + self.ensure_service_nic("ntp", zone_id, nic, &NTP_VPC_SUBNET).await?; + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nexus_test_utils_macros::nexus_test; + use nexus_types::deployment::OmicronZoneConfig; + use nexus_types::deployment::OmicronZoneDataset; + use nexus_types::identity::Resource; + use omicron_common::address::IpRange; + use omicron_common::address::DNS_OPTE_IPV4_SUBNET; + use omicron_common::address::NEXUS_OPTE_IPV4_SUBNET; + use omicron_common::address::NTP_OPTE_IPV4_SUBNET; + use omicron_common::address::NUM_SOURCE_NAT_PORTS; + use omicron_common::api::external::Generation; + use omicron_common::api::external::IpNet; + use omicron_common::api::external::MacAddr; + use omicron_common::api::external::Vni; + use omicron_common::nexus_config::NUM_INITIAL_RESERVED_IP_ADDRESSES; + use std::net::IpAddr; + use std::net::Ipv6Addr; + use std::net::SocketAddrV6; + + type ControlPlaneTestContext = + nexus_test_utils::ControlPlaneTestContext; + + #[nexus_test] + async fn test_allocate_external_networking( + cptestctx: &ControlPlaneTestContext, + ) { + // Set up. + let nexus = &cptestctx.server.apictx().nexus; + let datastore = nexus.datastore(); + let opctx = OpContext::for_tests( + cptestctx.logctx.log.clone(), + datastore.clone(), + ); + + // Create an external IP range we can use for our services. + let external_ip_range = IpRange::try_from(( + "192.0.2.1".parse::().unwrap(), + "192.0.2.100".parse::().unwrap(), + )) + .expect("bad IP range"); + let mut external_ips = external_ip_range.iter(); + + // Add the external IP range to the services IP pool. + let (ip_pool, _) = datastore + .ip_pools_service_lookup(&opctx) + .await + .expect("failed to find service IP pool"); + datastore + .ip_pool_add_range(&opctx, &ip_pool, &external_ip_range) + .await + .expect("failed to expand service IP pool"); + + // Generate the values we care about. (Other required zone config params + // that we don't care about will be filled in below arbitrarily.) + + // Nexus: + let nexus_id = Uuid::new_v4(); + let nexus_external_ip = + external_ips.next().expect("exhausted external_ips"); + let nexus_nic = NetworkInterface { + id: Uuid::new_v4(), + kind: NetworkInterfaceKind::Service(nexus_id), + name: "test-nexus".parse().expect("bad name"), + ip: NEXUS_OPTE_IPV4_SUBNET + .iter() + .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES) + .unwrap() + .into(), + mac: MacAddr::random_system(), + subnet: IpNet::from(*NEXUS_OPTE_IPV4_SUBNET).into(), + vni: Vni::SERVICES_VNI, + primary: true, + slot: 0, + }; + + // External DNS: + let dns_id = Uuid::new_v4(); + let dns_external_ip = + external_ips.next().expect("exhausted external_ips"); + let dns_nic = NetworkInterface { + id: Uuid::new_v4(), + kind: NetworkInterfaceKind::Service(dns_id), + name: "test-external-dns".parse().expect("bad name"), + ip: DNS_OPTE_IPV4_SUBNET + .iter() + .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES) + .unwrap() + .into(), + mac: MacAddr::random_system(), + subnet: IpNet::from(*DNS_OPTE_IPV4_SUBNET).into(), + vni: Vni::SERVICES_VNI, + primary: true, + slot: 0, + }; + + // Boundary NTP: + let ntp_id = Uuid::new_v4(); + let ntp_snat = SourceNatConfig { + ip: external_ips.next().expect("exhausted external_ips"), + first_port: NUM_SOURCE_NAT_PORTS, + last_port: 2 * NUM_SOURCE_NAT_PORTS - 1, + }; + let ntp_nic = NetworkInterface { + id: Uuid::new_v4(), + kind: NetworkInterfaceKind::Service(ntp_id), + name: "test-external-ntp".parse().expect("bad name"), + ip: NTP_OPTE_IPV4_SUBNET + .iter() + .nth(NUM_INITIAL_RESERVED_IP_ADDRESSES) + .unwrap() + .into(), + mac: MacAddr::random_system(), + subnet: IpNet::from(*NTP_OPTE_IPV4_SUBNET).into(), + vni: Vni::SERVICES_VNI, + primary: true, + slot: 0, + }; + + // Build the `zones` map needed by `ensure_zone_resources_allocated`, + // with an arbitrary sled_id. + let mut zones = BTreeMap::new(); + let sled_id = Uuid::new_v4(); + zones.insert( + sled_id, + OmicronZonesConfig { + generation: Generation::new().next(), + zones: vec![ + OmicronZoneConfig { + id: nexus_id, + underlay_address: Ipv6Addr::LOCALHOST, + zone_type: OmicronZoneType::Nexus { + internal_address: Ipv6Addr::LOCALHOST.to_string(), + external_ip: nexus_external_ip, + nic: nexus_nic.clone(), + external_tls: false, + external_dns_servers: Vec::new(), + }, + }, + OmicronZoneConfig { + id: dns_id, + underlay_address: Ipv6Addr::LOCALHOST, + zone_type: OmicronZoneType::ExternalDns { + dataset: OmicronZoneDataset { + pool_name: format!("oxp_{}", Uuid::new_v4()) + .parse() + .expect("bad name"), + }, + http_address: SocketAddrV6::new( + Ipv6Addr::LOCALHOST, + 0, + 0, + 0, + ) + .to_string(), + dns_address: SocketAddr::new(dns_external_ip, 0) + .to_string(), + nic: dns_nic.clone(), + }, + }, + OmicronZoneConfig { + id: ntp_id, + underlay_address: Ipv6Addr::LOCALHOST, + zone_type: OmicronZoneType::BoundaryNtp { + address: SocketAddr::new(dns_external_ip, 0) + .to_string(), + ntp_servers: Vec::new(), + dns_servers: Vec::new(), + domain: None, + nic: ntp_nic.clone(), + snat_cfg: ntp_snat, + }, + }, + ], + }, + ); + + // Initialize resource allocation: this should succeed and create all + // the relevant db records. + ensure_zone_resources_allocated(&opctx, datastore, &zones) + .await + .with_context(|| format!("{zones:#?}")) + .unwrap(); + + // Check that the external IP records were created. + let db_nexus_ips = datastore + .service_lookup_external_ips(&opctx, nexus_id) + .await + .expect("failed to get external IPs"); + assert_eq!(db_nexus_ips.len(), 1); + assert!(db_nexus_ips[0].is_service); + assert_eq!(db_nexus_ips[0].parent_id, Some(nexus_id)); + assert_eq!(db_nexus_ips[0].ip, nexus_external_ip.into()); + assert_eq!(db_nexus_ips[0].first_port, SqlU16(0)); + assert_eq!(db_nexus_ips[0].last_port, SqlU16(65535)); + + let db_dns_ips = datastore + .service_lookup_external_ips(&opctx, dns_id) + .await + .expect("failed to get external IPs"); + assert_eq!(db_dns_ips.len(), 1); + assert!(db_dns_ips[0].is_service); + assert_eq!(db_dns_ips[0].parent_id, Some(dns_id)); + assert_eq!(db_dns_ips[0].ip, dns_external_ip.into()); + assert_eq!(db_dns_ips[0].first_port, SqlU16(0)); + assert_eq!(db_dns_ips[0].last_port, SqlU16(65535)); + + let db_ntp_ips = datastore + .service_lookup_external_ips(&opctx, ntp_id) + .await + .expect("failed to get external IPs"); + assert_eq!(db_ntp_ips.len(), 1); + assert!(db_ntp_ips[0].is_service); + assert_eq!(db_ntp_ips[0].parent_id, Some(ntp_id)); + assert_eq!(db_ntp_ips[0].ip, ntp_snat.ip.into()); + assert_eq!(db_ntp_ips[0].first_port, SqlU16(ntp_snat.first_port)); + assert_eq!(db_ntp_ips[0].last_port, SqlU16(ntp_snat.last_port)); + + // Check that the NIC records were created. + let db_nexus_nics = datastore + .service_list_network_interfaces(&opctx, nexus_id) + .await + .expect("failed to get NICs"); + assert_eq!(db_nexus_nics.len(), 1); + assert_eq!(db_nexus_nics[0].id(), nexus_nic.id); + assert_eq!(db_nexus_nics[0].service_id, nexus_id); + assert_eq!(db_nexus_nics[0].vpc_id, NEXUS_VPC_SUBNET.vpc_id); + assert_eq!(db_nexus_nics[0].subnet_id, NEXUS_VPC_SUBNET.id()); + assert_eq!(*db_nexus_nics[0].mac, nexus_nic.mac); + assert_eq!(db_nexus_nics[0].ip, nexus_nic.ip.into()); + assert_eq!(db_nexus_nics[0].slot, i16::from(nexus_nic.slot)); + assert_eq!(db_nexus_nics[0].primary, nexus_nic.primary); + + let db_dns_nics = datastore + .service_list_network_interfaces(&opctx, dns_id) + .await + .expect("failed to get NICs"); + assert_eq!(db_dns_nics.len(), 1); + assert_eq!(db_dns_nics[0].id(), dns_nic.id); + assert_eq!(db_dns_nics[0].service_id, dns_id); + assert_eq!(db_dns_nics[0].vpc_id, DNS_VPC_SUBNET.vpc_id); + assert_eq!(db_dns_nics[0].subnet_id, DNS_VPC_SUBNET.id()); + assert_eq!(*db_dns_nics[0].mac, dns_nic.mac); + assert_eq!(db_dns_nics[0].ip, dns_nic.ip.into()); + assert_eq!(db_dns_nics[0].slot, i16::from(dns_nic.slot)); + assert_eq!(db_dns_nics[0].primary, dns_nic.primary); + + let db_ntp_nics = datastore + .service_list_network_interfaces(&opctx, ntp_id) + .await + .expect("failed to get NICs"); + assert_eq!(db_ntp_nics.len(), 1); + assert_eq!(db_ntp_nics[0].id(), ntp_nic.id); + assert_eq!(db_ntp_nics[0].service_id, ntp_id); + assert_eq!(db_ntp_nics[0].vpc_id, NTP_VPC_SUBNET.vpc_id); + assert_eq!(db_ntp_nics[0].subnet_id, NTP_VPC_SUBNET.id()); + assert_eq!(*db_ntp_nics[0].mac, ntp_nic.mac); + assert_eq!(db_ntp_nics[0].ip, ntp_nic.ip.into()); + assert_eq!(db_ntp_nics[0].slot, i16::from(ntp_nic.slot)); + assert_eq!(db_ntp_nics[0].primary, ntp_nic.primary); + + // We should be able to run the function again with the same inputs, and + // it should succeed without inserting any new records. + ensure_zone_resources_allocated(&opctx, datastore, &zones) + .await + .with_context(|| format!("{zones:#?}")) + .unwrap(); + assert_eq!( + db_nexus_ips, + datastore + .service_lookup_external_ips(&opctx, nexus_id) + .await + .expect("failed to get external IPs") + ); + assert_eq!( + db_dns_ips, + datastore + .service_lookup_external_ips(&opctx, dns_id) + .await + .expect("failed to get external IPs") + ); + assert_eq!( + db_ntp_ips, + datastore + .service_lookup_external_ips(&opctx, ntp_id) + .await + .expect("failed to get external IPs") + ); + assert_eq!( + db_nexus_nics, + datastore + .service_list_network_interfaces(&opctx, nexus_id) + .await + .expect("failed to get NICs") + ); + assert_eq!( + db_dns_nics, + datastore + .service_list_network_interfaces(&opctx, dns_id) + .await + .expect("failed to get NICs") + ); + assert_eq!( + db_ntp_nics, + datastore + .service_list_network_interfaces(&opctx, ntp_id) + .await + .expect("failed to get NICs") + ); + + // Now that we've tested the happy path, try some requests that ought to + // fail because the request includes an external IP that doesn't match + // the already-allocated external IPs from above. + let bogus_ip = external_ips.next().expect("exhausted external_ips"); + for mutate_zones_fn in [ + // non-matching IP on Nexus + (&|config: &mut OmicronZonesConfig| { + for zone in &mut config.zones { + if let OmicronZoneType::Nexus { + ref mut external_ip, .. + } = &mut zone.zone_type + { + *external_ip = bogus_ip; + return format!( + "zone {} already has 1 non-matching IP", + zone.id + ); + } + } + panic!("didn't find expected zone"); + }) as &dyn Fn(&mut OmicronZonesConfig) -> String, + // non-matching IP on External DNS + &|config| { + for zone in &mut config.zones { + if let OmicronZoneType::ExternalDns { + ref mut dns_address, + .. + } = &mut zone.zone_type + { + *dns_address = SocketAddr::new(bogus_ip, 0).to_string(); + return format!( + "zone {} already has 1 non-matching IP", + zone.id + ); + } + } + panic!("didn't find expected zone"); + }, + // non-matching SNAT port range on Boundary NTP + &|config| { + for zone in &mut config.zones { + if let OmicronZoneType::BoundaryNtp { + ref mut snat_cfg, + .. + } = &mut zone.zone_type + { + snat_cfg.first_port += NUM_SOURCE_NAT_PORTS; + snat_cfg.last_port += NUM_SOURCE_NAT_PORTS; + return format!( + "zone {} already has 1 non-matching IP", + zone.id + ); + } + } + panic!("didn't find expected zone"); + }, + ] { + // Run `mutate_zones_fn` on our config... + let mut config = + zones.remove(&sled_id).expect("missing zone config"); + let orig_config = config.clone(); + let expected_error = mutate_zones_fn(&mut config); + zones.insert(sled_id, config); + + // ... check that we get the error we expect + let err = + ensure_zone_resources_allocated(&opctx, datastore, &zones) + .await + .expect_err("unexpected success"); + assert!( + err.to_string().contains(&expected_error), + "expected {expected_error:?}, got {err:#}" + ); + + // ... and restore the original, valid config before iterating. + zones.insert(sled_id, orig_config); + } + + // Also try some requests that ought to fail because the request + // includes a NIC that doesn't match the already-allocated NICs from + // above. + // + // All three zone types have a `nic` property, so here our mutating + // function only modifies that, and the body of our loop tries it on all + // three to ensure we get the errors we expect no matter the zone type. + for mutate_nic_fn in [ + // switch kind from Service to Instance + (&|_: Uuid, nic: &mut NetworkInterface| { + match &nic.kind { + NetworkInterfaceKind::Instance { .. } => { + panic!( + "invalid NIC kind (expected service, got instance)" + ) + } + NetworkInterfaceKind::Service(id) => { + let id = *id; + nic.kind = NetworkInterfaceKind::Instance(id); + } + } + "invalid NIC kind".to_string() + }) as &dyn Fn(Uuid, &mut NetworkInterface) -> String, + // non-matching IP + &|zone_id, nic| { + nic.ip = bogus_ip; + format!("zone {zone_id} already has 1 non-matching NIC") + }, + ] { + // Try this NIC mutation on Nexus... + let mut mutated_zones = zones.clone(); + for zone in &mut mutated_zones + .get_mut(&sled_id) + .expect("missing sled") + .zones + { + if let OmicronZoneType::Nexus { ref mut nic, .. } = + &mut zone.zone_type + { + let expected_error = mutate_nic_fn(zone.id, nic); + + let err = ensure_zone_resources_allocated( + &opctx, + datastore, + &mutated_zones, + ) + .await + .expect_err("unexpected success"); + + assert!( + err.to_string().contains(&expected_error), + "expected {expected_error:?}, got {err:#}" + ); + + break; + } + } + + // ... and again on ExternalDns + let mut mutated_zones = zones.clone(); + for zone in &mut mutated_zones + .get_mut(&sled_id) + .expect("missing sled") + .zones + { + if let OmicronZoneType::ExternalDns { ref mut nic, .. } = + &mut zone.zone_type + { + let expected_error = mutate_nic_fn(zone.id, nic); + + let err = ensure_zone_resources_allocated( + &opctx, + datastore, + &mutated_zones, + ) + .await + .expect_err("unexpected success"); + + assert!( + err.to_string().contains(&expected_error), + "expected {expected_error:?}, got {err:#}" + ); + + break; + } + } + + // ... and again on BoundaryNtp + let mut mutated_zones = zones.clone(); + for zone in &mut mutated_zones + .get_mut(&sled_id) + .expect("missing sled") + .zones + { + if let OmicronZoneType::BoundaryNtp { ref mut nic, .. } = + &mut zone.zone_type + { + let expected_error = mutate_nic_fn(zone.id, nic); + + let err = ensure_zone_resources_allocated( + &opctx, + datastore, + &mutated_zones, + ) + .await + .expect_err("unexpected success"); + + assert!( + err.to_string().contains(&expected_error), + "expected {expected_error:?}, got {err:#}" + ); + + break; + } + } + } + } +} diff --git a/nexus/db-model/src/external_ip.rs b/nexus/db-model/src/external_ip.rs index 1e9def4182..b30f91c7c0 100644 --- a/nexus/db-model/src/external_ip.rs +++ b/nexus/db-model/src/external_ip.rs @@ -33,7 +33,7 @@ impl_enum_type!( #[diesel(postgres_type(name = "ip_kind", schema = "public"))] pub struct IpKindEnum; - #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Deserialize, Serialize)] + #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Eq, Deserialize, Serialize)] #[diesel(sql_type = IpKindEnum)] pub enum IpKind; @@ -47,7 +47,7 @@ impl_enum_type!( #[diesel(postgres_type(name = "ip_attach_state"))] pub struct IpAttachStateEnum; - #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Deserialize, Serialize)] + #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, PartialEq, Eq, Deserialize, Serialize)] #[diesel(sql_type = IpAttachStateEnum)] pub enum IpAttachState; @@ -89,7 +89,15 @@ impl std::fmt::Display for IpKind { /// API at all, and only provide outbound connectivity to instances, not /// inbound. #[derive( - Debug, Clone, Selectable, Queryable, Insertable, Deserialize, Serialize, + Debug, + Clone, + Selectable, + Queryable, + Insertable, + Deserialize, + Serialize, + PartialEq, + Eq, )] #[diesel(table_name = external_ip)] pub struct ExternalIp { diff --git a/nexus/db-model/src/network_interface.rs b/nexus/db-model/src/network_interface.rs index 01317fc160..72752ae3f8 100644 --- a/nexus/db-model/src/network_interface.rs +++ b/nexus/db-model/src/network_interface.rs @@ -91,7 +91,7 @@ pub struct InstanceNetworkInterface { /// The underlying "table" (`service_network_interface`) is actually a view /// over the `network_interface` table, that contains only rows with /// `kind = 'service'`. -#[derive(Selectable, Queryable, Clone, Debug, Resource)] +#[derive(Selectable, Queryable, Clone, Debug, PartialEq, Eq, Resource)] #[diesel(table_name = service_network_interface)] pub struct ServiceNetworkInterface { #[diesel(embed)] diff --git a/nexus/db-model/src/schema.rs b/nexus/db-model/src/schema.rs index bd18ba260a..d8344c2258 100644 --- a/nexus/db-model/src/schema.rs +++ b/nexus/db-model/src/schema.rs @@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion; /// /// This should be updated whenever the schema is changed. For more details, /// refer to: schema/crdb/README.adoc -pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(36, 0, 0); +pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(37, 0, 1); table! { disk (id) { diff --git a/nexus/db-queries/src/context.rs b/nexus/db-queries/src/context.rs index d93b269255..dfd1fe4322 100644 --- a/nexus/db-queries/src/context.rs +++ b/nexus/db-queries/src/context.rs @@ -303,10 +303,17 @@ impl OpContext { /// either. Clients and proxies often don't expect long requests and /// apply aggressive timeouts. Depending on the HTTP version, a /// long-running request can tie up the TCP connection. + /// + /// We shouldn't allow these in either internal or external API handlers, + /// but we currently have some internal APIs for exercising some expensive + /// blueprint operations and so we allow these cases here. pub fn check_complex_operations_allowed(&self) -> Result<(), Error> { let api_handler = match self.kind { - OpKind::ExternalApiRequest | OpKind::InternalApiRequest => true, - OpKind::Saga | OpKind::Background | OpKind::Test => false, + OpKind::ExternalApiRequest => true, + OpKind::InternalApiRequest + | OpKind::Saga + | OpKind::Background + | OpKind::Test => false, }; if api_handler { Err(Error::internal_error( diff --git a/nexus/db-queries/src/db/datastore/external_ip.rs b/nexus/db-queries/src/db/datastore/external_ip.rs index 9d4d947476..24439aa3a0 100644 --- a/nexus/db-queries/src/db/datastore/external_ip.rs +++ b/nexus/db-queries/src/db/datastore/external_ip.rs @@ -167,6 +167,23 @@ impl DataStore { } } + /// Fetch all external IP addresses of any kind for the provided service. + pub async fn service_lookup_external_ips( + &self, + opctx: &OpContext, + service_id: Uuid, + ) -> LookupResult> { + use db::schema::external_ip::dsl; + dsl::external_ip + .filter(dsl::is_service.eq(true)) + .filter(dsl::parent_id.eq(service_id)) + .filter(dsl::time_deleted.is_null()) + .select(ExternalIp::as_select()) + .get_results_async(&*self.pool_connection_authorized(opctx).await?) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + /// Allocates an IP address for internal service usage. pub async fn allocate_service_ip( &self, @@ -337,7 +354,8 @@ impl DataStore { service_id: Uuid, ip: IpAddr, ) -> CreateResult { - let (.., pool) = self.ip_pools_service_lookup(opctx).await?; + let (authz_pool, pool) = self.ip_pools_service_lookup(opctx).await?; + opctx.authorize(authz::Action::CreateChild, &authz_pool).await?; let data = IncompleteExternalIp::for_service_explicit( ip_id, name, @@ -361,7 +379,8 @@ impl DataStore { ip: IpAddr, port_range: (u16, u16), ) -> CreateResult { - let (.., pool) = self.ip_pools_service_lookup(opctx).await?; + let (authz_pool, pool) = self.ip_pools_service_lookup(opctx).await?; + opctx.authorize(authz::Action::CreateChild, &authz_pool).await?; let data = IncompleteExternalIp::for_service_explicit_snat( ip_id, service_id, diff --git a/nexus/db-queries/src/db/datastore/network_interface.rs b/nexus/db-queries/src/db/datastore/network_interface.rs index d715bf3889..f2782e8f67 100644 --- a/nexus/db-queries/src/db/datastore/network_interface.rs +++ b/nexus/db-queries/src/db/datastore/network_interface.rs @@ -29,6 +29,7 @@ use async_bb8_diesel::AsyncRunQueryDsl; use chrono::Utc; use diesel::prelude::*; use diesel::result::Error as DieselError; +use nexus_db_model::ServiceNetworkInterface; use omicron_common::api::external; use omicron_common::api::external::http_pagination::PaginatedBy; use omicron_common::api::external::DeleteResult; @@ -126,15 +127,69 @@ impl DataStore { .map(NetworkInterface::as_instance) } - #[cfg(test)] + /// List network interfaces associated with a given service. + pub async fn service_list_network_interfaces( + &self, + opctx: &OpContext, + service_id: Uuid, + ) -> ListResultVec { + // See the comment in `service_create_network_interface`. There's no + // obvious parent for a service network interface (as opposed to + // instance network interfaces, which require ListChildren on the + // instance to list). As a logical proxy, we check for listing children + // of the service IP pool. + let (authz_service_ip_pool, _) = + self.ip_pools_service_lookup(opctx).await?; + opctx + .authorize(authz::Action::ListChildren, &authz_service_ip_pool) + .await?; + + use db::schema::service_network_interface::dsl; + dsl::service_network_interface + .filter(dsl::time_deleted.is_null()) + .filter(dsl::service_id.eq(service_id)) + .select(ServiceNetworkInterface::as_select()) + .get_results_async::( + &*self.pool_connection_authorized(opctx).await?, + ) + .await + .map_err(|e| public_error_from_diesel(e, ErrorHandler::Server)) + } + + /// Create a network interface attached to the provided service zone. + pub async fn service_create_network_interface( + &self, + opctx: &OpContext, + interface: IncompleteNetworkInterface, + ) -> Result { + // In `instance_create_network_interface`, the authz checks are for + // creating children of the VpcSubnet and the instance. We don't have an + // instance. We do have a VpcSubet, but for services these are all + // fixed data subnets. + // + // As a proxy auth check that isn't really guarding the right resource + // but should logically be equivalent, we can insert a authz check for + // creating children of the service IP pool. For any service zone with + // external networking, we create an external IP (in the service IP + // pool) and a network interface (in the relevant VpcSubnet). Putting + // this check here ensures that the caller can't proceed if they also + // couldn't proceed with creating the corresponding external IP. + let (authz_service_ip_pool, _) = self + .ip_pools_service_lookup(opctx) + .await + .map_err(network_interface::InsertError::External)?; + opctx + .authorize(authz::Action::CreateChild, &authz_service_ip_pool) + .await + .map_err(network_interface::InsertError::External)?; + self.service_create_network_interface_raw(opctx, interface).await + } + pub(crate) async fn service_create_network_interface_raw( &self, opctx: &OpContext, interface: IncompleteNetworkInterface, - ) -> Result< - db::model::ServiceNetworkInterface, - network_interface::InsertError, - > { + ) -> Result { if interface.kind != NetworkInterfaceKind::Service { return Err(network_interface::InsertError::External( Error::invalid_request( diff --git a/nexus/db-queries/src/db/queries/external_ip.rs b/nexus/db-queries/src/db/queries/external_ip.rs index 6676e5d531..739c0b0809 100644 --- a/nexus/db-queries/src/db/queries/external_ip.rs +++ b/nexus/db-queries/src/db/queries/external_ip.rs @@ -1332,6 +1332,18 @@ mod tests { // Allocate an IP address as we would for an external, rack-associated // service. let service1_id = Uuid::new_v4(); + + // Check that `service_lookup_external_ips` returns an empty vector for + // a service with no external IPs. + assert_eq!( + context + .db_datastore + .service_lookup_external_ips(&context.opctx, service1_id) + .await + .expect("Failed to look up service external IPs"), + Vec::new(), + ); + let id1 = Uuid::new_v4(); let ip1 = context .db_datastore @@ -1350,6 +1362,14 @@ mod tests { assert_eq!(ip1.first_port.0, 0); assert_eq!(ip1.last_port.0, u16::MAX); assert_eq!(ip1.parent_id, Some(service1_id)); + assert_eq!( + context + .db_datastore + .service_lookup_external_ips(&context.opctx, service1_id) + .await + .expect("Failed to look up service external IPs"), + vec![ip1], + ); // Allocate an SNat IP let service2_id = Uuid::new_v4(); @@ -1365,6 +1385,14 @@ mod tests { assert_eq!(ip2.first_port.0, 0); assert_eq!(ip2.last_port.0, 16383); assert_eq!(ip2.parent_id, Some(service2_id)); + assert_eq!( + context + .db_datastore + .service_lookup_external_ips(&context.opctx, service2_id) + .await + .expect("Failed to look up service external IPs"), + vec![ip2], + ); // Allocate the next IP address let service3_id = Uuid::new_v4(); @@ -1386,6 +1414,14 @@ mod tests { assert_eq!(ip3.first_port.0, 0); assert_eq!(ip3.last_port.0, u16::MAX); assert_eq!(ip3.parent_id, Some(service3_id)); + assert_eq!( + context + .db_datastore + .service_lookup_external_ips(&context.opctx, service3_id) + .await + .expect("Failed to look up service external IPs"), + vec![ip3], + ); // Once we're out of IP addresses, test that we see the right error. let service3_id = Uuid::new_v4(); @@ -1423,6 +1459,14 @@ mod tests { assert_eq!(ip4.first_port.0, 16384); assert_eq!(ip4.last_port.0, 32767); assert_eq!(ip4.parent_id, Some(service4_id)); + assert_eq!( + context + .db_datastore + .service_lookup_external_ips(&context.opctx, service4_id) + .await + .expect("Failed to look up service external IPs"), + vec![ip4], + ); context.success().await; } diff --git a/nexus/src/app/rack.rs b/nexus/src/app/rack.rs index 7a1ad0e6a9..a137f19434 100644 --- a/nexus/src/app/rack.rs +++ b/nexus/src/app/rack.rs @@ -61,6 +61,7 @@ use sled_agent_client::types::{ BgpConfig, BgpPeerConfig as SledBgpPeerConfig, EarlyNetworkConfig, PortConfigV1, RackNetworkConfigV1, RouteConfig as SledRouteConfig, }; +use slog_error_chain::InlineErrorChain; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashMap; @@ -647,7 +648,7 @@ impl super::Nexus { if rack.rack_subnet.is_some() { return Ok(()); } - let sa = self.get_any_sled_agent(opctx).await?; + let sa = self.get_any_sled_agent_client(opctx).await?; let result = sa .read_network_bootstore_config_cache() .await @@ -883,7 +884,27 @@ impl super::Nexus { }, }, }; - let sa = self.get_any_sled_agent(opctx).await?; + + // This timeout value is fairly arbitrary (as they usually are). As of + // this writing, this operation is known to take close to two minutes on + // production hardware. + let dur = std::time::Duration::from_secs(300); + let sa_url = self.get_any_sled_agent_url(opctx).await?; + let reqwest_client = reqwest::ClientBuilder::new() + .connect_timeout(dur) + .timeout(dur) + .build() + .map_err(|e| { + Error::internal_error(&format!( + "failed to create reqwest client for sled agent: {}", + InlineErrorChain::new(&e) + )) + })?; + let sa = sled_agent_client::Client::new_with_client( + &sa_url, + reqwest_client, + self.log.new(o!("sled_agent_url" => sa_url.clone())), + ); sa.sled_add(&req).await.map_err(|e| Error::InternalError { internal_message: format!( "failed to add sled with baseboard {:?} to rack {}: {e}", @@ -899,10 +920,10 @@ impl super::Nexus { Ok(()) } - async fn get_any_sled_agent( + async fn get_any_sled_agent_url( &self, opctx: &OpContext, - ) -> Result { + ) -> Result { let addr = self .sled_list(opctx, &DataPageParams::max_page()) .await? @@ -911,11 +932,15 @@ impl super::Nexus { internal_message: "no sled agents available".into(), })? .address(); + Ok(format!("http://{}", addr)) + } - Ok(sled_agent_client::Client::new( - &format!("http://{}", addr), - self.log.clone(), - )) + async fn get_any_sled_agent_client( + &self, + opctx: &OpContext, + ) -> Result { + let url = self.get_any_sled_agent_url(opctx).await?; + Ok(sled_agent_client::Client::new(&url, self.log.clone())) } } diff --git a/nexus/tests/integration_tests/schema.rs b/nexus/tests/integration_tests/schema.rs index 1c23f1b842..380ec1c975 100644 --- a/nexus/tests/integration_tests/schema.rs +++ b/nexus/tests/integration_tests/schema.rs @@ -1048,7 +1048,8 @@ fn after_23_0_0(client: &Client) -> BoxFuture<'_, ()> { fn before_24_0_0(client: &Client) -> BoxFuture<'_, ()> { // IP addresses were pulled off dogfood sled 16 Box::pin(async move { - // Create two sleds + // Create two sleds. (SLED2 is marked non_provisionable for + // after_37_0_1.) client .batch_execute(&format!( "INSERT INTO sled @@ -1062,7 +1063,7 @@ fn before_24_0_0(client: &Client) -> BoxFuture<'_, ()> { 'fd00:1122:3344:104::1ac', 'provisionable'), ('{SLED2}', now(), now(), NULL, 1, '{RACK1}', false, 'zzzz', 'xxxx', '2', 64, 12345678, 77,'fd00:1122:3344:107::1', 12345, - 'fd00:1122:3344:107::d4', 'provisionable'); + 'fd00:1122:3344:107::d4', 'non_provisionable'); " )) .await @@ -1095,6 +1096,45 @@ fn after_24_0_0(client: &Client) -> BoxFuture<'_, ()> { }) } +// This reuses the sleds created in before_24_0_0. +fn after_37_0_1(client: &Client) -> BoxFuture<'_, ()> { + Box::pin(async { + // Confirm that the IP Addresses have the last 2 bytes changed to `0xFFFF` + let rows = client + .query("SELECT sled_policy, sled_state FROM sled ORDER BY id", &[]) + .await + .expect("Failed to select sled policy and state"); + let policy_and_state = process_rows(&rows); + + assert_eq!( + policy_and_state[0].values, + vec![ + ColumnValue::new( + "sled_policy", + SqlEnum::from(("sled_policy", "in_service")) + ), + ColumnValue::new( + "sled_state", + SqlEnum::from(("sled_state", "active")) + ), + ] + ); + assert_eq!( + policy_and_state[1].values, + vec![ + ColumnValue::new( + "sled_policy", + SqlEnum::from(("sled_policy", "no_provision")) + ), + ColumnValue::new( + "sled_state", + SqlEnum::from(("sled_state", "active")) + ), + ] + ); + }) +} + // Lazily initializes all migration checks. The combination of Rust function // pointers and async makes defining a static table fairly painful, so we're // using lazy initialization instead. @@ -1112,6 +1152,10 @@ fn get_migration_checks() -> BTreeMap { SemverVersion(semver::Version::parse("24.0.0").unwrap()), DataMigrationFns { before: Some(before_24_0_0), after: after_24_0_0 }, ); + map.insert( + SemverVersion(semver::Version::parse("37.0.1").unwrap()), + DataMigrationFns { before: None, after: after_37_0_1 }, + ); map } diff --git a/nexus/types/src/external_api/views.rs b/nexus/types/src/external_api/views.rs index bff0f5cfe7..a3e87b162e 100644 --- a/nexus/types/src/external_api/views.rs +++ b/nexus/types/src/external_api/views.rs @@ -470,7 +470,7 @@ impl SledProvisionPolicy { #[derive( Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, PartialEq, Eq, )] -#[serde(rename_all = "snake_case")] +#[serde(rename_all = "snake_case", tag = "kind")] pub enum SledPolicy { /// The operator has indicated that the sled is in-service. InService { diff --git a/openapi/nexus.json b/openapi/nexus.json index fe7bd96579..8f0fd997ca 100644 --- a/openapi/nexus.json +++ b/openapi/nexus.json @@ -14839,11 +14839,11 @@ "type": "string", "format": "uuid" }, - "provision_state": { - "description": "The provision state of the sled.", + "policy": { + "description": "The operator-defined policy of a sled.", "allOf": [ { - "$ref": "#/components/schemas/SledProvisionPolicy" + "$ref": "#/components/schemas/SledPolicy" } ] }, @@ -14852,6 +14852,14 @@ "type": "string", "format": "uuid" }, + "state": { + "description": "The current state Nexus believes the sled to be in.", + "allOf": [ + { + "$ref": "#/components/schemas/SledState" + } + ] + }, "time_created": { "description": "timestamp when this resource was created", "type": "string", @@ -14880,8 +14888,9 @@ "required": [ "baseboard", "id", - "provision_state", + "policy", "rack_id", + "state", "time_created", "time_modified", "usable_hardware_threads", @@ -14971,8 +14980,52 @@ "items" ] }, + "SledPolicy": { + "description": "The operator-defined policy of a sled.", + "oneOf": [ + { + "description": "The operator has indicated that the sled is in-service.", + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "in_service" + ] + }, + "provision_policy": { + "description": "Determines whether new resources can be provisioned onto the sled.", + "allOf": [ + { + "$ref": "#/components/schemas/SledProvisionPolicy" + } + ] + } + }, + "required": [ + "kind", + "provision_policy" + ] + }, + { + "description": "The operator has indicated that the sled has been permanently removed from service.\n\nThis is a terminal state: once a particular sled ID is expunged, it will never return to service. (The actual hardware may be reused, but it will be treated as a brand-new sled.)\n\nAn expunged sled is always non-provisionable.", + "type": "object", + "properties": { + "kind": { + "type": "string", + "enum": [ + "expunged" + ] + } + }, + "required": [ + "kind" + ] + } + ] + }, "SledProvisionPolicy": { - "description": "The provision state of a sled.\n\nThis controls whether new resources are going to be provisioned on this sled.", + "description": "The operator-defined provision policy of a sled.\n\nThis controls whether new resources are going to be provisioned on this sled.", "oneOf": [ { "description": "New resources will be provisioned on this sled.", @@ -14982,7 +15035,7 @@ ] }, { - "description": "New resources will not be provisioned on this sled. However, existing resources will continue to be on this sled unless manually migrated off.", + "description": "New resources will not be provisioned on this sled. However, if the sled is currently in service, existing resources will continue to be on this sled unless manually migrated off.", "type": "string", "enum": [ "non_provisionable" @@ -15054,6 +15107,25 @@ "items" ] }, + "SledState": { + "description": "The current state of the sled, as determined by Nexus.", + "oneOf": [ + { + "description": "The sled is currently active, and has resources allocated on it.", + "type": "string", + "enum": [ + "active" + ] + }, + { + "description": "The sled has been permanently removed from service.\n\nThis is a terminal state: once a particular sled ID is decommissioned, it will never return to service. (The actual hardware may be reused, but it will be treated as a brand-new sled.)", + "type": "string", + "enum": [ + "decommissioned" + ] + } + ] + }, "Snapshot": { "description": "View of a Snapshot", "type": "object", @@ -17333,4 +17405,4 @@ } } ] -} +} \ No newline at end of file diff --git a/package-manifest.toml b/package-manifest.toml index d7f42794ee..1a749c5b61 100644 --- a/package-manifest.toml +++ b/package-manifest.toml @@ -463,10 +463,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "41a69a11db6cfa8fc0c8686dc2d725708e0586ce" +source.commit = "4b0e584eec455a43c36af08ae207086965cef833" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//maghemite.sha256.txt -source.sha256 = "19d5eaa744257c32ccdca52af79d718aeb88a0af188345d33a4564a69b259632" +source.sha256 = "f1407cb9aac188d6493d2b0f948c75aad2c36668ddf4ae2a1ed80e9dd395b35d" output.type = "tarball" [package.mg-ddm] @@ -479,10 +479,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "41a69a11db6cfa8fc0c8686dc2d725708e0586ce" +source.commit = "4b0e584eec455a43c36af08ae207086965cef833" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "ffb647b3297ec616d3d9ea93396ad9edd16ed146048a660b34e9b86e85d466b7" +source.sha256 = "fae53cb39536dc92d97cb9610de65b0acbce285e685d7167b719ea6311844fec" output.type = "zone" output.intermediate_only = true @@ -494,10 +494,10 @@ source.repo = "maghemite" # `tools/maghemite_openapi_version`. Failing to do so will cause a failure when # building `ddm-admin-client` (which will instruct you to update # `tools/maghemite_openapi_version`). -source.commit = "41a69a11db6cfa8fc0c8686dc2d725708e0586ce" +source.commit = "4b0e584eec455a43c36af08ae207086965cef833" # The SHA256 digest is automatically posted to: # https://buildomat.eng.oxide.computer/public/file/oxidecomputer/maghemite/image//mg-ddm.sha256.txt -source.sha256 = "26d34f61589f63be64eaa77a6e9e2db4c95d6675798386a1d61721c1ccc59d4d" +source.sha256 = "22996a6f3353296b848be729f14e78a42e7d3d6e62a4a918a5c2358ae011c8eb" output.type = "zone" output.intermediate_only = true diff --git a/schema/crdb/37.0.0/up01.sql b/schema/crdb/37.0.0/up01.sql new file mode 100644 index 0000000000..bb66ff283e --- /dev/null +++ b/schema/crdb/37.0.0/up01.sql @@ -0,0 +1,13 @@ +-- The disposition for a particular sled. This is updated solely by the +-- operator, and not by Nexus. +CREATE TYPE IF NOT EXISTS omicron.public.sled_policy AS ENUM ( + -- The sled is in service, and new resources can be provisioned onto it. + 'in_service', + -- The sled is in service, but the operator has indicated that new + -- resources should not be provisioned onto it. + 'no_provision', + -- The operator has marked that the sled has, or will be, removed from the + -- rack, and it should be assumed that any resources currently on it are + -- now permanently missing. + 'expunged' +); diff --git a/schema/crdb/37.0.0/up02.sql b/schema/crdb/37.0.0/up02.sql new file mode 100644 index 0000000000..636a78f83f --- /dev/null +++ b/schema/crdb/37.0.0/up02.sql @@ -0,0 +1,20 @@ +-- The actual state of the sled. This is updated exclusively by Nexus. +-- +-- Nexus's goal is to match the sled's state with the operator-indicated +-- policy. For example, if the sled_policy is "expunged" and the sled_state is +-- "active", Nexus will start removing zones from the sled, reallocating them +-- elsewhere, etc. Once that is done, Nexus will mark it as decommissioned. +CREATE TYPE IF NOT EXISTS omicron.public.sled_state AS ENUM ( + -- The sled has resources of any kind allocated on it, or, is available for + -- new resources. + -- + -- The sled can be in this state and have a different sled policy, e.g. + -- "expunged". + 'active', + + -- The sled no longer has resources allocated on it, now or in the future. + -- + -- This is a terminal state. This state is only valid if the sled policy is + -- 'expunged'. + 'decommissioned' +); diff --git a/schema/crdb/37.0.0/up03.sql b/schema/crdb/37.0.0/up03.sql new file mode 100644 index 0000000000..7e51cf9546 --- /dev/null +++ b/schema/crdb/37.0.0/up03.sql @@ -0,0 +1,7 @@ +-- Modify the existing sled table to add the columns as required. +ALTER TABLE omicron.public.sled + -- Nullable for now -- we're going to set the data in sled_policy in the + -- next migration statement. + ADD COLUMN IF NOT EXISTS sled_policy omicron.public.sled_policy, + ADD COLUMN IF NOT EXISTS sled_state omicron.public.sled_state + NOT NULL DEFAULT 'active'; diff --git a/schema/crdb/37.0.0/up04.sql b/schema/crdb/37.0.0/up04.sql new file mode 100644 index 0000000000..a85367ec10 --- /dev/null +++ b/schema/crdb/37.0.0/up04.sql @@ -0,0 +1,14 @@ +-- Mass-update the sled_policy column to match the sled_provision_state column. + +-- This is a full table scan, but is unavoidable. +SET + LOCAL disallow_full_table_scans = OFF; + +UPDATE omicron.public.sled + SET sled_policy = + (CASE provision_state + WHEN 'provisionable' THEN 'in_service' + WHEN 'non_provisionable' THEN 'no_provision' + -- No need to specify the ELSE case because the enum has been + -- exhaustively matched (sled_provision_state already bans NULL). + END); diff --git a/schema/crdb/37.0.1/up01.sql b/schema/crdb/37.0.1/up01.sql new file mode 100644 index 0000000000..c23e3e5a11 --- /dev/null +++ b/schema/crdb/37.0.1/up01.sql @@ -0,0 +1,8 @@ +-- This is a follow-up to the previous migration, done separately to ensure +-- that the updated values for sled_policy are committed before the +-- provision_state column is dropped. + +ALTER TABLE omicron.public.sled + DROP COLUMN IF EXISTS provision_state, + ALTER COLUMN sled_policy SET NOT NULL, + ALTER COLUMN sled_state DROP DEFAULT; diff --git a/schema/crdb/37.0.1/up02.sql b/schema/crdb/37.0.1/up02.sql new file mode 100644 index 0000000000..342f794c82 --- /dev/null +++ b/schema/crdb/37.0.1/up02.sql @@ -0,0 +1 @@ +DROP TYPE IF EXISTS omicron.public.sled_provision_state; diff --git a/schema/crdb/dbinit.sql b/schema/crdb/dbinit.sql index 2f8e2fd7f5..1b81ac6da2 100644 --- a/schema/crdb/dbinit.sql +++ b/schema/crdb/dbinit.sql @@ -3549,7 +3549,7 @@ INSERT INTO omicron.public.db_metadata ( version, target_version ) VALUES - ( TRUE, NOW(), NOW(), '36.0.0', NULL) + ( TRUE, NOW(), NOW(), '37.0.1', NULL) ON CONFLICT DO NOTHING; COMMIT; diff --git a/tools/dvt_dock_version b/tools/dvt_dock_version index 047065135b..fd988dc876 100644 --- a/tools/dvt_dock_version +++ b/tools/dvt_dock_version @@ -1 +1 @@ -COMMIT=e384836415e05ae0ba648810ab1c87e9093cdabb +COMMIT=9e8e16f508bb41f02455aeddcece0a7edae2f672 diff --git a/tools/hubris_checksums b/tools/hubris_checksums index d451f7a86c..dca8ea0ab6 100644 --- a/tools/hubris_checksums +++ b/tools/hubris_checksums @@ -1,8 +1,8 @@ -e1b3dc5c7da643b27c0dd5bf8e915d13661446e711bfdeb1d8274eed63fa5843 build-gimlet-c-image-default-v1.0.6.zip -3002444307047429531ef862435a034c64b89a698921bf19794ac97b777a2f95 build-gimlet-d-image-default-v1.0.6.zip -9e783bc92fb1c8a91f4b117241ed4c0ff2818f32f46c5193cdcdbbe02d56af9a build-gimlet-e-image-default-v1.0.6.zip -458c4f02310fe79f27841ce87b2a7c163494f0196890e6420fac17dc4803b51c build-gimlet-f-image-default-v1.0.6.zip -dece7d39f7fcd2f15dc62d91e94046b1f438a3e0fd2c804efd5f67e12ce0dd58 build-psc-b-image-default-v1.0.6.zip -7e94035b52f1dcb137b477750bf9e215d4fcd07fe95b2cfdbbc0d7fada79eb28 build-psc-c-image-default-v1.0.6.zip -ccf09dc7c9c2a946b89bcfafb391100504880fa395c9079dfb7a3b28635a4abb build-sidecar-b-image-default-v1.0.6.zip -b5d91c212f813dbdba06c1f5b098fd37fe6cb93fe33fd3c58325cb6504dc6d05 build-sidecar-c-image-default-v1.0.6.zip +28f432735a15f40101c133e4e9974d1681a33bb7b3386ccbe15b465b613f4826 build-gimlet-c-image-default-v1.0.8.zip +db927999398f0723d5d614db78a5abb4a1d515c711ffba944477bdac10c48907 build-gimlet-d-image-default-v1.0.8.zip +629a53b5d9d4bf3d410687d0ecedf4837a54233ce62b6223d209494777cc7ebc build-gimlet-e-image-default-v1.0.8.zip +e3c2a243257929a65de638f3be425370f084aeeefafbd1773d01ee71cf0b8ea7 build-gimlet-f-image-default-v1.0.8.zip +556595b42d05508ebfdac9dd71b38fe9b72e0cb30f6aa4be626c02141e375a71 build-psc-b-image-default-v1.0.8.zip +39fbf92cbc935b4eaecb81a9768357828cf3e79b5c74d36c9a655ae9023cc50c build-psc-c-image-default-v1.0.8.zip +4225dff721b034fe7cf1dc26277557e1f15f2710014dd46dfa7c92ff04c7e054 build-sidecar-b-image-default-v1.0.8.zip +ae8f12d7b66d0bcc372dd79abf515255f6ca97bb0339c570a058684e04e12cf8 build-sidecar-c-image-default-v1.0.8.zip diff --git a/tools/hubris_version b/tools/hubris_version index f2c1e74f2b..4ee8ac61fe 100644 --- a/tools/hubris_version +++ b/tools/hubris_version @@ -1 +1 @@ -TAGS=(gimlet-v1.0.6 psc-v1.0.6 sidecar-v1.0.6) +TAGS=(gimlet-v1.0.8 psc-v1.0.8 sidecar-v1.0.8) diff --git a/tools/maghemite_ddm_openapi_version b/tools/maghemite_ddm_openapi_version index 6c58d83ea3..d161091fa8 100644 --- a/tools/maghemite_ddm_openapi_version +++ b/tools/maghemite_ddm_openapi_version @@ -1,2 +1,2 @@ -COMMIT="41a69a11db6cfa8fc0c8686dc2d725708e0586ce" +COMMIT="4b0e584eec455a43c36af08ae207086965cef833" SHA2="0b0dbc2f8bbc5d2d9be92d64c4865f8f9335355aae62f7de9f67f81dfb3f1803" diff --git a/tools/maghemite_mg_openapi_version b/tools/maghemite_mg_openapi_version index 896be8d38c..475d273f4a 100644 --- a/tools/maghemite_mg_openapi_version +++ b/tools/maghemite_mg_openapi_version @@ -1,2 +1,2 @@ -COMMIT="41a69a11db6cfa8fc0c8686dc2d725708e0586ce" +COMMIT="4b0e584eec455a43c36af08ae207086965cef833" SHA2="0ac038bbaa54d0ae0ac5ccaeff48f03070618372cca26c9d09b716b909bf9355" diff --git a/tools/maghemite_mgd_checksums b/tools/maghemite_mgd_checksums index 8fc4d083f8..ab84fafc01 100644 --- a/tools/maghemite_mgd_checksums +++ b/tools/maghemite_mgd_checksums @@ -1,2 +1,2 @@ -CIDL_SHA256="26d34f61589f63be64eaa77a6e9e2db4c95d6675798386a1d61721c1ccc59d4d" -MGD_LINUX_SHA256="b2c823dd714fad67546a0e0c0d4ae56f2fe2e7c43434469b38e13b78de9f6968" \ No newline at end of file +CIDL_SHA256="22996a6f3353296b848be729f14e78a42e7d3d6e62a4a918a5c2358ae011c8eb" +MGD_LINUX_SHA256="943b0a52d279bde55a419e2cdb24873acc32703bc97bd599376117ee0edc1511" \ No newline at end of file