Merge remote-tracking branch 'origin/main' into john/blueprint-add-da…

…tasets

.github/workflows/hakari.yml

@@ -24,7 +24,7 @@ jobs:
         with:
           toolchain: stable
       - name: Install cargo-hakari
-        uses: taiki-e/install-action@19e9b549a48620cc50fcf6e6e866b8fb4eca1b01 # v2
+        uses: taiki-e/install-action@4ce8785db2a8a56c9ede16f705c2c49c5c61669c # v2
         with:
           tool: cargo-hakari
       - name: Check workspace-hack Cargo.toml is up-to-date

illumos-utils/src/ipadm.rs

@@ -107,4 +107,29 @@ impl Ipadm {
         };
         Ok(())
     }
+
+    // Create gateway on the IP interface if it doesn't already exist
+    pub fn create_opte_gateway(
+        opte_iface: &String,
+    ) -> Result<(), ExecutionError> {
+        let addrobj = format!("{}/public", opte_iface);
+        let mut cmd = std::process::Command::new(PFEXEC);
+        let cmd = cmd.args(&[IPADM, "show-addr", &addrobj]);
+        match execute(cmd) {
+            Err(_) => {
+                let mut cmd = std::process::Command::new(PFEXEC);
+                let cmd = cmd.args(&[
+                    IPADM,
+                    "create-addr",
+                    "-t",
+                    "-T",
+                    "dhcp",
+                    &addrobj,
+                ]);
+                execute(cmd)?;
+            }
+            Ok(_) => (),
+        };
+        Ok(())
+    }
 }

illumos-utils/src/opte/port.rs

@@ -15,7 +15,7 @@ struct PortInner {
     // Name of the port as identified by OPTE
     name: String,
     // IP address within the VPC Subnet
-    _ip: IpAddr,
+    ip: IpAddr,
     // VPC-private MAC address
     mac: MacAddr6,
     // Emulated PCI slot for the guest NIC, passed to Propolis
@@ -95,7 +95,7 @@ impl Port {
         Self {
             inner: Arc::new(PortInner {
                 name,
-                _ip: ip,
+                ip,
                 mac,
                 slot,
                 vni,
@@ -105,6 +105,10 @@ impl Port {
         }
     }
 
+    pub fn ip(&self) -> &IpAddr {
+        &self.inner.ip
+    }
+
     pub fn name(&self) -> &str {
         &self.inner.name
     }

illumos-utils/src/route.rs

@@ -7,27 +7,76 @@
 use crate::zone::ROUTE;
 use crate::{execute, inner, output_to_exec_error, ExecutionError, PFEXEC};
 use libc::ESRCH;
-use std::net::Ipv6Addr;
+use std::net::{IpAddr, Ipv4Addr, Ipv6Addr};
 
 /// Wraps commands for interacting with routing tables.
 pub struct Route {}
 
+pub enum Gateway {
+    Ipv4(Ipv4Addr),
+    Ipv6(Ipv6Addr),
+}
+
 #[cfg_attr(any(test, feature = "testing"), mockall::automock)]
 impl Route {
     pub fn ensure_default_route_with_gateway(
-        gateway: &Ipv6Addr,
+        gateway: Gateway,
     ) -> Result<(), ExecutionError> {
+        let inet;
+        let gw;
+        match gateway {
+            Gateway::Ipv4(addr) => {
+                inet = "-inet";
+                gw = addr.to_string();
+            }
+            Gateway::Ipv6(addr) => {
+                inet = "-inet6";
+                gw = addr.to_string();
+            }
+        }
         // Add the desired route if it doesn't already exist
         let destination = "default";
         let mut cmd = std::process::Command::new(PFEXEC);
+        let cmd = cmd.args(&[ROUTE, "-n", "get", inet, destination, inet, &gw]);
+
+        let out =
+            cmd.output().map_err(|err| ExecutionError::ExecutionStart {
+                command: inner::to_string(cmd),
+                err,
+            })?;
+        match out.status.code() {
+            Some(0) => (),
+            // If the entry is not found in the table,
+            // the exit status of the command will be 3 (ESRCH).
+            // When that is the case, we'll add the route.
+            Some(ESRCH) => {
+                let mut cmd = std::process::Command::new(PFEXEC);
+                let cmd =
+                    cmd.args(&[ROUTE, "add", inet, destination, inet, &gw]);
+                execute(cmd)?;
+            }
+            Some(_) | None => return Err(output_to_exec_error(cmd, &out)),
+        };
+        Ok(())
+    }
+
+    pub fn ensure_opte_route(
+        gateway: &Ipv4Addr,
+        iface: &String,
+        opte_ip: &IpAddr,
+    ) -> Result<(), ExecutionError> {
+        // Add the desired route if it doesn't already exist
+        let mut cmd = std::process::Command::new(PFEXEC);
         let cmd = cmd.args(&[
             ROUTE,
             "-n",
             "get",
-            "-inet6",
-            destination,
-            "-inet6",
+            "-host",
             &gateway.to_string(),
+            &opte_ip.to_string(),
+            "-interface",
+            "-ifp",
+            &iface.to_string(),
         ]);
 
         let out =
@@ -45,10 +94,12 @@ impl Route {
                 let cmd = cmd.args(&[
                     ROUTE,
                     "add",
-                    "-inet6",
-                    destination,
-                    "-inet6",
+                    "-host",
                     &gateway.to_string(),
+                    &opte_ip.to_string(),
+                    "-interface",
+                    "-ifp",
+                    &iface.to_string(),
                 ]);
                 execute(cmd)?;
             }

illumos-utils/src/running_zone.rs

@@ -888,7 +888,7 @@ impl RunningZone {
 
     /// Return references to the OPTE ports for this zone.
     pub fn opte_ports(&self) -> impl Iterator<Item = &Port> {
-        self.inner.opte_ports.iter().map(|(port, _)| port)
+        self.inner.opte_ports()
     }
 
     /// Remove the OPTE ports on this zone from the port manager.
@@ -1130,6 +1130,11 @@ impl InstalledZone {
         path.push("root/var/svc/profile/site.xml");
         path
     }
+
+    /// Returns references to the OPTE ports for this zone.
+    pub fn opte_ports(&self) -> impl Iterator<Item = &Port> {
+        self.opte_ports.iter().map(|(port, _)| port)
+    }
 }
 
 #[derive(Clone)]

nexus/blueprint-execution/src/dns.rs

@@ -328,7 +328,8 @@ mod test {
     use nexus_types::deployment::Policy;
     use nexus_types::deployment::SledResources;
     use nexus_types::deployment::ZpoolName;
-    use nexus_types::external_api::views::SledProvisionState;
+    use nexus_types::external_api::views::SledPolicy;
+    use nexus_types::external_api::views::SledState;
     use nexus_types::internal_api::params::DnsConfigParams;
     use nexus_types::internal_api::params::DnsConfigZone;
     use nexus_types::internal_api::params::DnsRecord;
@@ -409,7 +410,8 @@ mod test {
             .zip(possible_sled_subnets)
             .map(|(sled_id, subnet)| {
                 let sled_resources = SledResources {
-                    provision_state: SledProvisionState::Provisionable,
+                    policy: SledPolicy::provisionable(),
+                    state: SledState::Active,
                     zpools: BTreeSet::from([ZpoolName::from_str(&format!(
                         "oxp_{}",
                         Uuid::new_v4()

nexus/db-model/src/lib.rs

@@ -73,9 +73,10 @@ mod silo_user;
 mod silo_user_password_hash;
 mod sled;
 mod sled_instance;
-mod sled_provision_state;
+mod sled_policy;
 mod sled_resource;
 mod sled_resource_kind;
+mod sled_state;
 mod sled_underlay_subnet_allocation;
 mod snapshot;
 mod ssh_key;
@@ -161,9 +162,10 @@ pub use silo_user::*;
 pub use silo_user_password_hash::*;
 pub use sled::*;
 pub use sled_instance::*;
-pub use sled_provision_state::*;
+pub use sled_policy::to_db_sled_policy; // Do not expose DbSledPolicy
 pub use sled_resource::*;
 pub use sled_resource_kind::*;
+pub use sled_state::*;
 pub use sled_underlay_subnet_allocation::*;
 pub use snapshot::*;
 pub use ssh_key::*;

nexus/db-model/src/schema.rs

@@ -13,7 +13,7 @@ use omicron_common::api::external::SemverVersion;
 ///
 /// This should be updated whenever the schema is changed. For more details,
 /// refer to: schema/crdb/README.adoc
-pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(36, 0, 0);
+pub const SCHEMA_VERSION: SemverVersion = SemverVersion::new(37, 0, 1);
 
 table! {
     disk (id) {
@@ -824,7 +824,8 @@ table! {
         ip -> Inet,
         port -> Int4,
         last_used_address -> Inet,
-        provision_state -> crate::SledProvisionStateEnum,
+        sled_policy -> crate::sled_policy::SledPolicyEnum,
+        sled_state -> crate::SledStateEnum,
     }
 }
 

nexus/db-model/src/sled.rs

@@ -2,10 +2,11 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use super::{ByteCount, Generation, SqlU16, SqlU32};
+use super::{ByteCount, Generation, SledState, SqlU16, SqlU32};
 use crate::collection::DatastoreCollectionConfig;
+use crate::ipv6;
 use crate::schema::{physical_disk, service, sled, zpool};
-use crate::{ipv6, SledProvisionState};
+use crate::sled_policy::DbSledPolicy;
 use chrono::{DateTime, Utc};
 use db_macros::Asset;
 use nexus_types::{external_api::shared, external_api::views, identity::Asset};
@@ -60,7 +61,11 @@ pub struct Sled {
     /// The last IP address provided to a propolis instance on this sled
     pub last_used_address: ipv6::Ipv6Addr,
 
-    provision_state: SledProvisionState,
+    #[diesel(column_name = sled_policy)]
+    policy: DbSledPolicy,
+
+    #[diesel(column_name = sled_state)]
+    state: SledState,
 }
 
 impl Sled {
@@ -84,8 +89,15 @@ impl Sled {
         &self.serial_number
     }
 
-    pub fn provision_state(&self) -> SledProvisionState {
-        self.provision_state
+    /// The policy here is the `views::SledPolicy` because we expect external
+    /// users to always use that.
+    pub fn policy(&self) -> views::SledPolicy {
+        self.policy.into()
+    }
+
+    /// Returns the sled's state.
+    pub fn state(&self) -> SledState {
+        self.state
     }
 }
 
@@ -99,7 +111,8 @@ impl From<Sled> for views::Sled {
                 part: sled.part_number,
                 revision: sled.revision,
             },
-            provision_state: sled.provision_state.into(),
+            policy: sled.policy.into(),
+            state: sled.state.into(),
             usable_hardware_threads: sled.usable_hardware_threads.0,
             usable_physical_ram: *sled.usable_physical_ram,
         }
@@ -197,8 +210,10 @@ impl SledUpdate {
             serial_number: self.serial_number,
             part_number: self.part_number,
             revision: self.revision,
-            // By default, sleds start as provisionable.
-            provision_state: SledProvisionState::Provisionable,
+            // By default, sleds start in-service.
+            policy: DbSledPolicy::InService,
+            // Currently, new sleds start in the "active" state.
+            state: SledState::Active,
             usable_hardware_threads: self.usable_hardware_threads,
             usable_physical_ram: self.usable_physical_ram,
             reservoir_size: self.reservoir_size,

nexus/db-model/src/sled_policy.rs

@@ -0,0 +1,68 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Database representation of a sled's operator-defined policy.
+//!
+//! This is related to, but different from `SledState`: a sled's **policy** is
+//! its disposition as specified by the operator, while its **state** refers to
+//! what's currently on it, as determined by Nexus.
+//!
+//! For example, a sled might be in the `Active` state, but have a policy of
+//! `Expunged` -- this would mean that Nexus knows about resources currently
+//! provisioned on the sled, but the operator has said that it should be marked
+//! as gone.
+
+use super::impl_enum_type;
+use nexus_types::external_api::views::{SledPolicy, SledProvisionPolicy};
+use serde::{Deserialize, Serialize};
+
+impl_enum_type!(
+    #[derive(Clone, SqlType, Debug, QueryId)]
+    #[diesel(postgres_type(name = "sled_policy", schema = "public"))]
+    pub struct SledPolicyEnum;
+
+    /// This type is not actually public, because [`SledPolicy`] has a somewhat
+    /// different, friendlier shape while being equivalent -- external code
+    /// should always use [`SledPolicy`].
+    ///
+    /// However, it must be marked `pub` to avoid errors like `crate-private
+    /// type `DbSledPolicy` in public interface`. Marking this type `pub`,
+    /// without actually making it public, tricks rustc in a desirable way.
+    #[derive(Clone, Copy, Debug, AsExpression, FromSqlRow, Serialize, Deserialize, PartialEq)]
+    #[diesel(sql_type = SledPolicyEnum)]
+    pub enum DbSledPolicy;
+
+    // Enum values
+    InService => b"in_service"
+    NoProvision => b"no_provision"
+    Expunged => b"expunged"
+);
+
+/// Converts a [`SledPolicy`] to a version that can be inserted into a
+/// database.
+pub fn to_db_sled_policy(policy: SledPolicy) -> DbSledPolicy {
+    match policy {
+        SledPolicy::InService {
+            provision_policy: SledProvisionPolicy::Provisionable,
+        } => DbSledPolicy::InService,
+        SledPolicy::InService {
+            provision_policy: SledProvisionPolicy::NonProvisionable,
+        } => DbSledPolicy::NoProvision,
+        SledPolicy::Expunged => DbSledPolicy::Expunged,
+    }
+}
+
+impl From<DbSledPolicy> for SledPolicy {
+    fn from(policy: DbSledPolicy) -> Self {
+        match policy {
+            DbSledPolicy::InService => SledPolicy::InService {
+                provision_policy: SledProvisionPolicy::Provisionable,
+            },
+            DbSledPolicy::NoProvision => SledPolicy::InService {
+                provision_policy: SledProvisionPolicy::NonProvisionable,
+            },
+            DbSledPolicy::Expunged => SledPolicy::Expunged,
+        }
+    }
+}