[nexus] Expunge disk internal API, omdb commands

dev-tools/omdb/src/bin/omdb/db.rs

@@ -62,6 +62,7 @@ use nexus_db_model::IpAttachState;
 use nexus_db_model::IpKind;
 use nexus_db_model::NetworkInterface;
 use nexus_db_model::NetworkInterfaceKind;
+use nexus_db_model::PhysicalDisk;
 use nexus_db_model::Probe;
 use nexus_db_model::Project;
 use nexus_db_model::Region;
@@ -96,7 +97,10 @@ use nexus_types::deployment::Blueprint;
 use nexus_types::deployment::BlueprintZoneDisposition;
 use nexus_types::deployment::BlueprintZoneFilter;
 use nexus_types::deployment::BlueprintZoneType;
+use nexus_types::deployment::DiskFilter;
 use nexus_types::deployment::SledFilter;
+use nexus_types::external_api::views::PhysicalDiskPolicy;
+use nexus_types::external_api::views::PhysicalDiskState;
 use nexus_types::external_api::views::SledPolicy;
 use nexus_types::external_api::views::SledState;
 use nexus_types::identity::Resource;
@@ -281,12 +285,14 @@ pub struct DbFetchOptions {
 enum DbCommands {
     /// Print information about the rack
     Rack(RackArgs),
-    /// Print information about disks
+    /// Print information about virtual disks
     Disks(DiskArgs),
     /// Print information about internal and external DNS
     Dns(DnsArgs),
     /// Print information about collected hardware/software inventory
     Inventory(InventoryArgs),
+    /// Print information about physical disks
+    PhysicalDisks(PhysicalDisksArgs),
     /// Save the current Reconfigurator inputs to a file
     ReconfiguratorSave(ReconfiguratorSaveArgs),
     /// Print information about regions
@@ -407,8 +413,8 @@ enum InventoryCommands {
     Cabooses,
     /// list and show details from particular collections
     Collections(CollectionsArgs),
-    /// show all physical disks every found
-    PhysicalDisks(PhysicalDisksArgs),
+    /// show all physical disks ever found
+    PhysicalDisks(InvPhysicalDisksArgs),
     /// list all root of trust pages ever found
     RotPages,
 }
@@ -437,14 +443,21 @@ struct CollectionsShowArgs {
 }
 
 #[derive(Debug, Args, Clone, Copy)]
-struct PhysicalDisksArgs {
+struct InvPhysicalDisksArgs {
     #[clap(long)]
     collection_id: Option<CollectionUuid>,
 
     #[clap(long, requires("collection_id"))]
     sled_id: Option<SledUuid>,
 }
 
+#[derive(Debug, Args)]
+struct PhysicalDisksArgs {
+    /// Show disks that match the given filter
+    #[clap(short = 'F', long, value_enum)]
+    filter: Option<DiskFilter>,
+}
+
 #[derive(Debug, Args)]
 struct ReconfiguratorSaveArgs {
     /// where to save the output
@@ -611,6 +624,15 @@ impl DbArgs {
                 )
                 .await
             }
+            DbCommands::PhysicalDisks(args) => {
+                cmd_db_physical_disks(
+                    &opctx,
+                    &datastore,
+                    &self.fetch_opts,
+                    args,
+                )
+                .await
+            }
             DbCommands::ReconfiguratorSave(reconfig_save_args) => {
                 cmd_db_reconfigurator_save(
                     &opctx,
@@ -1385,6 +1407,68 @@ async fn cmd_db_disk_physical(
     Ok(())
 }
 
+#[derive(Tabled)]
+#[tabled(rename_all = "SCREAMING_SNAKE_CASE")]
+struct PhysicalDiskRow {
+    id: Uuid,
+    serial: String,
+    vendor: String,
+    model: String,
+    sled_id: Uuid,
+    policy: PhysicalDiskPolicy,
+    state: PhysicalDiskState,
+}
+
+impl From<PhysicalDisk> for PhysicalDiskRow {
+    fn from(d: PhysicalDisk) -> Self {
+        PhysicalDiskRow {
+            id: d.id(),
+            serial: d.serial.clone(),
+            vendor: d.vendor.clone(),
+            model: d.model.clone(),
+            sled_id: d.sled_id,
+            policy: d.disk_policy.into(),
+            state: d.disk_state.into(),
+        }
+    }
+}
+
+/// Run `omdb db physical-disks`.
+async fn cmd_db_physical_disks(
+    opctx: &OpContext,
+    datastore: &DataStore,
+    fetch_opts: &DbFetchOptions,
+    args: &PhysicalDisksArgs,
+) -> Result<(), anyhow::Error> {
+    let limit = fetch_opts.fetch_limit;
+    let filter = match args.filter {
+        Some(filter) => filter,
+        None => {
+            eprintln!(
+                "note: listing all in-service disks \
+                 (use -F to filter, e.g. -F in-service)"
+            );
+            DiskFilter::InService
+        }
+    };
+
+    let sleds = datastore
+        .physical_disk_list(&opctx, &first_page(limit), filter)
+        .await
+        .context("listing physical disks")?;
+    check_limit(&sleds, limit, || String::from("listing physical disks"));
+
+    let rows = sleds.into_iter().map(|s| PhysicalDiskRow::from(s));
+    let table = tabled::Table::new(rows)
+        .with(tabled::settings::Style::empty())
+        .with(tabled::settings::Padding::new(1, 1, 0, 0))
+        .to_string();
+
+    println!("{}", table);
+
+    Ok(())
+}
+
 // SERVICES
 
 // Snapshots
@@ -3187,7 +3271,7 @@ async fn cmd_db_inventory_cabooses(
 async fn cmd_db_inventory_physical_disks(
     conn: &DataStoreConnection<'_>,
     limit: NonZeroU32,
-    args: PhysicalDisksArgs,
+    args: InvPhysicalDisksArgs,
 ) -> Result<(), anyhow::Error> {
     #[derive(Tabled)]
     #[tabled(rename_all = "SCREAMING_SNAKE_CASE")]

dev-tools/omdb/src/bin/omdb/nexus.rs

@@ -24,6 +24,7 @@ use nexus_client::types::BackgroundTask;
 use nexus_client::types::BackgroundTasksActivateRequest;
 use nexus_client::types::CurrentStatus;
 use nexus_client::types::LastResult;
+use nexus_client::types::PhysicalDiskPath;
 use nexus_client::types::SledSelector;
 use nexus_client::types::UninitializedSledId;
 use nexus_db_queries::db::lookup::LookupPath;
@@ -33,6 +34,7 @@ use nexus_types::internal_api::background::RegionReplacementDriverStatus;
 use nexus_types::inventory::BaseboardId;
 use omicron_uuid_kinds::CollectionUuid;
 use omicron_uuid_kinds::GenericUuid;
+use omicron_uuid_kinds::PhysicalDiskUuid;
 use omicron_uuid_kinds::SledUuid;
 use reedline::DefaultPrompt;
 use reedline::DefaultPromptSegment;
@@ -256,6 +258,8 @@ enum SledsCommands {
     Add(SledAddArgs),
     /// Expunge a sled (DANGEROUS)
     Expunge(SledExpungeArgs),
+    /// Expunge a disk (DANGEROUS)
+    ExpungeDisk(DiskExpungeArgs),
 }
 
 #[derive(Debug, Args)]
@@ -277,6 +281,17 @@ struct SledExpungeArgs {
     sled_id: SledUuid,
 }
 
+#[derive(Debug, Args)]
+struct DiskExpungeArgs {
+    // expunge is _extremely_ dangerous, so we also require a database
+    // connection to perform some safety checks
+    #[clap(flatten)]
+    db_url_opts: DbUrlOptions,
+
+    /// Physical disk ID
+    physical_disk_id: PhysicalDiskUuid,
+}
+
 impl NexusArgs {
     /// Run a `omdb nexus` subcommand.
     pub(crate) async fn run_cmd(
@@ -401,6 +416,13 @@ impl NexusArgs {
                 let token = omdb.check_allow_destructive()?;
                 cmd_nexus_sled_expunge(&client, args, omdb, log, token).await
             }
+            NexusCommands::Sleds(SledsArgs {
+                command: SledsCommands::ExpungeDisk(args),
+            }) => {
+                let token = omdb.check_allow_destructive()?;
+                cmd_nexus_sled_expunge_disk(&client, args, omdb, log, token)
+                    .await
+            }
         }
     }
 }
@@ -1569,3 +1591,140 @@ async fn cmd_nexus_sled_expunge(
     );
     Ok(())
 }
+
+/// Runs `omdb nexus sleds expunge-disk`
+async fn cmd_nexus_sled_expunge_disk(
+    client: &nexus_client::Client,
+    args: &DiskExpungeArgs,
+    omdb: &Omdb,
+    log: &slog::Logger,
+    _destruction_token: DestructiveOperationToken,
+) -> Result<(), anyhow::Error> {
+    use nexus_db_queries::context::OpContext;
+
+    let datastore = args.db_url_opts.connect(omdb, log).await?;
+    let opctx = OpContext::for_tests(log.clone(), datastore.clone());
+    let opctx = &opctx;
+
+    // First, we need to look up the disk so we can lookup identity information.
+    let (_authz_physical_disk, physical_disk) =
+        LookupPath::new(opctx, &datastore)
+            .physical_disk(args.physical_disk_id.into_untyped_uuid())
+            .fetch()
+            .await
+            .with_context(|| {
+                format!(
+                    "failed to find physical disk {}",
+                    args.physical_disk_id
+                )
+            })?;
+
+    // Helper to get confirmation messages from the user.
+    let mut line_editor = Reedline::create();
+    let mut read_with_prompt = move |message: &str| {
+        let prompt = DefaultPrompt::new(
+            DefaultPromptSegment::Basic(message.to_string()),
+            DefaultPromptSegment::Empty,
+        );
+        if let Ok(reedline::Signal::Success(input)) =
+            line_editor.read_line(&prompt)
+        {
+            Ok(input)
+        } else {
+            bail!("expungement aborted")
+        }
+    };
+
+    // Now check whether its sled-agent was found in the most recent
+    // inventory collection.
+    match datastore
+        .inventory_get_latest_collection(opctx)
+        .await
+        .context("loading latest collection")?
+    {
+        Some(collection) => {
+            let disk_identity = omicron_common::disk::DiskIdentity {
+                vendor: physical_disk.vendor.clone(),
+                serial: physical_disk.serial.clone(),
+                model: physical_disk.model.clone(),
+            };
+
+            let mut sleds_containing_disk = vec![];
+
+            for (sled_id, sled_agent) in collection.sled_agents {
+                for sled_disk in sled_agent.disks {
+                    if sled_disk.identity == disk_identity {
+                        sleds_containing_disk.push(sled_id);
+                    }
+                }
+            }
+
+            match sleds_containing_disk.len() {
+                0 => {}
+                1 => {
+                    eprintln!(
+                        "WARNING: physical disk {} is PRESENT in the most \
+                         recent inventory collection (spotted at {}). It is \
+                         dangerous to expunge a disk that is still running, and \
+                         safer to expunge a disk from a system where it has been \
+                         removed. Are you sure you want to proceed anyway?",
+                        args.physical_disk_id, collection.time_done,
+                    );
+                    let confirm = read_with_prompt("y/N")?;
+                    if confirm != "y" {
+                        eprintln!("expungement not confirmed: aborting");
+                        return Ok(());
+                    }
+                }
+                _ => {
+                    // This should be impossible due to a unique database index,
+                    // "vendor_serial_model_unique".
+                    //
+                    // Even if someone tried moving a disk, it would need to be
+                    // decommissioned before being re-commissioned elsewhere.
+                    //
+                    // However, we still print out an error message here in the
+                    // (unlikely) even that it happens anyway.
+                    eprintln!(
+                        "ERROR: physical disk {} is PRESENT MULTIPLE TIMES in \
+                        the most recent inventory collection (spotted at {}).
+                        This should not be possible, and is an indication of a \
+                        database issue.",
+                        args.physical_disk_id, collection.time_done,
+                    );
+                    bail!("Physical Disk appeared on multiple sleds");
+                }
+            }
+        }
+        None => {
+            eprintln!(
+                "ERROR: cannot verify that the physical disk inventory status \
+                 because there are no inventory collections present. Please \
+                 make sure that the physical disk has been physically removed, \
+                 or ensure that inventory may be collected."
+            );
+            bail!("No inventory");
+        }
+    }
+
+    eprintln!(
+        "WARNING: This operation will PERMANENTLY and IRRECOVABLY mark physical disk \
+        {} ({}) expunged. To proceed, type the physical disk's serial number.",
+        args.physical_disk_id,
+        physical_disk.serial,
+    );
+    let confirm = read_with_prompt("disk serial number")?;
+    if confirm != physical_disk.serial {
+        eprintln!("disk serial number not confirmed: aborting");
+        return Ok(());
+    }
+
+    client
+        .physical_disk_expunge(&PhysicalDiskPath {
+            disk_id: args.physical_disk_id.into_untyped_uuid(),
+        })
+        .await
+        .context("expunging disk")?;
+    eprintln!("expunged disk {}", args.physical_disk_id);
+    Ok(())
+}