quiesce needs to keep track of blueprint ids

common/src/address.rs

@@ -396,7 +396,9 @@ impl std::fmt::Display for IpVersion {
 ///
 /// The first address in the range is guaranteed to be no greater than the last
 /// address.
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize)]
+#[derive(
+    Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize, Ord, PartialOrd,
+)]
 #[serde(untagged)]
 pub enum IpRange {
     V4(Ipv4Range),
@@ -548,7 +550,16 @@ impl From<Ipv6Range> for IpRange {
 ///
 /// The first address must be less than or equal to the last address.
 #[derive(
-    Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize, JsonSchema,
+    Clone,
+    Copy,
+    Debug,
+    PartialEq,
+    Eq,
+    Deserialize,
+    Serialize,
+    JsonSchema,
+    PartialOrd,
+    Ord,
 )]
 #[serde(try_from = "AnyIpv4Range")]
 pub struct Ipv4Range {
@@ -612,7 +623,16 @@ impl TryFrom<AnyIpv4Range> for Ipv4Range {
 ///
 /// The first address must be less than or equal to the last address.
 #[derive(
-    Clone, Copy, Debug, PartialEq, Eq, Deserialize, Serialize, JsonSchema,
+    PartialOrd,
+    Ord,
+    Clone,
+    Copy,
+    Debug,
+    PartialEq,
+    Eq,
+    Deserialize,
+    Serialize,
+    JsonSchema,
 )]
 #[serde(try_from = "AnyIpv6Range")]
 pub struct Ipv6Range {

dev-tools/omdb/src/bin/omdb/db.rs

@@ -5016,7 +5016,7 @@ async fn cmd_db_dns_diff(
         // Load the added and removed items.
         use nexus_db_schema::schema::dns_name::dsl;
 
-        let added = dsl::dns_name
+        let mut added = dsl::dns_name
             .filter(dsl::dns_zone_id.eq(zone.id))
             .filter(dsl::version_added.eq(version.version))
             .limit(i64::from(u32::from(limit)))
@@ -5026,7 +5026,7 @@ async fn cmd_db_dns_diff(
             .context("loading added names")?;
         check_limit(&added, limit, || "loading added names");
 
-        let removed = dsl::dns_name
+        let mut removed = dsl::dns_name
             .filter(dsl::dns_zone_id.eq(zone.id))
             .filter(dsl::version_removed.eq(version.version))
             .limit(i64::from(u32::from(limit)))
@@ -5042,6 +5042,11 @@ async fn cmd_db_dns_diff(
         );
         println!("");
 
+        // This is kind of stupid-expensive, but there aren't a lot of records
+        // here and it's helpful for this output to be stable.
+        added.sort_by_cached_key(|k| format!("{} {:?}", k.name, k.records()));
+        removed.sort_by_cached_key(|k| format!("{} {:?}", k.name, k.records()));
+
         for a in added {
             print_name("+", &a.name, a.records().context("parsing records"));
         }
@@ -5097,7 +5102,8 @@ async fn cmd_db_dns_names(
             }
         });
 
-        for (name, records) in names {
+        for (name, mut records) in names {
+            records.sort();
             print_name("", &name, Ok(records));
         }
     }

dev-tools/omdb/src/bin/omdb/nexus/quiesce.rs

@@ -11,7 +11,10 @@ use chrono::TimeDelta;
 use chrono::Utc;
 use clap::Args;
 use clap::Subcommand;
+use nexus_client::types::PendingRecovery;
 use nexus_client::types::QuiesceState;
+use nexus_client::types::QuiesceStatus;
+use nexus_client::types::SagaQuiesceStatus;
 use std::time::Duration;
 
 #[derive(Debug, Args)]
@@ -31,9 +34,9 @@ pub enum QuiesceCommands {
 
 #[derive(Debug, Args)]
 pub struct QuiesceShowArgs {
-    /// Show details about held database connections
+    /// Show stack traces for held database connections
     #[clap(short, long, default_value_t = false)]
-    verbose: bool,
+    stacks: bool,
 }
 
 pub async fn cmd_nexus_quiesce(
@@ -60,21 +63,27 @@ async fn quiesce_show(
         .await
         .context("fetching quiesce state")?
         .into_inner();
-    match quiesce.state {
+
+    let QuiesceStatus { db_claims, sagas, state } = quiesce;
+
+    match state {
+        QuiesceState::Undetermined => {
+            println!("has not yet determined if it is quiescing");
+        }
         QuiesceState::Running => {
             println!("running normally (not quiesced, not quiescing)");
         }
-        QuiesceState::WaitingForSagas { time_requested } => {
+        QuiesceState::DrainingSagas { time_requested } => {
             println!(
                 "quiescing since {} ({} ago)",
                 humantime::format_rfc3339_millis(time_requested.into()),
                 format_time_delta(now - time_requested),
             );
             println!("details: waiting for running sagas to finish");
         }
-        QuiesceState::WaitingForDb {
+        QuiesceState::DrainingDb {
             time_requested,
-            duration_waiting_for_sagas,
+            duration_draining_sagas,
             ..
         } => {
             println!(
@@ -87,13 +96,34 @@ async fn quiesce_show(
             );
             println!(
                 "    previously: waiting for sagas took {}",
-                format_duration_ms(duration_waiting_for_sagas.into()),
+                format_duration_ms(duration_draining_sagas.into()),
+            );
+        }
+        QuiesceState::RecordingQuiesce {
+            time_requested,
+            duration_draining_sagas,
+            duration_draining_db,
+            ..
+        } => {
+            println!(
+                "quiescing since {} ({} ago)",
+                humantime::format_rfc3339_millis(time_requested.into()),
+                format_time_delta(now - time_requested),
+            );
+            println!(
+                "    waiting for sagas took {}",
+                format_duration_ms(duration_draining_sagas.into()),
+            );
+            println!(
+                "    waiting for db quiesce took {}",
+                format_duration_ms(duration_draining_db.into()),
             );
         }
         QuiesceState::Quiesced {
             time_quiesced,
-            duration_waiting_for_sagas,
-            duration_waiting_for_db,
+            duration_draining_sagas,
+            duration_draining_db,
+            duration_recording_quiesce,
             duration_total,
             ..
         } => {
@@ -104,11 +134,15 @@ async fn quiesce_show(
             );
             println!(
                 "    waiting for sagas took {}",
-                format_duration_ms(duration_waiting_for_sagas.into()),
+                format_duration_ms(duration_draining_sagas.into()),
             );
             println!(
                 "    waiting for db quiesce took {}",
-                format_duration_ms(duration_waiting_for_db.into()),
+                format_duration_ms(duration_draining_db.into()),
+            );
+            println!(
+                "    recording quiesce took {}",
+                format_duration_ms(duration_recording_quiesce.into()),
             );
             println!(
                 "    total quiesce time: {}",
@@ -117,25 +151,83 @@ async fn quiesce_show(
         }
     }
 
-    println!("sagas running: {}", quiesce.sagas_pending.len());
-    for saga in &quiesce.sagas_pending {
+    let SagaQuiesceStatus {
+        sagas_pending,
+        drained_blueprint_id,
+        first_recovery_complete,
+        new_sagas_allowed,
+        reassignment_blueprint_id,
+        reassignment_generation,
+        reassignment_pending,
+        recovered_blueprint_id,
+        recovered_reassignment_generation,
+        recovery_pending,
+    } = sagas;
+
+    println!("saga quiesce:");
+    println!("    new sagas: {:?}", new_sagas_allowed);
+    println!(
+        "    drained as of blueprint: {}",
+        drained_blueprint_id
+            .map(|s| s.to_string())
+            .as_deref()
+            .unwrap_or("none")
+    );
+    println!(
+        "    blueprint for last completed recovery pass: {}",
+        recovered_blueprint_id
+            .map(|s| s.to_string())
+            .as_deref()
+            .unwrap_or("none")
+    );
+    println!(
+        "    blueprint for last reassignment pass: {}",
+        reassignment_blueprint_id
+            .map(|s| s.to_string())
+            .as_deref()
+            .unwrap_or("none")
+    );
+    println!(
+        "    reassignment generation: {} (pass running: {})",
+        reassignment_generation,
+        if reassignment_pending { "yes" } else { "no" }
+    );
+    println!("    recovered generation: {}", recovered_reassignment_generation);
+    println!(
+        "    recovered at least once successfully: {}",
+        if first_recovery_complete { "yes" } else { "no" },
+    );
+    print!("    recovery pending: ");
+    if let Some(PendingRecovery { generation, blueprint_id }) = recovery_pending
+    {
+        println!(
+            "yes (generation {}, blueprint id {})",
+            generation,
+            blueprint_id.map(|s| s.to_string()).as_deref().unwrap_or("none")
+        );
+    } else {
+        println!("no");
+    }
+
+    println!("    sagas running: {}", sagas_pending.len());
+    for saga in &sagas_pending {
         println!(
-            "    saga {} pending since {} ({})",
+            "        saga {} pending since {} ({})",
             saga.saga_id,
             humantime::format_rfc3339_millis(saga.time_pending.into()),
             saga.saga_name
         );
     }
 
-    println!("database connections held: {}", quiesce.db_claims.len());
-    for claim in &quiesce.db_claims {
+    println!("database connections held: {}", db_claims.len());
+    for claim in &db_claims {
         println!(
             "    claim {} held since {} ({} ago)",
             claim.id,
             claim.held_since,
             format_time_delta(Utc::now() - claim.held_since),
         );
-        if args.verbose {
+        if args.stacks {
             println!("    acquired by:");
             println!("{}", textwrap::indent(&claim.debug, "        "));
         }
@@ -149,7 +241,7 @@ async fn quiesce_start(
     _token: DestructiveOperationToken,
 ) -> Result<(), anyhow::Error> {
     client.quiesce_start().await.context("quiescing Nexus")?;
-    quiesce_show(client, &QuiesceShowArgs { verbose: false }).await
+    quiesce_show(client, &QuiesceShowArgs { stacks: false }).await
 }
 
 fn format_duration_ms(duration: Duration) -> String {