[40/n] blueprint planner logic + sled agent code to honor mupdate overrides

dev-tools/reconfigurator-cli/src/lib.rs

@@ -375,6 +375,8 @@ enum SledSetCommand {
     Policy(SledSetPolicyArgs),
     #[clap(flatten)]
     Visibility(SledSetVisibilityCommand),
+    /// set the mupdate override for this sled
+    MupdateOverride(SledSetMupdateOverrideArgs),
 }
 
 #[derive(Debug, Args)]
@@ -496,6 +498,23 @@ struct SledUpdateSpArgs {
     inactive: Option<ExpectedVersion>,
 }
 
+#[derive(Debug, Args)]
+struct SledSetMupdateOverrideArgs {
+    #[clap(flatten)]
+    source: SledMupdateOverrideSource,
+}
+
+#[derive(Debug, Args)]
+#[group(id = "sled-mupdate-override-source", required = true, multiple = false)]
+struct SledMupdateOverrideSource {
+    /// the new value of the mupdate override, or "unset"
+    mupdate_override_id: Option<MupdateOverrideUuidOpt>,
+
+    /// simulate an error reading the mupdate override
+    #[clap(long, conflicts_with = "mupdate_override_id")]
+    with_error: bool,
+}
+
 #[derive(Debug, Args)]
 struct SledRemoveArgs {
     /// id of the sled
@@ -1301,6 +1320,51 @@ fn cmd_sled_set(
                 )))
             }
         }
+        SledSetCommand::MupdateOverride(SledSetMupdateOverrideArgs {
+            source:
+                SledMupdateOverrideSource { mupdate_override_id, with_error },
+        }) => {
+            let (desc, prev) = if with_error {
+                let prev =
+                    system.description_mut().sled_set_mupdate_override_error(
+                        sled_id,
+                        "reconfigurator-cli simulated mupdate-override error"
+                            .to_owned(),
+                    )?;
+                ("error".to_owned(), prev)
+            } else {
+                let mupdate_override_id =
+                    mupdate_override_id.expect("clap ensures that this is set");
+                let prev = system.description_mut().sled_set_mupdate_override(
+                    sled_id,
+                    mupdate_override_id.into(),
+                )?;
+                let desc = match mupdate_override_id {
+                    MupdateOverrideUuidOpt::Set(id) => id.to_string(),
+                    MupdateOverrideUuidOpt::Unset => "unset".to_owned(),
+                };
+                (desc, prev)
+            };
+
+            let prev_desc = match prev {
+                Ok(Some(id)) => id.to_string(),
+                Ok(None) => "unset".to_owned(),
+                Err(_) => "error".to_owned(),
+            };
+
+            sim.commit_and_bump(
+                format!(
+                    "reconfigurator-cli sled-set-mupdate-override: {}: {} -> {}",
+                    sled_id, prev_desc, desc,
+                ),
+                state,
+            );
+
+            Ok(Some(format!(
+                "set sled {} mupdate override: {} -> {}",
+                sled_id, prev_desc, desc,
+            )))
+        }
     }
 }
 

dev-tools/reconfigurator-cli/tests/input/cmds-mupdate-update-flow.txt

@@ -0,0 +1,117 @@
+# Load an example system.
+
+load-example --nsleds 3 --ndisks-per-sled 1
+
+# Create a TUF repository from a fake manifest. We're going to use this
+# repository to test out the minimum release generation flow.
+tuf-assemble ../../update-common/manifests/fake.toml
+set target-release repo-1.0.0.zip
+
+# Update the install dataset on this sled to the target release.
+# (This populates the zone manifest, used for no-op conversions from
+# install dataset to artifact down the road.)
+sled-update-install-dataset serial0 --to-target-release
+
+# Set one of sled 0's zone's image sources to a specific artifact, and
+# also set an MGS update on the sled. Both should be reset as part of
+# this process.
+blueprint-edit latest set-zone-image 0c71b3b2-6ceb-4e8f-b020-b08675e83038 artifact 1.2.3 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+blueprint-edit latest set-sp-update serial0 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 1.1.0 sp 1.0.0 1.0.1
+
+# Simulate a mupdate on sled 0 by setting the mupdate override field to a
+# new UUID (generated using uuidgen).
+sled-set serial0 mupdate-override 6123eac1-ec5b-42ba-b73f-9845105a9971
+
+# On sled 1, simulate an error obtaining the mupdate override.
+sled-set serial1 mupdate-override --with-error
+
+# Also set its SP update, which will not be cleared.
+blueprint-edit latest set-sp-update serial1 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 newest sp newer older
+
+# Simulate a mupdate on sled 2 as well.
+sled-set serial2 mupdate-override 203fa72c-85c1-466a-8ed3-338ee029530d
+
+# Generate a new inventory and plan against that.
+inventory-generate
+blueprint-plan latest latest
+
+# Diff the blueprints. This diff should show:
+#
+# * for sled 0:
+#   * "+ will remove mupdate override"
+#   * for zone 0c71b3b2-6ceb-4e8f-b020-b08675e83038, a change from artifact to install-dataset
+#   * the pending MGS update cleared
+# * for sled 1, no change, because the mupdate override field had an error
+# * for sled 2, "+ will remove mupdate override"
+# * the target release's minimum generation bumped from 1 to 3
+#   (the 3 is because generation 2 is repo-1.0.0.zip)
+blueprint-diff latest
+
+# Hide sled 0 from inventory temporarily -- this does two things:
+# 1. Tests that mupdate/update state transitions don't happen when
+#    the sled isn't present in inventory.
+# 2. We don't want sled 0 to participate in the next few operations
+#    below.
+sled-set serial0 inventory-hidden
+
+# Set the target release to a new repo, causing a generation number bump
+# to 3.
+set target-release repo-1.0.0.zip
+
+# Invoke the planner -- should not proceed with adding or updating zones
+# because sled 0 has a remove-mupdate-override set in the blueprint.
+inventory-generate
+blueprint-plan latest latest
+blueprint-diff latest
+
+# Now simulate the new config being applied to sled 0, which would
+# cause the mupdate override to be removed.
+sled-set serial0 mupdate-override unset
+sled-set serial0 inventory-visible
+
+# But simulate a second mupdate on sled 2. This should invalidate the existing
+# mupdate override on sled 2 and cause another target release minimum
+# generation bump.
+tuf-assemble ../../update-common/manifests/fake-non-semver.toml --allow-non-semver
+sled-update-install-dataset serial2 --from-repo repo-2.0.0.zip
+sled-set serial2 mupdate-override 1c0ce176-6dc8-4a90-adea-d4a8000751da
+
+# Generate a new inventory and plan against that.
+inventory-generate
+blueprint-plan latest latest
+
+# Diff the blueprints. This diff should show:
+# * on sled 0:
+#   * the "remove mupdate override" line going away
+#   * no-op image source switches from install dataset to artifact
+# * on sled 1, no changes
+# * on sled 2, a _change_ in the will-remove-mupdate-override field
+# * another bump to the target release minimum generation, this time to 4.
+blueprint-diff latest
+
+# Clear the mupdate override on sled 2, signifying that the config has been
+# applied.
+sled-set serial2 mupdate-override unset
+
+# Run the planner again. This should cause sled 2's mupdate override field
+# to be unset, but no further planning steps to happen because the
+# target release generation is not new enough.
+inventory-generate
+blueprint-plan latest latest
+blueprint-show latest
+blueprint-diff latest
+
+# Now set the target release -- with this, the rest of the planner starts
+# working again.
+set target-release repo-2.0.0.zip
+blueprint-plan latest latest
+blueprint-show latest
+blueprint-diff latest
+
+# Set the target release minimum generation to a large value -- we're going to
+# test that the planner bails if it attempts a rollback of the target release
+# minimum generation.
+blueprint-edit latest set-target-release-min-gen 1000
+sled-set serial1 mupdate-override cc724abe-80c1-47e6-9771-19e6540531a9
+inventory-generate
+blueprint-plan latest latest

dev-tools/reconfigurator-cli/tests/input/cmds-noop-image-source.txt

@@ -23,10 +23,10 @@ sled-update-install-dataset serial0 --to-target-release
 sled-update-install-dataset serial1 --with-manifest-error
 
 # On a third sled, update the install dataset and simulate a mupdate override.
-# (Currently we do this in the blueprint, but with
-# https://github.com/oxidecomputer/omicron/pull/8456 we should update this test and
-# set a mupdate-override on the sled directly.)
+# Also set it in the blueprint -- this simulates the situation where the mupdate
+# override is in progress and will be cleared in the future.
 sled-update-install-dataset serial2 --to-target-release
+sled-set serial2 mupdate-override ffffffff-ffff-ffff-ffff-ffffffffffff
 blueprint-edit latest set-remove-mupdate-override serial2 ffffffff-ffff-ffff-ffff-ffffffffffff
 
 # On a fourth sled, simulate an error validating the install dataset image on one zone.