Check hashes during repair and upon startup

.github/buildomat/jobs/test-live-repair.sh

@@ -106,7 +106,7 @@ $BINDIR/dsc start \
 # This gives dsc time to fail, as it is known to happen.  If we don't check,
 # then the later test will just hang forever waiting for downstairs that
 # will never show up.
-sleep 5
+sleep 60 # bonus time for region checking
 dsc_pid=$(pgrep dsc);
 
 if [[ "$dsc_pid" -eq 0 ]]; then

Cargo.toml

@@ -32,6 +32,7 @@ members = [
     "workspace-hack",
     "xtask",
     "agent-antagonist",
+    "verify-raw",
 ]
 resolver = "2"
 

common/src/lib.rs

@@ -145,8 +145,8 @@ pub enum CrucibleError {
     #[error("Invalid downstairs replace {0}")]
     ReplaceRequestInvalid(String),
 
-    #[error("missing context slot for block {0}")]
-    MissingContextSlot(u64),
+    #[error("missing context slot for block {block} in extent {extent}")]
+    MissingContextSlot { block: u64, extent: u32 },
 
     #[error("metadata deserialization failed: {0}")]
     BadMetadata(String),
@@ -458,7 +458,7 @@ impl From<CrucibleError> for dropshot::HttpError {
             | CrucibleError::UpstairsActivateInProgress
             | CrucibleError::UpstairsDeactivating
             | CrucibleError::UuidMismatch
-            | CrucibleError::MissingContextSlot(..)
+            | CrucibleError::MissingContextSlot { .. }
             | CrucibleError::BadMetadata(..)
             | CrucibleError::BadContextSlot(..)
             | CrucibleError::MissingBlockContext

downstairs/src/dump.rs

@@ -11,28 +11,52 @@ struct ExtInfo {
     ei_hm: HashMap<u32, ExtentMeta>,
 }
 
-pub fn verify_region(region_dir: PathBuf, log: Logger) -> Result<()> {
+pub fn verify_region(
+    region_dir: PathBuf,
+    thread_count: Option<usize>,
+    log: Logger,
+) -> Result<()> {
     let region = Region::open(region_dir, false, true, &log)?;
-    let errors: Vec<_> = region
-        .extents
-        .par_iter()
-        .filter_map(|e| {
-            let extent = match e {
-                extent::ExtentState::Opened(extent) => extent,
-                extent::ExtentState::Closed => panic!("dump on closed extent!"),
-            };
 
-            if let Err(err) = extent.validate() {
-                Some((extent.number, err))
-            } else {
-                None
-            }
-        })
-        .collect();
+    // Configure thread pool based on parameter
+    let pool = if let Some(count) = thread_count {
+        rayon::ThreadPoolBuilder::new()
+            .num_threads(count)
+            .build()
+            .expect("Failed to build thread pool")
+    } else {
+        rayon::ThreadPoolBuilder::new()
+            .build()
+            .expect("Failed to build thread pool")
+    };
+
+    let errors: Vec<_> = pool.install(|| {
+        region
+            .extents
+            .par_iter()
+            .filter_map(|e| {
+                let extent = match e {
+                    extent::ExtentState::Opened(extent) => extent,
+                    extent::ExtentState::Closed => {
+                        panic!("dump on closed extent!")
+                    }
+                };
+
+                if let Err(err) = extent.validate() {
+                    Some((extent.number, err))
+                } else {
+                    None
+                }
+            })
+            .collect()
+    });
 
     if !errors.is_empty() {
         for (number, err) in &errors {
-            println!("validation failed for extent {}: {:?}", number, err);
+            println!(
+                "validation failed for extent {} 0x{:03X} : {:?}",
+                number, number.0, err
+            );
         }
         bail!("Region failed to verify");
     }

downstairs/src/extent.rs

@@ -251,7 +251,22 @@ pub fn extent_dir<P: AsRef<Path>>(dir: P, number: ExtentId) -> PathBuf {
  * anchored under "dir".
  */
 pub fn extent_path<P: AsRef<Path>>(dir: P, number: ExtentId) -> PathBuf {
-    extent_dir(dir, number).join(extent_file_name(number, ExtentType::Data))
+    let e = extent_file_name(number, ExtentType::Data);
+
+    // XXX terrible hack: if someone has already provided a full directory tree
+    // ending in `.copy`, then just append the extent file name.  This lets us
+    // open individual extent files during live-repair.
+    if dir
+        .as_ref()
+        .iter()
+        .next_back()
+        .and_then(|s| s.to_str())
+        .is_some_and(|s| s.ends_with(".copy"))
+    {
+        dir.as_ref().join(e)
+    } else {
+        extent_dir(dir, number).join(e)
+    }
 }
 
 /**