Do single-block validation after live-repair and when opening extent

Author: mkeeter

downstairs/src/extent.rs

@@ -35,6 +35,7 @@ pub(crate) trait ExtentInner: Send + Sync + Debug {
     fn flush_number(&self) -> Result<u64, CrucibleError>;
     fn dirty(&self) -> Result<bool, CrucibleError>;
     fn validate(&self) -> Result<(), CrucibleError>;
+    fn validate_block(&self, i: u64) -> Result<(), CrucibleError>;
 
     /// Performs any metadata updates needed before a flush
     fn pre_flush(
@@ -250,7 +251,22 @@ pub fn extent_dir<P: AsRef<Path>>(dir: P, number: ExtentId) -> PathBuf {
  * anchored under "dir".
  */
 pub fn extent_path<P: AsRef<Path>>(dir: P, number: ExtentId) -> PathBuf {
-    extent_dir(dir, number).join(extent_file_name(number, ExtentType::Data))
+    let e = extent_file_name(number, ExtentType::Data);
+
+    // XXX terrible hack: if someone has already provided a full directory tree
+    // ending in `.copy`, then just append the extent file name.  This lets us
+    // open individual extent files during live-repair.
+    if dir
+        .as_ref()
+        .iter()
+        .next_back()
+        .and_then(|s| s.to_str())
+        .is_some_and(|s| s.ends_with(".copy"))
+    {
+        dir.as_ref().join(e)
+    } else {
+        extent_dir(dir, number).join(e)
+    }
 }
 
 /**
@@ -488,6 +504,11 @@ impl Extent {
             }
         };
 
+        // XXX debug validation after opening the extent
+        if let Err(e) = inner.validate_block(0) {
+            panic!("could not validate extent {number}: {e:#?}");
+        }
+
         let extent = Extent {
             number,
             read_only,

downstairs/src/extent_inner_raw.rs

@@ -572,6 +572,65 @@ impl ExtentInner for RawInner {
         r
     }
 
+    fn validate_block(&self, block: u64) -> Result<(), CrucibleError> {
+        let block_size = self.extent_size.block_size_in_bytes() as usize;
+
+        // Read context data to local arrays
+        let ctx_a = self.layout.read_context_slots_contiguous(
+            &self.file,
+            block,
+            1,
+            ContextSlot::A,
+        )?[0];
+        let ctx_b = self.layout.read_context_slots_contiguous(
+            &self.file,
+            block,
+            1,
+            ContextSlot::B,
+        )?[0];
+
+        let mut data = vec![0; block_size];
+        pread_all(
+            self.file.as_fd(),
+            &mut data,
+            block_size as i64 * block as i64,
+        )
+        .map_err(|e| {
+            CrucibleError::IoError(format!(
+                "extent {}: reading block {block} data failed: {e}",
+                self.extent_number
+            ))
+        })?;
+
+        let hash = integrity_hash(&[&data]);
+
+        // Pick out the active context slot
+        let context = match self.active_context[block] {
+            ContextSlot::A => ctx_a,
+            ContextSlot::B => ctx_b,
+        };
+
+        if let Some(context) = context {
+            if context.on_disk_hash == hash {
+                Ok(())
+            } else {
+                Err(CrucibleError::GenericError(format!(
+                    "block {block} has an active slot with mismatched hash"
+                )))
+            }
+        } else {
+            // context slot is empty, hopefully data is as well!
+            if data.iter().all(|v| *v == 0u8) {
+                Ok(())
+            } else {
+                Err(CrucibleError::GenericError(format!(
+                    "block {block} has an empty active slot, \
+                     but contains non-zero data",
+                )))
+            }
+        }
+    }
+
     fn validate(&self) -> Result<(), CrucibleError> {
         let block_size = self.extent_size.block_size_in_bytes() as usize;
 

downstairs/src/extent_inner_sqlite.rs

@@ -103,9 +103,13 @@ impl ExtentInner for SqliteInner {
     }
 
     fn validate(&self) -> Result<(), CrucibleError> {
-        Err(CrucibleError::GenericError(
-            "`validate` is not implemented for Sqlite extent".to_owned(),
-        ))
+        // We don't implement validation for SQLite extents
+        Ok(())
+    }
+
+    fn validate_block(&self, _i: u64) -> Result<(), CrucibleError> {
+        // We don't implement validation for SQLite extents
+        Ok(())
     }
 
     #[cfg(test)]

downstairs/src/region.rs

@@ -682,6 +682,21 @@ impl Region {
             );
         }
 
+        // XXX debug code
+        // Open the extent that we just received, which checks block 0.  We do
+        // this before copying it, to check what was received on the wire.
+        if let Err(e) = Extent::open(
+            &copy_dir,
+            &self.def(),
+            eid,
+            true, // read-only
+            &self.log.clone(),
+        ) {
+            panic!(
+                "Failed to open live-repair extent {eid} in {copy_dir:?}: {e:?}"
+            );
+        }
+
         // After we have all files: move the repair dir.
         info!(
             self.log,