Check hashes during repair

Author: mkeeter

downstairs/src/extent.rs

@@ -251,7 +251,22 @@ pub fn extent_dir<P: AsRef<Path>>(dir: P, number: ExtentId) -> PathBuf {
  * anchored under "dir".
  */
 pub fn extent_path<P: AsRef<Path>>(dir: P, number: ExtentId) -> PathBuf {
-    extent_dir(dir, number).join(extent_file_name(number, ExtentType::Data))
+    let e = extent_file_name(number, ExtentType::Data);
+
+    // XXX terrible hack: if someone has already provided a full directory tree
+    // ending in `.copy`, then just append the extent file name.  This lets us
+    // open individual extent files during live-repair.
+    if dir
+        .as_ref()
+        .iter()
+        .next_back()
+        .and_then(|s| s.to_str())
+        .is_some_and(|s| s.ends_with(".copy"))
+    {
+        dir.as_ref().join(e)
+    } else {
+        extent_dir(dir, number).join(e)
+    }
 }
 
 /**

downstairs/src/extent_inner_sqlite.rs

@@ -104,9 +104,8 @@ impl ExtentInner for SqliteInner {
     }
 
     fn validate(&self) -> Result<(), CrucibleError> {
-        Err(CrucibleError::GenericError(
-            "`validate` is not implemented for Sqlite extent".to_owned(),
-        ))
+        // SQLite databases are always perfect and have no problems
+        Ok(())
     }
 
     #[cfg(test)]

downstairs/src/region.rs

@@ -400,6 +400,30 @@ impl Region {
             assert_eq!(self.get_opened_extent(eid).number, eid);
         }
         assert_eq!(self.def.extent_count() as usize, self.extents.len());
+        use rayon::prelude::*;
+        let errors: Vec<_> = self
+            .extents
+            .par_iter()
+            .filter_map(|e| {
+                let ExtentState::Opened(extent) = e else {
+                    unreachable!("got closed extent");
+                };
+                if let Err(err) = extent.validate() {
+                    Some((extent.number, err))
+                } else {
+                    None
+                }
+            })
+            .collect();
+        if !errors.is_empty() {
+            for (number, err) in &errors {
+                warn!(
+                    self.log,
+                    "validation falied for extent {number}: {err:?}"
+                );
+            }
+            panic!("validation failed");
+        }
     }
 
     /// Walk the list of extents and close each one.
@@ -682,6 +706,28 @@ impl Region {
             );
         }
 
+        // Validate the extent that we just received, before copying it over
+        {
+            let new_extent = match Extent::open(
+                &copy_dir,
+                &self.def(),
+                eid,
+                true, // read-only
+                &self.log.clone(),
+            ) {
+                Ok(e) => e,
+                Err(e) => {
+                    panic!(
+                        "Failed to open live-repair extent {eid} in \
+                         {copy_dir:?}: {e:?}"
+                    );
+                }
+            };
+            if let Err(e) = new_extent.validate() {
+                panic!("Failed to validate live-repair extent {eid}: {e:?}");
+            }
+        }
+
         // After we have all files: move the repair dir.
         info!(
             self.log,