-
Notifications
You must be signed in to change notification settings - Fork 905
Allow importing of historical blobs via HTTP API #6656
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Open
macladson
wants to merge
13
commits into
sigp:unstable
Choose a base branch
from
macladson:import-blobs-api
base: unstable
Could not load branches
Branch not found: {{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline,
and old review comments may become outdated.
Open
Changes from all commits
Commits
Show all changes
13 commits
Select commit
Hold shift + click to select a range
f72df2c
First try
macladson 054f0d7
Additional checks and support SSZ
macladson 3c0f7be
Update blobs_manager
macladson cc6bb00
Merge branch 'unstable' into import-blobs-api
macladson 66a41bc
Add verify draft
macladson 5880350
Merge branch 'unstable' into import-blobs-api
macladson 1912cba
Merge remote-tracking branch 'origin/unstable' into import-blobs-api
michaelsproul 3408cbf
Merge branch 'unstable' into import-blobs-api
macladson 366a866
Merge branch 'unstable' into import-blobs-api
macladson 4559e2a
Update exporter
macladson 7d1ae6e
Add blob database verifier
macladson 8c506db
Handle empty blobsidecarlists
macladson 66da335
Cleanup
macladson File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -841,6 +841,95 @@ impl<E: EthSpec, Hot: ItemStore<E>, Cold: ItemStore<E>> HotColdDB<E, Hot, Cold> | |
Ok(()) | ||
} | ||
|
||
/// Import a batch of blobs. | ||
/// Implements the following checks: | ||
/// - Checks that `block_root` is consistent across each `BlobSidecarList`. | ||
/// - Checks that `block_root` exists in the database. | ||
/// - Checks if a `BlobSidecarList` is already stored for that `block_root`. | ||
/// If it is, ensure it matches the `BlobSidecarList` we are attempting to store. | ||
pub fn import_blobs_batch( | ||
&self, | ||
historical_blob_sidecars: Vec<BlobSidecarList<E>>, | ||
) -> Result<(), Error> { | ||
if historical_blob_sidecars.is_empty() { | ||
return Ok(()); | ||
} | ||
|
||
let mut total_imported = 0; | ||
|
||
let mut ops = vec![]; | ||
|
||
for blob_sidecar_list in historical_blob_sidecars { | ||
// Ensure all block_roots in the blob list are the same. | ||
let block_root = { | ||
let first_block_root = blob_sidecar_list[0].block_root(); | ||
if !blob_sidecar_list | ||
.iter() | ||
.all(|blob_sidecar| blob_sidecar.block_root() == first_block_root) | ||
{ | ||
return Err(Error::InvalidBlobImport( | ||
"Inconsistent block roots".to_string(), | ||
)); | ||
} | ||
first_block_root | ||
}; | ||
|
||
// Check block is stored for this block_root. | ||
if !self.block_exists(&block_root)? { | ||
michaelsproul marked this conversation as resolved.
Show resolved
Hide resolved
|
||
warn!( | ||
%block_root, | ||
num_blob_sidecars = blob_sidecar_list.len(), | ||
"Aborting blob import; block does not exist" | ||
); | ||
return Err(Error::InvalidBlobImport("Missing block".to_string())); | ||
} | ||
|
||
// Instead of the above, fully load the block. | ||
|
||
// Check if a `blob_sidecar_list` is already stored for this block root. | ||
match self.get_blobs(&block_root) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Lets try adding a log with the time that this step takes |
||
Ok(BlobSidecarListFromRoot::Blobs(existing_blob_sidecar_list)) => { | ||
// If blobs already exist, only proceed if they match exactly. | ||
if existing_blob_sidecar_list == blob_sidecar_list { | ||
debug!( | ||
block_root = ?block_root, | ||
num_blob_sidecars = blob_sidecar_list.len(), | ||
"Skipping blob sidecar import as identical blob exists" | ||
); | ||
continue; | ||
} else { | ||
return Err(Error::InvalidBlobImport(format!( | ||
"Conflicting blobs exist for block root {:?}", | ||
block_root | ||
))); | ||
} | ||
} | ||
Ok(BlobSidecarListFromRoot::NoRoot) => { | ||
// This block has no existing blobs: proceed with import. | ||
self.blobs_as_kv_store_ops(&block_root, blob_sidecar_list.clone(), &mut ops); | ||
total_imported += blob_sidecar_list.len(); | ||
} | ||
Ok(BlobSidecarListFromRoot::NoBlobs) => { | ||
// This block should not have any blobs: reject the import. | ||
warn!( | ||
%block_root, | ||
"Aborting blob import; blobs should not exist for this block_root" | ||
); | ||
return Err(Error::InvalidBlobImport( | ||
"No blobs should exist for this block_root".to_string(), | ||
)); | ||
} | ||
Err(e) => return Err(Error::InvalidBlobImport(format!("{e:?}"))), | ||
} | ||
} | ||
|
||
self.blobs_db.do_atomically(ops)?; | ||
|
||
debug!(total_imported, "Imported historical blobs"); | ||
|
||
Ok(()) | ||
} | ||
|
||
pub fn blobs_as_kv_store_ops( | ||
&self, | ||
key: &Hash256, | ||
|
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Uh oh!
There was an error while loading. Please reload this page.