Skip to content

Commit 179e164

Browse files
RjectedRjected
committed
feat: add StaticFileSegment::AccountChangeSets
chore: add changeset info to segment header feat(provider): implement append_account_changeset chore: formatting and more TODOs chore: add stub for account_before_block fix: use proper value for account changeset static files feat: impl binary search for address in changeset feat: finish writer fn, add tests chore: add fallback to db when not found in static files feat: add account_changesets_range to ChangesetReader feat: add support for account changesets in IndexAccountHistory feat: add support for account changesets in static file producer chore: serialization changes feat: add support for account changesets in db get chore: docs and attempt to fix db get fix: properly upgrade segment header serialize / deserialize feat: add read-only segments feat: add cli flag for enabling v2 static files chore: remove Segment for AccountChangeSets chore: add unreachable! for unreachable db get branch chore: remove noisy traces chore: make binary search more concise feat: add support for account changeset static file pruning feat: make HashedPostState::from_reverts work with static files fix: don't write to db chore: update book cli chore: make clippy happy chore: fix test compilation chore: fix feat propagation feat: add StaticFileRangeWalker skeleton fix: fix inclusive ranges feat: impl removal for changesets in exec unwind chore: replace prefixsetloader with fn that uses provider chore(trie): propagate errors from root methods chore: make clippy happy chore: no underscores chore(static-file): call increment_block when appending account changesets chore(merkle-changesets): add logs for changeset reverts fix: return 0 for start feat: add account_changeset_count to ChangeSetReader fix: introduce walker to prevent OOM in IndexAccountHistory fix: remove static file v2 read-only segment stuff chore: remove traces feat: add storage_settings branches chore: backwards compat segment header writing fix: only try to read offsets for offset segments fix: fix test for account changeset format chore: add lz4 and make database check better fix: custom serializer for SegmentHeader chore: update settings cli to have account changeset option chore: integrate EitherWriter for account changesets chore: rm outdated doc chore: check storage settings for reading always chore: add note on changeset static file initialization chore: doc fixes chore: make clippy happy chore: update book cli chore: use core instead of std chore: fix doc links fix: stop using stale storage settings fix: fix incremental walker loop chore: update snapshot test
1 parent 014f115 commit 179e164

File tree

69 files changed

+2523
-318
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

69 files changed

+2523
-318
lines changed

‎Cargo.lock‎

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/cli/commands/src/db/get.rs‎

Lines changed: 72 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,8 @@ use alloy_primitives::{hex, BlockHash};
22
use clap::Parser;
33
use reth_db::{
44
static_file::{
5-
ColumnSelectorOne, ColumnSelectorTwo, HeaderWithHashMask, ReceiptMask, TransactionMask,
6-
TransactionSenderMask,
5+
AccountChangesetMask, ColumnSelectorOne, ColumnSelectorTwo, HeaderWithHashMask,
6+
ReceiptMask, TransactionMask, TransactionSenderMask,
77
},
88
RawDupSort,
99
};
@@ -14,7 +14,7 @@ use reth_db_api::{
1414
use reth_db_common::DbTool;
1515
use reth_node_api::{HeaderTy, ReceiptTy, TxTy};
1616
use reth_node_builder::NodeTypesWithDB;
17-
use reth_provider::{providers::ProviderNodeTypes, StaticFileProviderFactory};
17+
use reth_provider::{providers::ProviderNodeTypes, ChangeSetReader, StaticFileProviderFactory};
1818
use reth_static_file_types::StaticFileSegment;
1919
use tracing::error;
2020

@@ -51,6 +51,10 @@ enum Subcommand {
5151
#[arg(value_parser = maybe_json_value_parser)]
5252
key: String,
5353

54+
/// The subkey to get content for, for example address in changeset
55+
#[arg(value_parser = maybe_json_value_parser)]
56+
subkey: Option<String>,
57+
5458
/// Output bytes instead of human-readable decoded value
5559
#[arg(long)]
5660
raw: bool,
@@ -64,33 +68,77 @@ impl Command {
6468
Subcommand::Mdbx { table, key, subkey, raw } => {
6569
table.view(&GetValueViewer { tool, key, subkey, raw })?
6670
}
67-
Subcommand::StaticFile { segment, key, raw } => {
68-
let (key, mask): (u64, _) = match segment {
71+
Subcommand::StaticFile { segment, key, subkey, raw } => {
72+
let (key, subkey, mask): (u64, _, _) = match segment {
6973
StaticFileSegment::Headers => (
7074
table_key::<tables::Headers>(&key)?,
75+
None,
7176
<HeaderWithHashMask<HeaderTy<N>>>::MASK,
7277
),
73-
StaticFileSegment::Transactions => {
74-
(table_key::<tables::Transactions>(&key)?, <TransactionMask<TxTy<N>>>::MASK)
75-
}
76-
StaticFileSegment::Receipts => {
77-
(table_key::<tables::Receipts>(&key)?, <ReceiptMask<ReceiptTy<N>>>::MASK)
78-
}
78+
StaticFileSegment::Transactions => (
79+
table_key::<tables::Transactions>(&key)?,
80+
None,
81+
<TransactionMask<TxTy<N>>>::MASK,
82+
),
83+
StaticFileSegment::Receipts => (
84+
table_key::<tables::Receipts>(&key)?,
85+
None,
86+
<ReceiptMask<ReceiptTy<N>>>::MASK,
87+
),
7988
StaticFileSegment::TransactionSenders => (
8089
table_key::<tables::TransactionSenders>(&key)?,
81-
<TransactionSenderMask>::MASK,
90+
None,
91+
TransactionSenderMask::MASK,
8292
),
93+
StaticFileSegment::AccountChangeSets => {
94+
let subkey =
95+
table_subkey::<tables::AccountChangeSets>(subkey.as_deref()).ok();
96+
(
97+
table_key::<tables::AccountChangeSets>(&key)?,
98+
subkey,
99+
AccountChangesetMask::MASK,
100+
)
101+
}
83102
};
84103

85-
let content = tool
86-
.provider_factory
87-
.static_file_provider()
88-
.get_segment_provider(segment, key)?
89-
.cursor()?
90-
.get(key.into(), mask)
91-
.map(|result| {
92-
result.map(|vec| vec.iter().map(|slice| slice.to_vec()).collect::<Vec<_>>())
93-
})?;
104+
// handle account changesets differently if a subkey is provided.
105+
if let StaticFileSegment::AccountChangeSets = segment {
106+
let Some(subkey) = subkey else {
107+
// get all changesets for the block
108+
let changesets = tool
109+
.provider_factory
110+
.static_file_provider()
111+
.account_block_changeset(key)?;
112+
113+
println!("{}", serde_json::to_string_pretty(&changesets)?);
114+
return Ok(())
115+
};
116+
117+
let account = tool
118+
.provider_factory
119+
.static_file_provider()
120+
.get_account_before_block(key, subkey)?;
121+
122+
if let Some(account) = account {
123+
println!("{}", serde_json::to_string_pretty(&account)?);
124+
} else {
125+
error!(target: "reth::cli", "No content for the given table key.");
126+
}
127+
128+
return Ok(())
129+
}
130+
131+
let content = tool.provider_factory.static_file_provider().find_static_file(
132+
segment,
133+
|provider| {
134+
let mut cursor = provider.cursor()?;
135+
cursor.get(key.into(), mask).map(|result| {
136+
result.map(|vec| {
137+
vec.iter().map(|slice| slice.to_vec()).collect::<Vec<_>>()
138+
})
139+
})
140+
},
141+
)?;
94142

95143
match content {
96144
Some(content) => {
@@ -126,6 +174,9 @@ impl Command {
126174
)?;
127175
println!("{}", serde_json::to_string_pretty(&sender)?);
128176
}
177+
StaticFileSegment::AccountChangeSets => {
178+
unreachable!("account changeset static files are special cased before this match")
179+
}
129180
}
130181
}
131182
}

‎crates/cli/commands/src/db/settings.rs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,11 @@ pub enum SetCommand {
4949
#[clap(action(ArgAction::Set))]
5050
value: bool,
5151
},
52+
/// Store account changesets in static files instead of the database
53+
AccountChangesetsInStaticFiles {
54+
#[clap(action(ArgAction::Set))]
55+
value: bool,
56+
},
5257
}
5358

5459
impl Command {
@@ -91,6 +96,7 @@ impl Command {
9196
let mut settings @ StorageSettings {
9297
receipts_in_static_files: _,
9398
transaction_senders_in_static_files: _,
99+
account_changesets_in_static_files: _,
94100
} = settings.unwrap_or_else(StorageSettings::legacy);
95101

96102
// Update the setting based on the key
@@ -111,6 +117,14 @@ impl Command {
111117
settings.transaction_senders_in_static_files = value;
112118
println!("Set transaction_senders_in_static_files = {}", value);
113119
}
120+
SetCommand::AccountChangesetsInStaticFiles { value } => {
121+
if settings.account_changesets_in_static_files == value {
122+
println!("account_changesets_in_static_files is already set to {}", value);
123+
return Ok(());
124+
}
125+
settings.account_changesets_in_static_files = value;
126+
println!("Set account_changesets_in_static_files = {}", value);
127+
}
114128
}
115129

116130
// Write updated settings

‎crates/cli/commands/src/stage/drop.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,9 @@ impl<C: ChainSpecParser> Command<C> {
8787
.unwrap_or_default();
8888
writer.prune_transaction_senders(to_delete, 0)?;
8989
}
90+
StaticFileSegment::AccountChangeSets => {
91+
writer.prune_account_changesets(highest_block)?;
92+
}
9093
}
9194
}
9295
}

‎crates/config/src/config.rs‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -438,15 +438,22 @@ pub struct BlocksPerFileConfig {
438438
pub receipts: Option<u64>,
439439
/// Number of blocks per file for the transaction senders segment.
440440
pub transaction_senders: Option<u64>,
441+
/// Number of blocks per file for the account changesets segment.
442+
pub account_change_sets: Option<u64>,
441443
}
442444

443445
impl StaticFilesConfig {
444446
/// Validates the static files configuration.
445447
///
446448
/// Returns an error if any blocks per file value is zero.
447449
pub fn validate(&self) -> eyre::Result<()> {
448-
let BlocksPerFileConfig { headers, transactions, receipts, transaction_senders } =
449-
self.blocks_per_file;
450+
let BlocksPerFileConfig {
451+
headers,
452+
transactions,
453+
receipts,
454+
transaction_senders,
455+
account_change_sets,
456+
} = self.blocks_per_file;
450457
eyre::ensure!(headers != Some(0), "Headers segment blocks per file must be greater than 0");
451458
eyre::ensure!(
452459
transactions != Some(0),
@@ -460,13 +467,22 @@ impl StaticFilesConfig {
460467
transaction_senders != Some(0),
461468
"Transaction senders segment blocks per file must be greater than 0"
462469
);
470+
eyre::ensure!(
471+
account_change_sets != Some(0),
472+
"Account changesets segment blocks per file must be greater than 0"
473+
);
463474
Ok(())
464475
}
465476

466477
/// Converts the blocks per file configuration into a [`HashMap`] per segment.
467478
pub fn as_blocks_per_file_map(&self) -> HashMap<StaticFileSegment, u64> {
468-
let BlocksPerFileConfig { headers, transactions, receipts, transaction_senders } =
469-
self.blocks_per_file;
479+
let BlocksPerFileConfig {
480+
headers,
481+
transactions,
482+
receipts,
483+
transaction_senders,
484+
account_change_sets,
485+
} = self.blocks_per_file;
470486

471487
let mut map = HashMap::new();
472488
// Iterating over all possible segments allows us to do an exhaustive match here,
@@ -477,6 +493,7 @@ impl StaticFilesConfig {
477493
StaticFileSegment::Transactions => transactions,
478494
StaticFileSegment::Receipts => receipts,
479495
StaticFileSegment::TransactionSenders => transaction_senders,
496+
StaticFileSegment::AccountChangeSets => account_change_sets,
480497
};
481498

482499
if let Some(blocks_per_file) = blocks_per_file {

‎crates/e2e-test-utils/src/setup_import.rs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,6 @@ pub async fn setup_engine_with_chain_import(
112112
let static_files_path = datadir.join("static_files");
113113

114114
// Initialize the database using init_db (same as CLI import command)
115-
// Use the same database arguments as the node will use
116115
let db_args = reth_node_core::args::DatabaseArgs::default().database_args();
117116
let db_env = reth_db::init_db(&db_path, db_args)?;
118117
let db = Arc::new(db_env);
@@ -314,7 +313,8 @@ mod tests {
314313

315314
// Import the chain
316315
{
317-
let db_env = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap();
316+
let db_args = reth_node_core::args::DatabaseArgs::default().database_args();
317+
let db_env = reth_db::init_db(&db_path, db_args).unwrap();
318318
let db = Arc::new(db_env);
319319

320320
let provider_factory: ProviderFactory<
@@ -466,7 +466,8 @@ mod tests {
466466
let datadir = temp_dir.path().join("datadir");
467467
std::fs::create_dir_all(&datadir).unwrap();
468468
let db_path = datadir.join("db");
469-
let db_env = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap();
469+
let db_args = reth_node_core::args::DatabaseArgs::default().database_args();
470+
let db_env = reth_db::init_db(&db_path, db_args).unwrap();
470471
let db = Arc::new(reth_db::test_utils::TempDatabase::new(db_env, db_path));
471472

472473
// Create static files path

‎crates/engine/tree/src/tree/payload_processor/multiproof.rs‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1270,8 +1270,8 @@ mod tests {
12701270
use alloy_primitives::map::B256Set;
12711271
use reth_provider::{
12721272
providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory,
1273-
BlockReader, DatabaseProviderFactory, PruneCheckpointReader, StageCheckpointReader,
1274-
TrieReader,
1273+
BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory,
1274+
PruneCheckpointReader, StageCheckpointReader, TrieReader,
12751275
};
12761276
use reth_trie::MultiProof;
12771277
use reth_trie_parallel::proof_task::{ProofTaskCtx, ProofWorkerHandle};
@@ -1293,7 +1293,12 @@ mod tests {
12931293
fn create_test_state_root_task<F>(factory: F) -> MultiProofTask
12941294
where
12951295
F: DatabaseProviderFactory<
1296-
Provider: BlockReader + TrieReader + StageCheckpointReader + PruneCheckpointReader,
1296+
Provider: BlockReader
1297+
+ TrieReader
1298+
+ StageCheckpointReader
1299+
+ PruneCheckpointReader
1300+
+ ChangeSetReader
1301+
+ BlockNumReader,
12971302
> + Clone
12981303
+ Send
12991304
+ 'static,

‎crates/engine/tree/src/tree/payload_validator.rs‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,10 @@ use reth_primitives_traits::{
3232
AlloyBlockHeader, BlockTy, GotExpected, NodePrimitives, RecoveredBlock, SealedHeader,
3333
};
3434
use reth_provider::{
35-
providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockReader,
36-
DatabaseProviderFactory, ExecutionOutcome, HashedPostStateProvider, ProviderError,
37-
PruneCheckpointReader, StageCheckpointReader, StateProvider, StateProviderFactory, StateReader,
38-
StateRootProvider, TrieReader,
35+
providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader,
36+
ChangeSetReader, DatabaseProviderFactory, ExecutionOutcome, HashedPostStateProvider,
37+
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider,
38+
StateProviderFactory, StateReader, StateRootProvider, TrieReader,
3939
};
4040
use reth_revm::db::State;
4141
use reth_trie::{updates::TrieUpdates, HashedPostState, TrieInputSorted};
@@ -127,8 +127,15 @@ impl<N, P, Evm, V> BasicEngineValidator<P, Evm, V>
127127
where
128128
N: NodePrimitives,
129129
P: DatabaseProviderFactory<
130-
Provider: BlockReader + TrieReader + StageCheckpointReader + PruneCheckpointReader,
130+
Provider: BlockReader
131+
+ TrieReader
132+
+ StageCheckpointReader
133+
+ PruneCheckpointReader
134+
+ ChangeSetReader
135+
+ BlockNumReader,
131136
> + BlockReader<Header = N::BlockHeader>
137+
+ ChangeSetReader
138+
+ BlockNumReader
132139
+ StateProviderFactory
133140
+ StateReader
134141
+ HashedPostStateProvider
@@ -998,10 +1005,17 @@ pub trait EngineValidator<
9981005
impl<N, Types, P, Evm, V> EngineValidator<Types> for BasicEngineValidator<P, Evm, V>
9991006
where
10001007
P: DatabaseProviderFactory<
1001-
Provider: BlockReader + TrieReader + StageCheckpointReader + PruneCheckpointReader,
1008+
Provider: BlockReader
1009+
+ TrieReader
1010+
+ StageCheckpointReader
1011+
+ PruneCheckpointReader
1012+
+ ChangeSetReader
1013+
+ BlockNumReader,
10021014
> + BlockReader<Header = N::BlockHeader>
10031015
+ StateProviderFactory
10041016
+ StateReader
1017+
+ ChangeSetReader
1018+
+ BlockNumReader
10051019
+ HashedPostStateProvider
10061020
+ Clone
10071021
+ 'static,

‎crates/evm/execution-errors/src/trie.rs‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,21 +7,27 @@ use reth_storage_errors::{db::DatabaseError, provider::ProviderError};
77
use thiserror::Error;
88

99
/// State root errors.
10-
#[derive(Error, PartialEq, Eq, Clone, Debug)]
10+
#[derive(Error, Clone, Debug)]
1111
pub enum StateRootError {
1212
/// Internal database error.
1313
#[error(transparent)]
1414
Database(#[from] DatabaseError),
1515
/// Storage root error.
1616
#[error(transparent)]
1717
StorageRootError(#[from] StorageRootError),
18+
/// Provider error when loading prefix sets
19+
#[error(transparent)]
20+
PrefixSetLoadError(#[from] ProviderError),
1821
}
1922

20-
impl From<StateRootError> for DatabaseError {
21-
fn from(err: StateRootError) -> Self {
22-
match err {
23+
impl From<StateRootError> for ProviderError {
24+
fn from(value: StateRootError) -> Self {
25+
match value {
2326
StateRootError::Database(err) |
24-
StateRootError::StorageRootError(StorageRootError::Database(err)) => err,
27+
StateRootError::StorageRootError(StorageRootError::Database(err)) => {
28+
Self::Database(err)
29+
}
30+
StateRootError::PrefixSetLoadError(err) => err,
2531
}
2632
}
2733
}

0 commit comments

Comments
 (0)