fix: prune finalized blocks and attestations (#83)

Implements automatic pruning of finalized data to prevent unbounded
storage growth. Adds a LiveChain index table to optimize fork choice
operations while enabling efficient cleanup of finalized blocks,
attestations, and signatures.

## Summary

When slots become finalized, they no longer participate in fork choice.
This PR:
1. Adds a LiveChain index for fast fork choice (avoids Block
deserialization)
2. Automatically prunes finalized data when checkpoints advance
3. Moves signature-related types to storage module for proper
encapsulation

## Key Changes

### LiveChain Index
- New `LiveChain` table: `(slot, root) -> parent_root` mappings (40
bytes per block)
- Big-endian slot encoding enables efficient range scans and early
termination
- Refactored `compute_lmd_ghost_head` to use `HashMap<H256, (u64,
H256)>` instead of full `Block`
- Automatic pruning removes finalized entries while preserving Blocks
table for historical queries

### Automatic Pruning
- `update_checkpoints()` triggers pruning when finalization advances
- **LiveChain**: Prunes index entries for slots < finalized_slot (keeps
finalized block itself)
- **GossipSignatures**: Removes signatures for finalized attestations
(no longer needed for block building)
- **AggregatedPayloads**: Removes aggregated proofs for finalized
attestations

## Implementation Details

**Why pruning on finalization?**
- Finalized attestations can't contribute to future fork choice
decisions
- Gossip signatures are only needed for building new blocks on live
forks
- Aggregated payloads from finalized blocks won't be reused

## Testing

Ran a validator node for 1 hour with these changes. Storage reached ~1GB
with a downward slope (actively pruning) compared to ~5GB before pruning
was implemented.

Also tested via existing spec tests:
- `forkchoice_spectests.rs`: Fork choice with LiveChain index
- `signature_spectests.rs`: Block building with new signature types
- `stf_spectests.rs`: State transition with pruned storage

Author: MegaRedHand

crates/blockchain/fork_choice/src/lib.rs

@@ -1,6 +1,6 @@
 use std::collections::HashMap;
 
-use ethlambda_types::{attestation::AttestationData, block::Block, primitives::H256};
+use ethlambda_types::{attestation::AttestationData, primitives::H256};
 
 /// Compute the LMD GHOST head of the chain, given a starting root, a set of blocks,
 /// a set of attestations, and a minimum score threshold.
@@ -9,7 +9,7 @@ use ethlambda_types::{attestation::AttestationData, block::Block, primitives::H2
 // TODO: add proto-array implementation
 pub fn compute_lmd_ghost_head(
     mut start_root: H256,
-    blocks: &HashMap<H256, Block>,
+    blocks: &HashMap<H256, (u64, H256)>,
     attestations: &HashMap<u64, AttestationData>,
     min_score: u64,
 ) -> H256 {
@@ -19,36 +19,33 @@ pub fn compute_lmd_ghost_head(
     if start_root.is_zero() {
         start_root = *blocks
             .iter()
-            .min_by_key(|(_, block)| block.slot)
+            .min_by_key(|(_, (slot, _))| slot)
             .map(|(root, _)| root)
             .expect("we already checked blocks is non-empty");
     }
-    let start_slot = blocks[&start_root].slot;
+    let start_slot = blocks[&start_root].0;
     let mut weights: HashMap<H256, u64> = HashMap::new();
 
     for attestation_data in attestations.values() {
         let mut current_root = attestation_data.head.root;
-        while let Some(block) = blocks.get(&current_root)
-            && block.slot > start_slot
+        while let Some(&(slot, parent_root)) = blocks.get(&current_root)
+            && slot > start_slot
         {
             *weights.entry(current_root).or_default() += 1;
-            current_root = block.parent_root;
+            current_root = parent_root;
         }
     }
 
     let mut children_map: HashMap<H256, Vec<H256>> = HashMap::new();
 
-    for (root, block) in blocks {
-        if block.parent_root.is_zero() {
+    for (root, &(_, parent_root)) in blocks {
+        if parent_root.is_zero() {
             continue;
         }
         if min_score > 0 && *weights.get(root).unwrap_or(&0) < min_score {
             continue;
         }
-        children_map
-            .entry(block.parent_root)
-            .or_default()
-            .push(*root);
+        children_map.entry(parent_root).or_default().push(*root);
     }
 
     let mut head = start_root;

crates/blockchain/src/key_manager.rs

@@ -2,7 +2,7 @@ use std::collections::HashMap;
 
 use ethlambda_types::{
     attestation::{AttestationData, XmssSignature},
-    primitives::{H256, TreeHash},
+    primitives::{H256, ssz::TreeHash},
     signature::{ValidatorSecretKey, ValidatorSignature},
 };
 

crates/blockchain/src/lib.rs

@@ -3,12 +3,11 @@ use std::time::{Duration, SystemTime};
 
 use ethlambda_state_transition::is_proposer;
 use ethlambda_storage::Store;
-use ethlambda_types::primitives::H256;
 use ethlambda_types::{
     ShortRoot,
     attestation::{Attestation, AttestationData, SignedAttestation},
     block::{BlockSignatures, BlockWithAttestation, SignedBlockWithAttestation},
-    primitives::TreeHash,
+    primitives::{H256, ssz::TreeHash},
     signature::ValidatorSecretKey,
     state::Checkpoint,
 };

crates/blockchain/src/store.rs

@@ -12,7 +12,7 @@ use ethlambda_types::{
         AggregatedAttestations, AggregatedSignatureProof, AggregationBits, Block, BlockBody,
         SignedBlockWithAttestation,
     },
-    primitives::{H256, TreeHash},
+    primitives::{H256, ssz::TreeHash},
     signature::ValidatorSignature,
     state::{Checkpoint, State, Validator},
 };
@@ -30,7 +30,7 @@ fn accept_new_attestations(store: &mut Store) {
 
 /// Update the head based on the fork choice rule.
 fn update_head(store: &mut Store) {
-    let blocks: HashMap<H256, Block> = store.iter_blocks().collect();
+    let blocks = store.get_live_chain();
     let attestations: HashMap<u64, AttestationData> = store.iter_known_attestations().collect();
     let old_head = store.head();
     let new_head = ethlambda_fork_choice::compute_lmd_ghost_head(
@@ -69,7 +69,7 @@ fn update_safe_target(store: &mut Store) {
 
     let min_target_score = (num_validators * 2).div_ceil(3);
 
-    let blocks: HashMap<H256, Block> = store.iter_blocks().collect();
+    let blocks = store.get_live_chain();
     let attestations: HashMap<u64, AttestationData> = store.iter_new_attestations().collect();
     let safe_target = ethlambda_fork_choice::compute_lmd_ghost_head(
         store.latest_justified().root,
@@ -217,10 +217,9 @@ pub fn on_gossip_attestation(
 
     if cfg!(not(feature = "skip-signature-verification")) {
         // Store signature for later lookup during block building
-        let signature_key = (validator_id, message);
         let signature = ValidatorSignature::from_bytes(&signed_attestation.signature)
             .map_err(|_| StoreError::SignatureDecodingFailed)?;
-        store.insert_gossip_signature(signature_key, signature);
+        store.insert_gossip_signature(&attestation.data, validator_id, signature);
     }
     metrics::inc_attestations_valid("gossip");
 
@@ -382,12 +381,10 @@ pub fn on_block(
         .zip(attestation_signatures.iter())
     {
         let validator_ids = aggregation_bits_to_validator_indices(&att.aggregation_bits);
-        let data_root = att.data.tree_hash_root();
 
         for validator_id in validator_ids {
             // Update Proof Map - Store the proof so future block builders can reuse this aggregation
-            let key: SignatureKey = (validator_id, data_root);
-            store.push_aggregated_payload(key, proof.clone());
+            store.insert_aggregated_payload(&att.data, validator_id, proof.clone());
 
             // Update Fork Choice - Register the vote immediately (historical/on-chain)
             let attestation = Attestation {
@@ -415,14 +412,14 @@ pub fn on_block(
 
     if cfg!(not(feature = "skip-signature-verification")) {
         // Store the proposer's signature for potential future block building
-        let proposer_sig_key: SignatureKey = (
-            proposer_attestation.validator_id,
-            proposer_attestation.data.tree_hash_root(),
-        );
         let proposer_sig =
             ValidatorSignature::from_bytes(&signed_block.signature.proposer_signature)
                 .map_err(|_| StoreError::SignatureDecodingFailed)?;
-        store.insert_gossip_signature(proposer_sig_key, proposer_sig);
+        store.insert_gossip_signature(
+            &proposer_attestation.data,
+            proposer_attestation.validator_id,
+            proposer_sig,
+        );
     }
 
     // Process proposer attestation (enters "new" stage, not "known")
@@ -559,13 +556,20 @@ pub fn produce_block_with_signatures(
         .collect();
 
     // Get known block roots for attestation validation
-    let known_block_roots: HashSet<H256> = store.iter_blocks().map(|(root, _)| root).collect();
+    let known_block_roots = store.get_block_roots();
 
     // Collect signature data for block building
-    let gossip_signatures: HashMap<SignatureKey, ValidatorSignature> =
-        store.iter_gossip_signatures().collect();
-    let aggregated_payloads: HashMap<SignatureKey, Vec<AggregatedSignatureProof>> =
-        store.iter_aggregated_payloads().collect();
+    let gossip_signatures: HashMap<SignatureKey, ValidatorSignature> = store
+        .iter_gossip_signatures()
+        .filter_map(|(key, stored)| stored.to_validator_signature().ok().map(|sig| (key, sig)))
+        .collect();
+    let aggregated_payloads: HashMap<SignatureKey, Vec<AggregatedSignatureProof>> = store
+        .iter_aggregated_payloads()
+        .map(|(key, stored_payloads)| {
+            let proofs = stored_payloads.into_iter().map(|sp| sp.proof).collect();
+            (key, proofs)
+        })
+        .collect();
 
     // Build the block using fixed-point attestation collection
     let (block, _post_state, signatures) = build_block(

crates/blockchain/state_transition/src/lib.rs

@@ -3,7 +3,7 @@ use std::collections::HashMap;
 use ethlambda_types::{
     ShortRoot,
     block::{AggregatedAttestations, Block, BlockHeader},
-    primitives::{H256, TreeHash},
+    primitives::{H256, ssz::TreeHash},
     state::{Checkpoint, JustificationValidators, State},
 };
 use tracing::info;

crates/blockchain/tests/forkchoice_spectests.rs

@@ -9,7 +9,7 @@ use ethlambda_storage::{Store, backend::InMemoryBackend};
 use ethlambda_types::{
     attestation::Attestation,
     block::{Block, BlockSignatures, BlockWithAttestation, SignedBlockWithAttestation},
-    primitives::{H256, TreeHash, VariableList},
+    primitives::{H256, VariableList, ssz::TreeHash},
     state::State,
 };
 
@@ -165,15 +165,15 @@ fn validate_checks(
         }
 
         // Also validate the root matches a block at this slot
-        let blocks: HashMap<H256, Block> = st.iter_blocks().collect();
+        let blocks = st.get_live_chain();
         let block_found = blocks
             .iter()
-            .any(|(root, block)| block.slot == expected_slot && *root == target.root);
+            .any(|(root, (slot, _))| *slot == expected_slot && *root == target.root);
 
         if !block_found {
             let available: Vec<_> = blocks
                 .iter()
-                .filter(|(_, block)| block.slot == expected_slot)
+                .filter(|(_, (slot, _))| *slot == expected_slot)
                 .map(|(root, _)| format!("{:?}", root))
                 .collect();
             return Err(format!(
@@ -365,7 +365,7 @@ fn validate_lexicographic_head_among(
         .into());
     }
 
-    let blocks: HashMap<H256, Block> = st.iter_blocks().collect();
+    let blocks = st.get_live_chain();
     let known_attestations: HashMap<u64, AttestationData> = st.iter_known_attestations().collect();
 
     // Resolve all fork labels to roots and compute their weights
@@ -380,13 +380,12 @@ fn validate_lexicographic_head_among(
             )
         })?;
 
-        let block = blocks.get(root).ok_or_else(|| {
+        let (slot, _parent_root) = blocks.get(root).ok_or_else(|| {
             format!(
                 "Step {}: block for label '{}' not found in store",
                 step_idx, label
             )
         })?;
-        let slot = block.slot;
 
         // Calculate attestation weight: count attestations voting for this fork
         // An attestation votes for this fork if its head is this block or a descendant
@@ -396,26 +395,26 @@ fn validate_lexicographic_head_among(
             // Check if attestation head is this block or a descendant
             if att_head_root == *root {
                 weight += 1;
-            } else if let Some(att_block) = blocks.get(&att_head_root) {
+            } else if let Some(&(att_slot, _)) = blocks.get(&att_head_root) {
                 // Walk back from attestation head to see if we reach this block
                 let mut current = att_head_root;
-                let mut current_slot = att_block.slot;
-                while current_slot > slot {
-                    if let Some(blk) = blocks.get(&current) {
-                        if blk.parent_root == *root {
+                let mut current_slot = att_slot;
+                while current_slot > *slot {
+                    if let Some(&(_, parent_root)) = blocks.get(&current) {
+                        if parent_root == *root {
                             weight += 1;
                             break;
                         }
-                        current = blk.parent_root;
-                        current_slot = blocks.get(&current).map(|b| b.slot).unwrap_or(0);
+                        current = parent_root;
+                        current_slot = blocks.get(&current).map(|(s, _)| *s).unwrap_or(0);
                     } else {
                         break;
                     }
                 }
             }
         }
 
-        fork_data.insert(label.as_str(), (*root, slot, weight));
+        fork_data.insert(label.as_str(), (*root, *slot, weight));
     }
 
     // Verify all forks are at the same slot

crates/blockchain/tests/signature_spectests.rs

@@ -5,7 +5,7 @@ use ethlambda_blockchain::{SECONDS_PER_SLOT, store};
 use ethlambda_storage::{Store, backend::InMemoryBackend};
 use ethlambda_types::{
     block::{Block, SignedBlockWithAttestation},
-    primitives::TreeHash,
+    primitives::ssz::TreeHash,
     state::State,
 };
 

crates/blockchain/tests/signature_types.rs

@@ -7,10 +7,12 @@ use ethlambda_types::block::{
     AttestationSignatures, Block as EthBlock, BlockBody as EthBlockBody, BlockSignatures,
     BlockWithAttestation, SignedBlockWithAttestation,
 };
-use ethlambda_types::primitives::{BitList, Encode, H256, VariableList};
+use ethlambda_types::primitives::{
+    BitList, H256, VariableList,
+    ssz::{Decode as SszDecode, Encode as SszEncode},
+};
 use ethlambda_types::state::{Checkpoint as EthCheckpoint, State, ValidatorPubkeyBytes};
 use serde::Deserialize;
-use ssz_derive::{Decode as SszDecode, Encode as SszEncode};
 use ssz_types::FixedVector;
 use ssz_types::typenum::{U28, U32};
 use std::collections::HashMap;

crates/common/crypto/src/lib.rs

@@ -1,9 +1,11 @@
 use std::sync::Once;
 
-use ethlambda_types::primitives::{Decode, Encode};
 use ethlambda_types::{
     block::ByteListMiB,
-    primitives::H256,
+    primitives::{
+        H256,
+        ssz::{Decode, Encode},
+    },
     signature::{ValidatorPublicKey, ValidatorSignature},
 };
 use lean_multisig::{

crates/common/types/src/attestation.rs

@@ -1,7 +1,5 @@
-use ssz_derive::{Decode, Encode};
-use tree_hash_derive::TreeHash;
-
 use crate::{
+    primitives::ssz::{Decode, Encode, TreeHash},
     signature::SignatureSize,
     state::{Checkpoint, ValidatorRegistryLimit},
 };