Skip to content

Add Firewood state dump tooling for targeted block-height debugging #5226

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rkuris wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Add Firewood state dump tooling for targeted block-height debugging #5226

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 50 additions & 4 deletions graft/coreth/core/block_validator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,23 @@ package core
import (
"errors"
"fmt"
"os"
"strconv"

"github.com/ava-labs/avalanchego/graft/coreth/consensus"
"github.com/ava-labs/avalanchego/graft/coreth/params"
"github.com/ava-labs/avalanchego/graft/coreth/plugin/evm/upgrade/ap0"
"github.com/ava-labs/avalanchego/graft/evm/firewood"
"github.com/ava-labs/avalanchego/utils/maybe"
"github.com/ava-labs/libevm/core/state"
"github.com/ava-labs/libevm/core/types"
"github.com/ava-labs/libevm/log"
ethparams "github.com/ava-labs/libevm/params"
"github.com/ava-labs/libevm/trie"
)

const dumpOnBlockHeightEnvVar = "DUMP_ON_BLOCK_HEIGHT"

// BlockValidator is responsible for validating block headers, uncles and
// processed state.
//
Expand All @@ -48,18 +55,36 @@ type BlockValidator struct {
config *params.ChainConfig // Chain configuration options
bc *BlockChain // Canonical block chain
engine consensus.Engine // Consensus engine used for validating

dumpOnBlockHeight maybe.Maybe[uint64]
}

// NewBlockValidator returns a new block validator which is safe for re-use
func NewBlockValidator(config *params.ChainConfig, blockchain *BlockChain, engine consensus.Engine) *BlockValidator {
validator := &BlockValidator{
config: config,
engine: engine,
bc: blockchain,
config: config,
engine: engine,
bc: blockchain,
dumpOnBlockHeight: parseDumpOnBlockHeight(),
}
return validator
}

func parseDumpOnBlockHeight() maybe.Maybe[uint64] {
dumpHeightStr, ok := os.LookupEnv(dumpOnBlockHeightEnvVar)
if !ok {
return maybe.Nothing[uint64]()
}

dumpHeight, err := strconv.ParseUint(dumpHeightStr, 10, 64)
if err != nil {
log.Warn("invalid "+dumpOnBlockHeightEnvVar, "value", dumpHeightStr, "err", err)
return maybe.Nothing[uint64]()
}

return maybe.Some(dumpHeight)
}

// ValidateBody validates the given block's uncles and verifies the block
// header's transaction and uncle roots. The headers are assumed to be already
// validated at this point.
Expand Down Expand Up @@ -136,9 +161,30 @@ func (v *BlockValidator) ValidateState(block *types.Block, statedb *state.StateD
if receiptSha != header.ReceiptHash {
return fmt.Errorf("invalid receipt root hash (remote: %x local: %x)", header.ReceiptHash, receiptSha)
}

root := statedb.IntermediateRoot(v.config.IsEIP158(header.Number))
if v.dumpOnBlockHeight.HasValue() && block.NumberU64() == v.dumpOnBlockHeight.Value() {
dumpHeight := v.dumpOnBlockHeight.Value()
backend := statedb.Database().TrieDB().Backend()
if tdb, ok := backend.(*firewood.TrieDB); ok {
proposalDump, err := tdb.DumpProposal(root)
if err != nil {
log.Warn("failed to dump firewood proposal",
"env", dumpOnBlockHeightEnvVar, "height", dumpHeight,
"block", block.NumberU64(), "hash", block.Hash(),
"root", root.Hex(), "err", err)
} else {
fmt.Fprintf(os.Stderr, "BEGIN %s=%d block=%d hash=%s remote_root=%s local_root=%s\n%s\nEND %s\n", dumpOnBlockHeightEnvVar, dumpHeight, block.NumberU64(), block.Hash(), header.Root.Hex(), root.Hex(), proposalDump, dumpOnBlockHeightEnvVar)
}
} else {
log.Warn(dumpOnBlockHeightEnvVar+" set but trie backend is not firewood",
"height", dumpHeight, "backend", fmt.Sprintf("%T", backend))
}
}

// Validate the state root against the received state root and throw
// an error if they don't match.
if root := statedb.IntermediateRoot(v.config.IsEIP158(header.Number)); header.Root != root {
if header.Root != root {
return fmt.Errorf("invalid merkle root (remote: %x local: %x) dberr: %w", header.Root, root, statedb.Error())
}
return nil
Expand Down
67 changes: 67 additions & 0 deletions graft/evm/firewood/triedb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,73 @@ func (t *TrieDB) Commit(root common.Hash, report bool) error {
return nil
}

// DumpProposal returns a textual dump of the tracked Firewood proposal for root.
// This is intended for debugging and forensic analysis.
func (t *TrieDB) DumpProposal(root common.Hash) (string, error) {
t.proposals.Lock()

proposals, ok := t.proposals.byStateRoot[root]
if ok && len(proposals) > 0 {
for _, p := range proposals {
if p.handle != nil {
dump, err := p.handle.Dump()
t.proposals.Unlock()
if err != nil {
return "", err
}
return fmt.Sprintf("# source=byStateRoot root=%s\n%s", root.Hex(), dump), nil
}
}
}

for key, p := range t.possible {
if key.root != root || p == nil || p.handle == nil {
continue
}
dump, err := p.handle.Dump()
t.proposals.Unlock()
if err != nil {
return "", fmt.Errorf("dumping possible proposal for root %s (parent block %s): %w", root.Hex(), key.parentBlockHash.Hex(), err)
}
return fmt.Sprintf("# source=possible parent_block=%s\n%s", key.parentBlockHash.Hex(), dump), nil
}

if t.proposals.tree != nil && t.proposals.tree.handle != nil {
treeRoot := t.proposals.tree.root
dump, err := t.proposals.tree.handle.Dump()
t.proposals.Unlock()
if err != nil {
return "", fmt.Errorf("dumping current tree proposal while root %s was missing: %w", root.Hex(), err)
}
return fmt.Sprintf("# source=tree requested_root=%s current_tree_root=%s\n%s", root.Hex(), treeRoot.Hex(), dump), nil
}

t.proposals.Unlock()

Comment on lines +423 to +460
Copy link

Copilot AI Apr 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

DumpProposal calls p.handle.Dump() (Firewood FFI) while holding t.proposals mutex. If Dump() is slow, this blocks commits/proposals/updates that also need the same lock, undermining the stated goal of not blocking proposal operations during dumps. Consider selecting the handle + any metadata under the lock, releasing the lock, then performing the FFI dump; if concurrent commit/drop is a concern, add a dedicated handle-level lock/refcount so Dump() and Commit()/Drop() can’t race.

Suggested change
proposals, ok := t.proposals.byStateRoot[root]
if ok && len(proposals) > 0 {
for _, p := range proposals {
if p.handle != nil {
dump, err := p.handle.Dump()
t.proposals.Unlock()
if err != nil {
return "", err
}
return fmt.Sprintf("# source=byStateRoot root=%s\n%s", root.Hex(), dump), nil
}
}
}
for key, p := range t.possible {
if key.root != root || p == nil || p.handle == nil {
continue
}
dump, err := p.handle.Dump()
t.proposals.Unlock()
if err != nil {
return "", fmt.Errorf("dumping possible proposal for root %s (parent block %s): %w", root.Hex(), key.parentBlockHash.Hex(), err)
}
return fmt.Sprintf("# source=possible parent_block=%s\n%s", key.parentBlockHash.Hex(), dump), nil
}
if t.proposals.tree != nil && t.proposals.tree.handle != nil {
treeRoot := t.proposals.tree.root
dump, err := t.proposals.tree.handle.Dump()
t.proposals.Unlock()
if err != nil {
return "", fmt.Errorf("dumping current tree proposal while root %s was missing: %w", root.Hex(), err)
}
return fmt.Sprintf("# source=tree requested_root=%s current_tree_root=%s\n%s", root.Hex(), treeRoot.Hex(), dump), nil
}
t.proposals.Unlock()
var (
selectedHandle interface{ Dump() (string, error) }
selectedSource string
selectedParentBlock common.Hash
selectedTreeRoot common.Hash
)
proposals, ok := t.proposals.byStateRoot[root]
if ok && len(proposals) > 0 {
for _, p := range proposals {
if p.handle != nil {
selectedHandle = p.handle
selectedSource = "byStateRoot"
break
}
}
}
if selectedHandle == nil {
for key, p := range t.possible {
if key.root != root || p == nil || p.handle == nil {
continue
}
selectedHandle = p.handle
selectedSource = "possible"
selectedParentBlock = key.parentBlockHash
break
}
}
if selectedHandle == nil && t.proposals.tree != nil && t.proposals.tree.handle != nil {
selectedHandle = t.proposals.tree.handle
selectedSource = "tree"
selectedTreeRoot = t.proposals.tree.root
}
t.proposals.Unlock()
if selectedHandle != nil {
dump, err := selectedHandle.Dump()
if err != nil {
switch selectedSource {
case "possible":
return "", fmt.Errorf("dumping possible proposal for root %s (parent block %s): %w", root.Hex(), selectedParentBlock.Hex(), err)
case "tree":
return "", fmt.Errorf("dumping current tree proposal while root %s was missing: %w", root.Hex(), err)
default:
return "", err
}
}
switch selectedSource {
case "possible":
return fmt.Sprintf("# source=possible parent_block=%s\n%s", selectedParentBlock.Hex(), dump), nil
case "tree":
return fmt.Sprintf("# source=tree requested_root=%s current_tree_root=%s\n%s", root.Hex(), selectedTreeRoot.Hex(), dump), nil
default:
return fmt.Sprintf("# source=byStateRoot root=%s\n%s", root.Hex(), dump), nil
}
}

Copilot uses AI. Check for mistakes.
// The remaining lookups use only Firewood FFI (no proposals state),
// so we don't need to hold the proposals lock.
revision, err := t.Firewood.Revision(ffi.Hash(root))
if err == nil {
defer func() {
if err := revision.Drop(); err != nil {
log.Debug("failed to drop revision after dump", "root", root.Hex(), "err", err)
}
}()
dump, err := revision.Dump()
if err != nil {
return "", fmt.Errorf("dumping persisted revision for root %s: %w", root.Hex(), err)
}
return fmt.Sprintf("# source=revision root=%s\n%s", root.Hex(), dump), nil
}

dump, dumpErr := t.Firewood.Dump()
if dumpErr == nil {
return fmt.Sprintf("# source=database requested_root=%s current_root=%s\n%s", root.Hex(), common.Hash(t.Firewood.Root()).Hex(), dump), nil
}

return "", fmt.Errorf("%w for root %s (no active handle in byStateRoot/possible/tree; revision_err=%v; database_dump_err=%v)", errNoProposalFound, root.Hex(), err, dumpErr)
}

func (ps *proposals) findProposalToCommitWhenLocked(root common.Hash) (*proposal, error) {
var candidate *proposal

Expand Down
2 changes: 1 addition & 1 deletion scripts/benchmark_cchain_range.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -285,7 +285,7 @@ if [[ -n "${CHAOS_MODE:-}" ]]; then
--max-wait-time="${MAX_WAIT_TIME}" \
--config="${CONFIG}"
else
go run github.com/ava-labs/avalanchego/tests/reexecute/c \
go run ./tests/reexecute/c \
--block-dir="${BLOCK_DIR}" \
Comment on lines 287 to 289
Copy link

Copilot AI Apr 11, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

go run ./tests/reexecute/c is resolved relative to the caller’s current working directory, not the script location. If this script is invoked from outside the repo root (e.g., via an absolute path), this will fail to find ./tests/.... Consider cd-ing to the repo root early in the script or using a path derived from SCRIPT_DIR so the benchmark is runnable from any CWD.

Copilot uses AI. Check for mistakes.
--current-state-dir="${CURRENT_STATE_DIR}" \
${RUNNER_TYPE:+--runner="${RUNNER_TYPE}"} \
Expand Down
Loading