From c692bba3d9f44dc8a8393713a964795557c03ca3 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 1 Jul 2026 12:48:57 +0000 Subject: [PATCH 01/18] execution/cache, execution/commitment, db/state: consolidate cache stack on freelru + add persistent code cache Converge the in-memory caches onto freelru.ShardedLRU and fold in the #22120 review findings, then add the persistent (MDBX-backed) code cache. freelru everywhere: - CodeCache content layers (hashToCode/codeHashToCode/codeSizeByCodeHash): maphash.Map -> freelru.ShardedLRU, so a full layer LRU-evicts the coldest entry instead of freezing and refusing new contracts (#22120 finding 1); OnEvict keeps the byte counters honest; keyHash collision + coherence.Gen invalidation preserved. - BranchCache tail: maphash.ShardedLRU -> freelru.ShardedLRU. #22120 review fixes: - putContent over-cap back-out goes through the accounted removal so a racing stale-drop can't double-subtract (finding 3). - account cache entry clamp 1<<22 -> 1<<24 so the configured 1 GB budget is actually reachable instead of capping residency at ~384 MB (finding 2). - trim comment-policy violations in domain_shared.go (finding 8). Persistent code cache (CodeStore): otter in-mem tier over a persistent MDBX TblCodeCache backing, both holding decompressed code keyed by keccak(code). Read-through in stateObject.Code (via the reader's CodeStore()), write-through on the CodeDomain flush at Commit, pruned in the forkchoice prune cycle. Gated by USE_CODE_STORE (default on). otter is the deliberate exception to the freelru-everywhere rule for the in-mem tier. go build, go vet, golangci-lint clean; cache/state/execctx/commitment tests green (incl. a new two-tier CodeStore test); -race clean on the cache package. --- common/dbg/experiments.go | 1 + db/kv/tables.go | 6 + db/state/execctx/domain_shared.go | 140 +++- execution/cache/cache_test.go | 33 +- execution/cache/code_cache.go | 141 ++-- execution/cache/code_cache_codehash_test.go | 12 +- .../cache/code_cache_concurrency_test.go | 21 +- execution/cache/code_store.go | 134 ++++ execution/cache/code_store_test.go | 63 ++ execution/cache/generic_cache.go | 7 +- execution/cache/state_cache.go | 11 +- execution/commitment/adaptive_pin.go | 397 ++++++++++ execution/commitment/branch_cache.go | 544 +++++++++++-- execution/commitment/branch_cache_test.go | 151 ++-- .../commitmentdb/commitment_context.go | 22 + execution/commitment/hex_patricia_hashed.go | 11 +- execution/commitment/preload.go | 175 +++++ execution/commitment/preload_parallel.go | 301 +++++++ execution/commitment/preload_parallel_test.go | 735 ++++++++++++++++++ execution/commitment/preload_ranges.go | 59 ++ execution/commitment/trunk_pin_metrics.go | 44 ++ execution/commitment/warmuper.go | 16 + execution/execmodule/exec_module.go | 45 +- execution/execmodule/forkchoice.go | 7 + execution/execmodule/set_head.go | 1 + .../stagedsync/rawdbreset/reset_stages.go | 11 +- execution/state/rw_v3.go | 13 + execution/state/state_object.go | 20 + go.mod | 1 + go.sum | 2 + 30 files changed, 2845 insertions(+), 279 deletions(-) create mode 100644 execution/cache/code_store.go create mode 100644 execution/cache/code_store_test.go create mode 100644 execution/commitment/adaptive_pin.go create mode 100644 execution/commitment/preload.go create mode 100644 execution/commitment/preload_parallel.go create mode 100644 execution/commitment/preload_parallel_test.go create mode 100644 execution/commitment/preload_ranges.go create mode 100644 execution/commitment/trunk_pin_metrics.go diff --git a/common/dbg/experiments.go b/common/dbg/experiments.go index 6e1abc85475..0906ec55d66 100644 --- a/common/dbg/experiments.go +++ b/common/dbg/experiments.go @@ -114,6 +114,7 @@ var ( CaplinEfficientReorg = EnvBool("CAPLIN_EFFICIENT_REORG", true) UseTxDependencies = EnvBool("USE_TX_DEPENDENCIES", false) UseStateCache = EnvBool("USE_STATE_CACHE", true) + UseCodeStore = EnvBool("USE_CODE_STORE", true) AssertStateCache = EnvBool("ASSERT_STATE_CACHE", false) ReadAhead = EnvBool("READ_AHEAD", true) diff --git a/db/kv/tables.go b/db/kv/tables.go index c36f63ae048..c05d7a5c1c2 100644 --- a/db/kv/tables.go +++ b/db/kv/tables.go @@ -156,6 +156,11 @@ const ( TblCodeHistoryVals = "CodeHistoryVals" TblCodeIdx = "CodeIdx" + // TblCodeCache holds decompressed contract code keyed by keccak(code), the + // persistent backing tier for the in-memory code cache so reads skip the + // CodeDomain decompression across restarts. Immutable (content-addressed). + TblCodeCache = "CodeCache" + TblCommitmentVals = "CommitmentVals" TblCommitmentHistoryKeys = "CommitmentHistoryKeys" TblCommitmentHistoryVals = "CommitmentHistoryVals" @@ -362,6 +367,7 @@ var ChaindataTables = []string{ TblCodeHistoryKeys, TblCodeHistoryVals, TblCodeIdx, + TblCodeCache, TblCommitmentVals, TblCommitmentHistoryKeys, diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 68eaffc3ea7..5b1bde824af 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -129,14 +129,15 @@ type SharedDomains struct { // stateCache is an optional cache for state data (accounts, storage, code) stateCache *cache.StateCache + // codeStore is the optional two-tier (in-mem + MDBX) codehash-keyed code + // cache, reached via temporalGetter so an addr-keyed reader can serve a + // code-by-hash read with the application's authoritative codehash. + codeStore *cache.CodeStore + // changesetMu serializes the parallel commitment calculator's swap of the - // global current-changeset-accumulator pointer against DomainPut/DomainDel. - // Without it, account/storage writes for block N+1 can land in block N's - // changeset during the calculator's swap+compute+restore window, so a later - // unwind reads stale prev-values and computes wrong roots (reorg/fork tests). - // A band-aid for execution being coupled to unwind-side accumulator - // machinery; removable once per-block changesets are derived post-hoc from - // the (now tx-granular) sd entries at Flush time. + // global current-changeset-accumulator pointer against DomainPut/DomainDel: + // without it a block N+1 write can land in block N's changeset during the + // swap+compute+restore window, so a later unwind reads stale prev-values. changesetMu sync.Mutex // branchCache is the aggregator-scope commitment-branch cache. It sits @@ -161,6 +162,12 @@ type SharedDomains struct { // pass their own per-worker instance via AsGetterMetered. reqMetrics *kvmetrics.DomainMetrics reqSource kvmetrics.Source + + // adaptivePinController decides which contracts get pinned based on observed + // miss pressure. nil when branchCache is nil or the adaptive layer is disabled. + // Its miss callback is wired via Bind in EnableParaTrieDB; OnBlockComplete fires + // from Commit using the in-flight (pre-Commit) tx. + adaptivePinController *commitment.AdaptivePinController } // PickTrieVariant returns the commitment trie variant selected by the @@ -213,6 +220,14 @@ func NewSharedDomains(ctx context.Context, tx kv.TemporalTx, logger log.Logger, } sd.sdCtx = commitmentdb.NewSharedDomainsCommitmentContext(sd, commitment.ModeDirect, tx.Debug().Dirs().Tmp, trieCfg) + if branchCache != nil && !dbg.EnvBool("DISABLE_ADAPTIVE_PIN", false) { + sd.adaptivePinController = commitment.NewAdaptivePinController( + branchCache, + commitment.DefaultAdaptivePinControllerConfig(), + logger, + ) + } + _, blockNum, err := sd.SeekCommitment(ctx, tx) if err != nil { return sd, err @@ -393,15 +408,8 @@ func (sd *SharedDomains) flushPendingUpdates(ctx context.Context, tx kv.Temporal } // domainPutNoLock is the lock-held variant of DomainPut for callers -// (FlushPendingUpdates) that already hold changesetMu externally. It -// shares DomainPut's body via domainPut(..., lockHeld=true). -// -// Today DomainPut(kv.CommitmentDomain, ...) happens to skip the lock -// anyway (see the CommitmentDomain exemption in domainPut), so calling -// DomainPut directly from FlushPendingUpdates wouldn't deadlock on the -// current code path. This variant is defensive: it stays correct even if -// a future change removes that exemption (e.g. the lock-free refactor in -// #21106 reshapes how CommitmentDomain writes are routed). +// (FlushPendingUpdates) that already hold changesetMu externally; it stays +// correct even if the CommitmentDomain lock exemption in domainPut is removed. func (sd *SharedDomains) domainPutNoLock(domain kv.Domain, roTx kv.TemporalTx, k, v []byte, txNum uint64, prevVal []byte) error { return sd.domainPut(domain, roTx, k, v, txNum, prevVal, true) } @@ -748,6 +756,17 @@ func (sd *SharedDomains) SetStateCache(stateCache *cache.StateCache) { sd.stateCache = stateCache } +// SetCodeStore sets the persistent codehash-keyed code cache. +func (sd *SharedDomains) SetCodeStore(codeStore *cache.CodeStore) { + sd.codeStore = codeStore +} + +// CodeStore exposes the code store + the backing tx so an addr-keyed reader can +// serve a code-by-hash read using the application's authoritative codehash. +func (tg *temporalGetter) CodeStore() (*cache.CodeStore, kv.TemporalTx) { + return tg.sd.codeStore, tg.tx +} + // PrintCacheStats logs the state cache hit/miss counters and resets them. // No-op when the cache is disabled. The cache is an SD-internal detail, so // callers observe it through SD rather than reaching for the cache directly. @@ -911,7 +930,7 @@ type cacheUpdate struct { // the SD's internal caches after committing. Entries are stamped with the value's // per-key write txNum (delivered by the callback) as the unwind floor, so // invalidation is tx-precise: an unwind to a txNum inside the latest step drops -// exactly the entries above it, not the whole step (#21752). All caches honor the +// exactly the entries above it, not the whole step. All caches honor the // same (txNum, epoch) model. tx MUST be a flush-specific transaction: it is // committed here. func (sd *SharedDomains) Commit(ctx context.Context, tx kv.RwTx, validate ...func(tx kv.RwTx) error) error { @@ -929,7 +948,7 @@ func (sd *SharedDomains) Commit(ctx context.Context, tx kv.RwTx, validate ...fun return nil } - if sd.branchCache == nil && sd.stateCache == nil { + if sd.branchCache == nil && sd.stateCache == nil && sd.codeStore == nil { if err := sd.flushMem(ctx, tx); err != nil { return err } @@ -959,7 +978,25 @@ func (sd *SharedDomains) Commit(ctx context.Context, tx kv.RwTx, validate ...fun opts = append(opts, stash(kv.CommitmentDomain)) } if sd.stateCache != nil { - opts = append(opts, stash(kv.AccountsDomain), stash(kv.StorageDomain), stash(kv.CodeDomain)) + opts = append(opts, stash(kv.AccountsDomain), stash(kv.StorageDomain)) + } + // CodeDomain flush both populates the persistent code store (write path, has + // an RwTx) and stashes for the in-mem state cache. + if sd.stateCache != nil || sd.codeStore != nil { + opts = append(opts, kv.WithFlushCallback(kv.CodeDomain, func(k []byte, v []byte, step kv.Step, txNum uint64) { + if sd.codeStore != nil && len(v) > 0 { + _ = sd.codeStore.PutByHash(tx, crypto.Keccak256(v), v) + } + if sd.stateCache != nil { + pending = append(pending, cacheUpdate{ + domain: kv.CodeDomain, + key: append([]byte(nil), k...), + val: append([]byte(nil), v...), + step: step, + txN: txNum, + }) + } + })) } if err := sd.flushMem(ctx, tx, opts...); err != nil { return err @@ -967,6 +1004,64 @@ func (sd *SharedDomains) Commit(ctx context.Context, tx kv.RwTx, validate ...fun if err := runValidate(); err != nil { return err } + // Adaptive controller hook: decide promotions/demotions from this batch's + // miss pressure using the in-flight (pre-Commit) tx so commitment reads see + // the just-flushed bytes. Runs before Commit because the tx is finalized + // after; the coherence floor evicts pins from a rolled-back batch. + if sd.adaptivePinController != nil { + if ttx, ok := tx.(kv.TemporalTx); ok { + reader := func(prefix []byte) ([]byte, uint64, bool, error) { + v, step, err := ttx.GetLatest(kv.CommitmentDomain, prefix) + if err != nil { + return nil, 0, false, err + } + return v, uint64(step), len(v) > 0, nil + } + factory := func() (commitment.BatchBranchResolver, func(), error) { + resolve := func(keys [][]byte) ([][]byte, error) { + d := ttx.Debug() + vals := make([][]byte, len(keys)) + for i, k := range keys { + v, found, _, _, err := d.GetLatestFromFiles(kv.CommitmentDomain, k, 0) + if err != nil { + return nil, err + } + if found { + vals[i] = common.Copy(v) + } + } + return vals, nil + } + return resolve, nil, nil + } + provider := func(contractHash []byte) map[string][]byte { + m := map[string][]byte{} + c, cerr := ttx.CursorDupSort(kv.TblCommitmentVals) + if cerr != nil { + return m + } + defer c.Close() + evenFrom, evenTo, oddFrom, oddTo := commitment.ContractTrunkKeyRanges(commitment.ContractNibbles(contractHash)) + scan := func(from, to []byte) { + for k, v, err := c.Seek(from); k != nil && err == nil; k, v, err = c.NextNoDup() { + if bytes.Compare(k, to) >= 0 { + return + } + if len(v) < 8 { + continue + } + m[string(common.Copy(k))] = common.Copy(v[8:]) + } + } + scan(evenFrom, evenTo) + scan(oddFrom, oddTo) + return m + } + sd.adaptivePinController.SetParallelMode(factory, provider) + sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader) + sd.adaptivePinController.SetParallelMode(nil, nil) + } + } if err := tx.Commit(); err != nil { return err } @@ -1231,7 +1326,7 @@ func (sd *SharedDomains) getLatestMetered(domain kv.Domain, tx kv.TemporalTx, k } // GetCodeSize returns the length of the contract code at addr, probing a -// size-only cache (geth-style codeSizeCache) before falling through to the +// size-only cache before falling through to the // full bytes path. For workloads dominated by EXTCODESIZE / EXTCODEHASH // this avoids the file-accessor + decompression cost of the full bytes on // the second-and-later access to any codeHash seen anywhere in the process. @@ -1354,7 +1449,7 @@ func (sd *SharedDomains) codeHashForAddr(tx kv.TemporalTx, addr []byte, txNum ui } } - // Below mem: the Nethermind-style addr → codeHash LRU caches committed state + // Below mem: the addr → codeHash LRU caches committed state // (flush-invalidated). The zero-hash sentinel means "no code / missing // account" (negative cache). if sd.stateCache != nil { @@ -1718,6 +1813,9 @@ func (sd *SharedDomains) EnableTrieWarmup(trieWarmup bool) { func (sd *SharedDomains) EnableParaTrieDB(db kv.TemporalRoDB) { sd.sdCtx.EnableParaTrieDB(db) + if sd.adaptivePinController != nil { + sd.adaptivePinController.Bind() + } } // SetDeferCommitmentUpdates enables or disables deferred commitment updates. diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index 4587aa75440..1fb00f30165 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -306,9 +306,10 @@ func TestCodeCache_AddrCapacityLimit(t *testing.T) { assert.True(t, ok, "most recent entry should remain") assert.Equal(t, wideCode(1099), v) - // hashToCode stores all 1100 distinct codes (content-addressed, - // independent of addr LRU eviction). - assert.Equal(t, 1100, c.CodeLen()) + // hashToCode now LRU-evicts at its own entry cap (codeCapacityB / + // avgCodeEntryBytes), so it holds far fewer than the 1100 distinct codes + // rather than growing unbounded. + assert.Less(t, c.CodeLen(), 1100) // Updating an existing addr re-writes the entry (LRU promotes to MRU). c.Put(wideAddr(1099), wideCode(4242), 0) @@ -318,23 +319,25 @@ func TestCodeCache_AddrCapacityLimit(t *testing.T) { } func TestCodeCache_CodeCapacityLimit(t *testing.T) { - // Each code entry is 8 (hash) + 3 (code bytes) = 11 bytes - // Set code capacity to 25 bytes - enough for 2 entries but not 3 + // Tiny byte budget → a 1-entry code layer cap. Successive distinct codes + // LRU-evict the coldest rather than freezing the layer. c := NewCodeCache(25, 1024*1024) // 25 bytes code, 1MB addr - // Fill code capacity c.Put(makeAddr(1), makeCode(1), 0) c.Put(makeAddr(2), makeCode(2), 0) - assert.Equal(t, 2, c.CodeLen()) - - // Try to add more code - addr mapping added, but code not stored c.Put(makeAddr(3), makeCode(3), 0) - assert.Equal(t, 3, c.Len()) // addr mapping added - assert.Equal(t, 2, c.CodeLen()) // code not added (at capacity) - // Get for addr3 should fail (code not in cache) - _, ok := c.Get(makeAddr(3)) - assert.False(t, ok) + // Addr LRU keeps all three mappings (1MB); the code layer holds only the + // most-recent code(s) after eviction. + assert.Equal(t, 3, c.Len()) + assert.LessOrEqual(t, c.CodeLen(), 1) + + // Newest code is retrievable; the coldest was evicted from the code layer. + v, ok := c.Get(makeAddr(3)) + assert.True(t, ok) + assert.Equal(t, makeCode(3), v) + _, ok = c.Get(makeAddr(1)) + assert.False(t, ok, "coldest code should have been evicted") } func TestCodeCache_Delete(t *testing.T) { @@ -392,7 +395,7 @@ func TestCodeCache_GetMissingCode(t *testing.T) { c.Put(addr, code, 0) // Clear the code cache but keep addr mapping - c.hashToCode.Clear() + c.hashToCode.Purge() c.codeSize.Store(0) // Get should fail at code lookup stage diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index f3bca5ae7c7..78f987610d2 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -21,6 +21,7 @@ import ( "unsafe" "github.com/c2h5oh/datasize" + "github.com/elastic/go-freelru" lru "github.com/hashicorp/golang-lru/v2" "github.com/erigontech/erigon/common" @@ -29,11 +30,6 @@ import ( "github.com/erigontech/erigon/execution/cache/coherence" ) -// uint64AsBytes returns a []byte view of a uint64 without allocation. -func uint64AsBytes(v *uint64) []byte { - return unsafe.Slice((*byte)(unsafe.Pointer(v)), 8) -} - // hash32 copies a codeHash slice into a fixed [32]byte for storage/compare. func hash32(b []byte) [32]byte { var h [32]byte @@ -50,6 +46,11 @@ const ( // cache (code size answers without loading bytes for // EXTCODESIZE / EXTCODEHASH callers). DefaultCodeSizeCacheEntries int64 = 1_000_000 + // avgCodeEntryBytes translates the code byte budget into the freelru + // entry-count cap. Contract bytecode varies widely (a few bytes to 24 KB); + // the persistent (MDBX-backed) cold tier backstops entries evicted from this + // hot tier, so a loose byte bound here is acceptable. + avgCodeEntryBytes = 4096 ) // CodeCache is a multi-level concurrent cache for contract code, keyed by the @@ -119,8 +120,8 @@ type CodeCache struct { // codeID for the code at that address. An LRU so fresh-address workloads // evict oldest entries and warm up the working set. addrToHash *lru.Cache[common.Address, versionedAddressID] - hashToCode *maphash.Map[codeEntry] // maphash(code) → code, concurrent - codeSize atomic.Int64 // current size in bytes (code only, hash is fixed 8 bytes) + hashToCode *freelru.ShardedLRU[uint64, codeEntry] // codeID(maphash(code)) → code, LRU-evicting + codeSize atomic.Int64 // resident bytes (stat; hard bound is the entry cap) // addrToCodeHash maps a 20-byte address to its 32-byte Ethereum codeHash // (keccak), separately from addrToHash (which uses the cheap maphash @@ -134,14 +135,14 @@ type CodeCache struct { // of L1 — Get-by-codeHash bypasses addr lookup entirely. Memory cost: // duplicates code bytes vs L2 (worst case 2x byte storage); accepted // for the per-key fast-path on many-addrs-one-code workloads. - codeHashToCode *maphash.Map[codeEntry] // keccak(code) → code, concurrent - codeHashCodeSize atomic.Int64 // current size in bytes (codeHash layer) + codeHashToCode *freelru.ShardedLRU[uint64, codeEntry] // keccak(code) → code, LRU-evicting + codeHashCodeSize atomic.Int64 // resident bytes (stat; hard bound is the entry cap) // Size-only layer: ethCodeHash → int (length in bytes). Answers // EXTCODESIZE / EXTCODEHASH without loading the bytes. Tiny per-entry // footprint (32B key + 8B value) so the same memory budget gives ~1000x // the hit surface vs the bytes cache. - codeSizeByCodeHash *maphash.Map[codeSizeEntry] + codeSizeByCodeHash *freelru.ShardedLRU[uint64, codeSizeEntry] codeSizeEntries atomic.Int64 codeSizeCapEntries int64 @@ -171,44 +172,33 @@ func (c *CodeCache) isStale(txNum uint64, epoch uint32) bool { return c.coh.IsStale(txNum, epoch) } -// putAccounted is the shared insert path for the content-addressed code layers -// (hashToCode, codeHashToCode, codeSizeByCodeHash). Each is a maphash.Map of -// per-key-immutable entries carrying a (txNum, epoch) stamp, and the bookkeeping -// is identical across all three: skip a live entry (its bytes/size are invariant -// for a given key), drop a stale one through the accounted LoadAndDelete (the -// same primitive stale-Get eviction uses, so a racing eviction can't -// double-subtract), refuse the insert once the layer is full, and back it out if -// it raced past the cap so concurrent distinct inserts can't overshoot. An entry -// costs keyCost + valCost(e); capacity bounds counter. stamp/valCost are +// putContent is the shared insert path for the content-addressed code layers +// (hashToCode, codeHashToCode, codeSizeByCodeHash). Each is a freelru.ShardedLRU +// of per-key-immutable entries carrying a (txNum, epoch) stamp: a live entry is +// kept (its bytes/size are invariant for a given key), a stale one is removed +// (its OnEvict decrements counter) so the fresh entry can replace it, and once +// the entry-count cap is reached freelru.Add evicts the coldest entry (whose +// OnEvict decrements counter) rather than freezing. counter tracks resident +// bytes as a stat; the hard bound is the LRU's entry cap. stamp/valCost are // non-capturing so passing them allocates nothing on the put path. -func putAccounted[T any]( - m *maphash.Map[T], - key []byte, +func putContent[T any]( + lru *freelru.ShardedLRU[uint64, T], + h uint64, newEntry T, stamp func(T) (uint64, uint32), valCost func(T) int64, coh *coherence.Gen, counter *atomic.Int64, - keyCost, capacity int64, + keyCost int64, ) { - cost := keyCost + valCost(newEntry) - if existing, exists := m.Get(key); exists { + if existing, ok := lru.Get(h); ok { if txNum, epoch := stamp(existing); !coh.IsStale(txNum, epoch) { return } - if old, removed := m.LoadAndDelete(key); removed { - counter.Add(-(keyCost + valCost(old))) - } - } - if counter.Load()+cost > capacity { - return - } - if _, loaded := m.LoadOrStore(key, newEntry); !loaded { - if counter.Add(cost) > capacity { - counter.Add(-cost) - m.Delete(key) - } + lru.Remove(h) // stale — OnEvict decrements counter for the removed entry } + counter.Add(keyCost + valCost(newEntry)) + lru.Add(h, newEntry) // evicts the coldest entry when full; its OnEvict decrements counter } func codeEntryStamp(e codeEntry) (uint64, uint32) { return e.txNum, e.epoch } @@ -233,16 +223,41 @@ func NewCodeCache(codeCapacityBytes, addrCapacityBytes datasize.ByteSize) *CodeC if err != nil { panic(err) } + // Byte budget → entry-count cap for the two bytes layers; the size-only + // layer is entry-counted directly. Floor at 1 so tiny (test) budgets still + // construct a valid, evicting LRU. + codeEntries := uint32(uint64(codeCapacityBytes) / avgCodeEntryBytes) + if codeEntries < 1 { + codeEntries = 1 + } + hashToCode, err := freelru.NewSharded[uint64, codeEntry](codeEntries, u64identity) + if err != nil { + panic(err) + } + codeHashToCode, err := freelru.NewSharded[uint64, codeEntry](codeEntries, u64identity) + if err != nil { + panic(err) + } + sizeEntries := uint32(DefaultCodeSizeCacheEntries) + codeSizeByCodeHash, err := freelru.NewSharded[uint64, codeSizeEntry](sizeEntries, u64identity) + if err != nil { + panic(err) + } cc := &CodeCache{ addrToHash: addrLRU, addrToCodeHash: addrCodeHashLRU, - hashToCode: maphash.NewMap[codeEntry](), - codeHashToCode: maphash.NewMap[codeEntry](), - codeSizeByCodeHash: maphash.NewMap[codeSizeEntry](), + hashToCode: hashToCode, + codeHashToCode: codeHashToCode, + codeSizeByCodeHash: codeSizeByCodeHash, codeSizeCapEntries: DefaultCodeSizeCacheEntries, addrCapacityB: addrCapacityBytes, codeCapacityB: codeCapacityBytes, } + // OnEvict fires on capacity-driven LRU eviction and on explicit Remove, so + // the byte/entry counters follow residency without a separate scan. + hashToCode.SetOnEvict(func(_ uint64, e codeEntry) { cc.codeSize.Add(-(8 + int64(len(e.code)))) }) + codeHashToCode.SetOnEvict(func(_ uint64, e codeEntry) { cc.codeHashCodeSize.Add(-(32 + int64(len(e.code)))) }) + codeSizeByCodeHash.SetOnEvict(func(_ uint64, _ codeSizeEntry) { cc.codeSizeEntries.Add(-1) }) // Before any unwind every entry's txNum is below the floor, so the epoch // check never strands a valid entry. cc.coh.Init() @@ -276,18 +291,13 @@ func (c *CodeCache) GetWithTxNum(addr []byte) ([]byte, uint64, bool) { } c.addrHits.Add(1) - hashKey := uint64AsBytes(&vID.addrID) - ce, ok := c.hashToCode.Get(hashKey) + ce, ok := c.hashToCode.Get(vID.addrID) if !ok || len(ce.code) == 0 { c.codeMisses.Add(1) return nil, 0, false } if c.isStale(ce.txNum, ce.epoch) { - // Only the goroutine that actually removes the entry adjusts the byte - // counter, so concurrent stale readers can't double-subtract. - if old, removed := c.hashToCode.LoadAndDelete(hashKey); removed { - c.codeSize.Add(-int64(8 + len(old.code))) - } + c.hashToCode.Remove(vID.addrID) // OnEvict decrements codeSize c.codeMisses.Add(1) return nil, 0, false } @@ -327,11 +337,10 @@ func (c *CodeCache) putCode(addr []byte, code []byte, keyHash [32]byte, txNum ui c.addrToHash.Add(common.BytesToAddress(addr), versionedAddressID{addrID: codeID, codeHash: keyHash, txNum: txNum, epoch: ep}) - hashKey := uint64AsBytes(&codeID) entry := codeEntry{code: code, keyHash: keyHash, txNum: txNum, epoch: ep} - // 8-byte maphash key + code bytes. - putAccounted(c.hashToCode, hashKey, entry, codeEntryStamp, codeEntryCodeLen, - &c.coh, &c.codeSize, 8, int64(c.codeCapacityB)) + // freelru keyed by the codeID (maphash of code) directly; 8-byte key cost. + putContent(c.hashToCode, codeID, entry, codeEntryStamp, codeEntryCodeLen, + &c.coh, &c.codeSize, 8) } // GetAddrCodeHash returns the Ethereum codeHash for addr if cached. Lets @@ -374,7 +383,8 @@ func (c *CodeCache) DeleteAddrCodeHash(addr []byte) { // after account-load). Many addresses sharing one codeHash all hit this // single codeHashToCode entry after the first population. func (c *CodeCache) GetByCodeHash(codeHash []byte) ([]byte, bool) { - ce, ok := c.codeHashToCode.Get(codeHash) + h := maphash.Hash(codeHash) + ce, ok := c.codeHashToCode.Get(h) if !ok || len(ce.code) == 0 { c.codeHashMisses.Add(1) return nil, false @@ -386,9 +396,7 @@ func (c *CodeCache) GetByCodeHash(codeHash []byte) ([]byte, bool) { return nil, false } if c.isStale(ce.txNum, ce.epoch) { - if old, removed := c.codeHashToCode.LoadAndDelete(codeHash); removed { - c.codeHashCodeSize.Add(-int64(len(codeHash) + len(old.code))) - } + c.codeHashToCode.Remove(h) // OnEvict decrements codeHashCodeSize c.codeHashMisses.Add(1) return nil, false } @@ -420,9 +428,9 @@ func (c *CodeCache) PutWithCodeHash(addr []byte, code []byte, codeHash []byte, t c.PutCodeSizeByCodeHash(codeHash, len(code), txNum) entry := codeEntry{code: code, keyHash: kh, txNum: txNum, epoch: ep} - // 32-byte codeHash key + code bytes. - putAccounted(c.codeHashToCode, codeHash, entry, codeEntryStamp, codeEntryCodeLen, - &c.coh, &c.codeHashCodeSize, int64(len(codeHash)), int64(c.codeCapacityB)) + // freelru keyed by maphash(codeHash); 32-byte key cost. + putContent(c.codeHashToCode, maphash.Hash(codeHash), entry, codeEntryStamp, codeEntryCodeLen, + &c.coh, &c.codeHashCodeSize, int64(len(codeHash))) } // GetCodeSizeByCodeHash retrieves the size (in bytes) of a contract by its @@ -432,7 +440,8 @@ func (c *CodeCache) PutWithCodeHash(addr []byte, code []byte, codeHash []byte, t // cache hit the caller answers a 4-instruction map probe instead of paying // the file-accessor + decompression stack for the full bytes. func (c *CodeCache) GetCodeSizeByCodeHash(codeHash []byte) (int, bool) { - e, ok := c.codeSizeByCodeHash.Get(codeHash) + h := maphash.Hash(codeHash) + e, ok := c.codeSizeByCodeHash.Get(h) if !ok { c.codeSizeMisses.Add(1) return 0, false @@ -443,9 +452,7 @@ func (c *CodeCache) GetCodeSizeByCodeHash(codeHash []byte) (int, bool) { return 0, false } if c.isStale(e.txNum, e.epoch) { - if _, removed := c.codeSizeByCodeHash.LoadAndDelete(codeHash); removed { - c.codeSizeEntries.Add(-1) - } + c.codeSizeByCodeHash.Remove(h) // OnEvict decrements codeSizeEntries c.codeSizeMisses.Add(1) return 0, false } @@ -464,8 +471,8 @@ func (c *CodeCache) PutCodeSizeByCodeHash(codeHash []byte, size int, txNum uint6 kh := hash32(codeHash) entry := codeSizeEntry{size: size, keyHash: kh, txNum: txNum, epoch: ep} // Entry-counted layer: each entry costs 1 against the entry cap. - putAccounted(c.codeSizeByCodeHash, codeHash, entry, codeSizeEntryStamp, zeroCost, - &c.coh, &c.codeSizeEntries, 1, c.codeSizeCapEntries) + putContent(c.codeSizeByCodeHash, maphash.Hash(codeHash), entry, codeSizeEntryStamp, zeroCost, + &c.coh, &c.codeSizeEntries, 1) } // Delete removes the address → code mapping for addr. @@ -478,9 +485,9 @@ func (c *CodeCache) Delete(addr []byte) { func (c *CodeCache) Clear() { c.addrToHash.Purge() c.addrToCodeHash.Purge() - c.hashToCode.Clear() - c.codeHashToCode.Clear() - c.codeSizeByCodeHash.Clear() + c.hashToCode.Purge() + c.codeHashToCode.Purge() + c.codeSizeByCodeHash.Purge() c.codeSize.Store(0) c.codeHashCodeSize.Store(0) c.codeSizeEntries.Store(0) diff --git a/execution/cache/code_cache_codehash_test.go b/execution/cache/code_cache_codehash_test.go index 873509ca86e..ccd416fa044 100644 --- a/execution/cache/code_cache_codehash_test.go +++ b/execution/cache/code_cache_codehash_test.go @@ -97,16 +97,18 @@ func TestCodeCache_PutWithCodeHash_EmptyHashOrCodeIsNoOp(t *testing.T) { require.Nil(t, v) } -func TestCodeCache_PutWithCodeHash_RespectsCodeCapacity(t *testing.T) { - // 8-byte cap: 32-byte codeHash + 4-byte code > 32. New codeHashToCode puts must - // no-op when the layer is full. Use tiny code to keep math obvious. +func TestCodeCache_PutWithCodeHash_EvictsColdestWhenFull(t *testing.T) { + // Tiny byte budget → a 1-entry freelru cap. The second put must EVICT the + // coldest entry (LRU), not freeze the layer: the newest code is retrievable + // and the oldest is gone. c := NewCodeCache(8, 1*datasize.MB) c.PutWithCodeHash(makeAddr(1), []byte{1, 2, 3, 4}, makeCodeHash(1), 0) - // Second put exceeds the codeHashToCode budget — must no-op. c.PutWithCodeHash(makeAddr(2), []byte{5, 6, 7, 8}, makeCodeHash(2), 0) _, ok := c.GetByCodeHash(makeCodeHash(2)) - assert.False(t, ok, "second codeHashToCode entry should not exist when capacity is exceeded") + assert.True(t, ok, "newest codeHashToCode entry must be present after eviction (no freeze)") + _, ok = c.GetByCodeHash(makeCodeHash(1)) + assert.False(t, ok, "coldest codeHashToCode entry must have been evicted") } func TestCodeCache_CodeSize_PopulatedAlongsideBytes(t *testing.T) { diff --git a/execution/cache/code_cache_concurrency_test.go b/execution/cache/code_cache_concurrency_test.go index 8d28219694c..ac8c67c5338 100644 --- a/execution/cache/code_cache_concurrency_test.go +++ b/execution/cache/code_cache_concurrency_test.go @@ -24,6 +24,7 @@ import ( "github.com/stretchr/testify/require" "github.com/erigontech/erigon/common/crypto" + "github.com/erigontech/erigon/common/maphash" ) // TestCodeCache_ConcurrentPutSameCode_NoSizeDrift guards against the size @@ -83,19 +84,17 @@ func TestCodeCache_ByteCheckRejectsForeignKeyHash(t *testing.T) { foreign := make([]byte, 32) copy(foreign, realHash) foreign[0] ^= 0xff // different 32-byte key - cc.codeHashToCode.Set(foreign, codeEntry{code: code, keyHash: hash32(realHash), txNum: 1, epoch: cc.coh.Epoch()}) + cc.codeHashToCode.Add(maphash.Hash(foreign), codeEntry{code: code, keyHash: hash32(realHash), txNum: 1, epoch: cc.coh.Epoch()}) // The stored entry's keyHash is realHash, not foreign — Get must reject it. _, ok = cc.GetByCodeHash(foreign) require.False(t, ok, "byte-check must reject an entry whose keyHash differs from the requested codeHash") } -// TestCodeCache_ConcurrentDistinctPuts_RespectCap exercises the back-out branch -// of the shared insert path: many workers Put distinct codes whose combined size -// far exceeds the byte cap. Each insert that races past the cap must subtract its -// own cost and drop its entry, so after the dust settles the byte counters never -// exceed the cap and stay non-negative — the bound holds under concurrency, not -// just serial inserts. +// TestCodeCache_ConcurrentDistinctPuts_RespectCap drives many workers putting +// distinct codes whose combined size far exceeds a tiny cap. The freelru layer +// evicts the coldest entries to stay within its entry cap (no freeze), and the +// OnEvict-maintained byte counter must never drift negative under concurrency. func TestCodeCache_ConcurrentDistinctPuts_RespectCap(t *testing.T) { const codeCap = 4 * datasize.KB cc := NewCodeCache(codeCap, 16*datasize.MB) @@ -113,8 +112,10 @@ func TestCodeCache_ConcurrentDistinctPuts_RespectCap(t *testing.T) { } wg.Wait() - require.LessOrEqual(t, cc.codeHashCodeSize.Load(), int64(codeCap), - "codeHashToCode must never exceed the byte cap after concurrent distinct Puts") + // The entry cap (codeCap/avgCodeEntryBytes) is the hard bound; residency + // settled far below the 128 distinct puts rather than freezing at the first. + require.Less(t, cc.codeHashToCode.Len(), workers, + "freelru must evict to its entry cap, not hold all 128 distinct codes") require.GreaterOrEqual(t, cc.codeHashCodeSize.Load(), int64(0), - "codeHashToCode size must stay non-negative (no double back-out)") + "byte counter must stay non-negative (OnEvict accounting must not double-subtract)") } diff --git a/execution/cache/code_store.go b/execution/cache/code_store.go new file mode 100644 index 00000000000..92ca1bea951 --- /dev/null +++ b/execution/cache/code_store.go @@ -0,0 +1,134 @@ +package cache + +import ( + "sync/atomic" + + "github.com/maypok86/otter/v2" + + "github.com/erigontech/erigon/db/kv" +) + +// CodeStore is a two-tier code cache keyed by keccak(code): an in-memory otter +// tier over a persistent MDBX TblCodeCache backing, both holding DECOMPRESSED +// code so a hit skips the CodeDomain btree+decompress cost. Content-addressed, +// so entries are immutable and callers must key by the authoritative account +// codehash — a wrong codehash can only miss, never return wrong bytes. +type CodeStore struct { + mem *otter.Cache[[32]byte, []byte] + + // tableSizeBytes is an in-memory approximation (not rescanned at startup); + // Evict prunes the MDBX tier in cursor order, safe since RoTx reads preclude + // LRU and entries re-derive from CodeDomain. + tableCapBytes uint64 + tableSizeBytes atomic.Int64 + + memHits atomic.Uint64 + tableHits atomic.Uint64 + misses atomic.Uint64 +} + +// Stats returns and resets the (memHits, tableHits, misses) counters. +func (s *CodeStore) Stats() (memHits, tableHits, misses uint64) { + if s == nil { + return 0, 0, 0 + } + return s.memHits.Swap(0), s.tableHits.Swap(0), s.misses.Swap(0) +} + +const ( + DefaultCodeStoreMemBytes = 256 * 1024 * 1024 + DefaultCodeStoreTableBytes = 1024 * 1024 * 1024 +) + +func NewCodeStore(memCapBytes, tableCapBytes uint64) *CodeStore { + mem := otter.Must(&otter.Options[[32]byte, []byte]{ + MaximumWeight: memCapBytes, + Weigher: func(_ [32]byte, code []byte) uint32 { return uint32(len(code)) }, + }) + return &CodeStore{mem: mem, tableCapBytes: tableCapBytes} +} + +// GetByHash returns decompressed code for codehash, checking the in-memory tier +// then the MDBX backing (populating the in-memory tier on a backing hit). A miss +// means the caller must fall through to the authoritative CodeDomain read. +func (s *CodeStore) GetByHash(tx kv.Getter, codeHash []byte) ([]byte, bool) { + if s == nil || len(codeHash) != 32 { + return nil, false + } + var key [32]byte + copy(key[:], codeHash) + if code, ok := s.mem.GetIfPresent(key); ok { + s.memHits.Add(1) + return code, true + } + code, err := tx.GetOne(kv.TblCodeCache, codeHash) + if err != nil || len(code) == 0 { + s.misses.Add(1) + return nil, false + } + s.mem.Set(key, code) + s.tableHits.Add(1) + return code, true +} + +// PutByHash records decompressed code in both tiers. The MDBX write needs an +// RwTx, so this runs on the code write path (deploy/commit), not on reads. +func (s *CodeStore) PutByHash(tx kv.RwTx, codeHash, code []byte) error { + if s == nil || len(codeHash) != 32 || len(code) == 0 { + return nil + } + var key [32]byte + copy(key[:], codeHash) + s.mem.Set(key, code) + has, err := tx.Has(kv.TblCodeCache, codeHash) + if err != nil { + return err + } + if err := tx.Put(kv.TblCodeCache, codeHash, code); err != nil { + return err + } + if !has { + s.tableSizeBytes.Add(int64(len(codeHash) + len(code))) + } + return nil +} + +// Evict prunes the MDBX backing to ~90% of tableCapBytes in cursor (codehash) +// order when over capacity. Safe: evicted entries are re-derivable from +// CodeDomain on miss. Call on a write tx (e.g., at commit), never on reads. +func (s *CodeStore) Evict(tx kv.RwTx) error { + if s == nil || s.tableCapBytes == 0 || uint64(s.tableSizeBytes.Load()) <= s.tableCapBytes { + return nil + } + c, err := tx.RwCursor(kv.TblCodeCache) + if err != nil { + return err + } + defer c.Close() + target := int64(s.tableCapBytes / 10 * 9) + for s.tableSizeBytes.Load() > target { + k, v, err := c.Next() + if err != nil { + return err + } + if k == nil { + break + } + if err := c.DeleteCurrent(); err != nil { + return err + } + s.tableSizeBytes.Add(-int64(len(k) + len(v))) + } + return nil +} + +// SetMem populates only the in-memory tier — used on a read-path (RoTx) cold +// decompress where the MDBX backing cannot be written. +func (s *CodeStore) SetMem(codeHash, code []byte) { + if s == nil || len(codeHash) != 32 || len(code) == 0 { + return + } + var key [32]byte + copy(key[:], codeHash) + s.mem.Set(key, code) +} diff --git a/execution/cache/code_store_test.go b/execution/cache/code_store_test.go new file mode 100644 index 00000000000..59fb5cdb77d --- /dev/null +++ b/execution/cache/code_store_test.go @@ -0,0 +1,63 @@ +// Copyright 2024 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package cache + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/common/crypto" + "github.com/erigontech/erigon/db/kv/dbcfg" + "github.com/erigontech/erigon/db/kv/memdb" +) + +func TestCodeStore_TwoTierAndEvict(t *testing.T) { + db := memdb.NewTestDB(t, dbcfg.ChainDB) + tx, err := db.BeginRw(t.Context()) + require.NoError(t, err) + defer tx.Rollback() + + code := []byte{0x60, 0x80, 0x60, 0x40, 0x52} + hash := crypto.Keccak256(code) + + // Write path populates both tiers. + cs := NewCodeStore(1<<20, 1<<20) + require.NoError(t, cs.PutByHash(tx, hash, code)) + got, ok := cs.GetByHash(tx, hash) + require.True(t, ok) + require.Equal(t, code, got) + + // A fresh store (empty mem) serves from the MDBX backing tier. + cs2 := NewCodeStore(1<<20, 1<<20) + got, ok = cs2.GetByHash(tx, hash) + require.True(t, ok, "must serve from the persistent TblCodeCache backing") + require.Equal(t, code, got) + + // Unknown codehash misses cleanly. + _, ok = cs2.GetByHash(tx, crypto.Keccak256([]byte{0xff})) + require.False(t, ok) + + // Evict prunes the backing when over the table cap. + small := NewCodeStore(1<<20, 128) + for i := 0; i < 64; i++ { + c := []byte{byte(i), byte(i >> 8), 0xaa, 0xbb} + require.NoError(t, small.PutByHash(tx, crypto.Keccak256(c), c)) + } + require.NoError(t, small.Evict(tx)) + require.LessOrEqual(t, small.tableSizeBytes.Load(), int64(128)) +} diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index ce619376b6e..233b5f33b7b 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -84,8 +84,11 @@ func NewGenericCache[T any](capacityBytes datasize.ByteSize, sizeFunc func(T) in if capacityEntries < 1024 { capacityEntries = 1024 } - if capacityEntries > 1<<22 { - capacityEntries = 1 << 22 + // Absolute safety ceiling on the eagerly-allocated slot array; kept above the + // configured byte budgets' entry counts so it never caps residency below the + // budget (see newDomainCacheBytes). + if capacityEntries > 1<<24 { + capacityEntries = 1 << 24 } return newGenericCacheEntries(capacityBytes, capacityEntries, sizeFunc, mode) } diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index d94af694464..4cbb7119d6d 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -92,12 +92,11 @@ func newDomainCacheBytes(capacityBytes datasize.ByteSize, avgBytes uint32, mode if capacityEntries < 1024 { capacityEntries = 1024 } - // Clamp the slot count, same as NewGenericCache: freelru.NewSharded eagerly - // allocates the whole slot array up front, so an unclamped Account budget - // (1 GB / ~96 B ≈ 11M entries) would allocate gigabytes before caching - // anything. The byte budget still bounds residency below this cap. - if capacityEntries > 1<<22 { - capacityEntries = 1 << 22 + // Absolute safety ceiling on the eagerly-allocated slot array; must stay + // above the configured byte budgets' entry counts (Account 1 GB / ~96 B ≈ + // 11.2M) or it silently caps residency below the budget. + if capacityEntries > 1<<24 { + capacityEntries = 1 << 24 } return &DomainCache{ GenericCache: newGenericCacheEntries(capacityBytes, capacityEntries, func(v []byte) int { return len(v) }, mode), diff --git a/execution/commitment/adaptive_pin.go b/execution/commitment/adaptive_pin.go new file mode 100644 index 00000000000..baff8961978 --- /dev/null +++ b/execution/commitment/adaptive_pin.go @@ -0,0 +1,397 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package commitment + +import ( + "context" + "encoding/hex" + "sync" + "sync/atomic" + + "github.com/erigontech/erigon/common/log/v3" +) + +// AdaptivePinControllerConfig sets the policy knobs for the adaptive +// trunk-pin controller. Defaults target the SSTORE-bloat workload class +// (single contract dominating storage reads). +type AdaptivePinControllerConfig struct { + PromoteThresholdMisses uint64 + MaxPromotedContracts int + DemoteCooldownBlocks int + InitialViewBudgetBytes int + ExtensionBudgetBytes int + PerContractMaxBudgetBytes int +} + +func DefaultAdaptivePinControllerConfig() AdaptivePinControllerConfig { + return AdaptivePinControllerConfig{ + PromoteThresholdMisses: 100, + MaxPromotedContracts: 8, + DemoteCooldownBlocks: 5, + InitialViewBudgetBytes: 4 << 20, + ExtensionBudgetBytes: 8 << 20, + PerContractMaxBudgetBytes: 64 << 20, + } +} + +// AdaptivePinController watches per-contract miss pressure on a +// BranchCache and decides which contracts to pin (with a sync initial +// view), grow (per-block extension), or demote (invalidate the pin +// set after sustained inactivity). +type AdaptivePinController struct { + cache *BranchCache + cfg AdaptivePinControllerConfig + logger log.Logger + + misses sync.Map // [32]byte → *atomic.Uint64 + + mu sync.Mutex + states map[[32]byte]*adaptiveContractState + + parallelResolverFactory ParallelResolverFactory + dbBranchesProvider DbBranchesProvider +} + +// ParallelResolverFactory builds a fresh BatchBranchResolver for one +// OnBlockComplete call. release() is invoked after the controller is done +// with the resolver. Returning (nil, nil, err) makes the controller fall +// back to the serial-BFS path for this block. +type ParallelResolverFactory func() (resolve BatchBranchResolver, release func(), err error) + +// DbBranchesProvider returns the MDBX-resident branch overlay for one +// contract — values shadow file values in the parallel preload's wave. +// Empty/nil result is valid (no overlay; resolver is authoritative). +type DbBranchesProvider func(contractHash []byte) map[string][]byte + +type adaptiveContractState struct { + contractHash [32]byte + promotedAtBlock uint64 + preload *ContractTrunkPreload // serial-BFS path (nil when parallel) + parallel *ContractTrunkPreloadParallel // parallel-wave-BFS path (nil when serial) + coldBlocksInARow int +} + +func (s *adaptiveContractState) pinnedTotal() int { + if s.parallel != nil { + return s.parallel.PinnedTotal() + } + return s.preload.PinnedTotal() +} + +func (s *adaptiveContractState) usedBytes() int { + if s.parallel != nil { + return s.parallel.UsedBytes() + } + return s.preload.UsedBytes() +} + +func (s *adaptiveContractState) queueRemaining() int { + if s.parallel != nil { + return s.parallel.QueueRemaining() + } + return s.preload.QueueRemaining() +} + +func (s *adaptiveContractState) pinnedPrefixes() [][]byte { + if s.parallel != nil { + return s.parallel.PinnedPrefixes() + } + return s.preload.PinnedPrefixes() +} + +func NewAdaptivePinController(cache *BranchCache, cfg AdaptivePinControllerConfig, logger log.Logger) *AdaptivePinController { + if cfg.InitialViewBudgetBytes <= 0 { + cfg.InitialViewBudgetBytes = 4 << 20 + } + if cfg.ExtensionBudgetBytes <= 0 { + cfg.ExtensionBudgetBytes = 8 << 20 + } + if cfg.PerContractMaxBudgetBytes <= 0 { + cfg.PerContractMaxBudgetBytes = 64 << 20 + } + if cfg.MaxPromotedContracts <= 0 { + cfg.MaxPromotedContracts = 8 + } + if cfg.DemoteCooldownBlocks <= 0 { + cfg.DemoteCooldownBlocks = 5 + } + if cfg.PromoteThresholdMisses == 0 { + cfg.PromoteThresholdMisses = 100 + } + return &AdaptivePinController{ + cache: cache, + cfg: cfg, + logger: logger, + states: make(map[[32]byte]*adaptiveContractState), + } +} + +// Bind installs the controller's miss-callback on the cache. +// Safe to call multiple times — replaces any prior callback. +func (c *AdaptivePinController) Bind() { + c.cache.SetMissCallback(c.onCacheMiss) +} + +// SetParallelMode switches promote/extend to the wave-BFS parallel preload. +// Either argument may be nil to clear; with factory==nil the controller uses +// the serial-BFS CommitmentReader path. Already-promoted contracts keep +// their existing serial/parallel state until next demote. +func (c *AdaptivePinController) SetParallelMode(factory ParallelResolverFactory, provider DbBranchesProvider) { + c.mu.Lock() + defer c.mu.Unlock() + c.parallelResolverFactory = factory + c.dbBranchesProvider = provider +} + +func (c *AdaptivePinController) onCacheMiss(prefix []byte) { + hash, ok := ContractHashFromPrefix(prefix) + if !ok { + return + } + v, _ := c.misses.LoadOrStore(hash, new(atomic.Uint64)) + v.(*atomic.Uint64).Add(1) +} + +// OnBlockComplete consumes the per-block miss snapshot and decides +// promotions, extensions, and demotions. Synchronous — preloads run +// inline so the new pin set is available for the next block's reads. +func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum uint64, reader CommitmentReader) { + misses := c.snapshotMisses() + + c.mu.Lock() + defer c.mu.Unlock() + + // One factory call per block, shared across all contracts. nil falls back to serial. + var parallelResolve BatchBranchResolver + var releaseParallel func() + if c.parallelResolverFactory != nil { + r, release, err := c.parallelResolverFactory() + if err != nil { + c.warnf("[adaptive-pin] parallel resolver factory failed, falling back to serial", "err", err, "block", blockNum) + } else { + parallelResolve = r + releaseParallel = release + } + } + if releaseParallel != nil { + defer releaseParallel() + } + + var promoted, extended, demoted int + + for hash, state := range c.states { + n, hadMisses := misses[hash] + if hadMisses && n > 0 { + state.coldBlocksInARow = 0 + delete(misses, hash) + if state.queueRemaining() > 0 && state.usedBytes() < c.cfg.PerContractMaxBudgetBytes { + remaining := c.cfg.PerContractMaxBudgetBytes - state.usedBytes() + step := c.cfg.ExtensionBudgetBytes + if step > remaining { + step = remaining + } + if err := c.runExtensionLocked(ctx, state, step, parallelResolve, reader); err != nil { + c.warnf("[adaptive-pin] extend failed", "hash", hex.EncodeToString(hash[:]), "err", err) + } else { + extended++ + } + } + continue + } + state.coldBlocksInARow++ + if state.coldBlocksInARow >= c.cfg.DemoteCooldownBlocks { + c.demoteLocked(hash, state) + delete(c.states, hash) + demoted++ + } + } + + if len(misses) > 0 && len(c.states) < c.cfg.MaxPromotedContracts { + candidates := pickPromotionCandidates(misses, c.cfg.PromoteThresholdMisses, c.cfg.MaxPromotedContracts-len(c.states)) + for _, hash := range candidates { + state, err := c.promoteLocked(ctx, hash, blockNum, parallelResolve, reader) + if err != nil { + c.warnf("[adaptive-pin] initial-view failed", "hash", hex.EncodeToString(hash[:]), "err", err) + continue + } + c.states[hash] = state + promoted++ + } + } + + if promoted > 0 { + mxAdaptivePromoted.AddUint64(uint64(promoted)) + } + if extended > 0 { + mxAdaptiveExtended.AddUint64(uint64(extended)) + } + if demoted > 0 { + mxAdaptiveDemoted.AddUint64(uint64(demoted)) + } + mxAdaptiveActive.SetUint64(uint64(len(c.states))) + + if c.logger != nil && (promoted+extended+demoted > 0 || len(c.states) > 0) { + c.logger.Info("[adaptive-pin]", + "block", blockNum, + "promoted_total", len(c.states), + "promoted_this_block", promoted, + "extended_this_block", extended, + "demoted_this_block", demoted, + "cache_pinned_total", c.cache.PinnedCount()) + } +} + +func (c *AdaptivePinController) snapshotMisses() map[[32]byte]uint64 { + out := make(map[[32]byte]uint64) + c.misses.Range(func(k, v any) bool { + hash := k.([32]byte) + n := v.(*atomic.Uint64).Swap(0) + if n > 0 { + out[hash] = n + } + return true + }) + return out +} + +// demoteLocked: caller must hold c.mu. +func (c *AdaptivePinController) demoteLocked(hash [32]byte, state *adaptiveContractState) { + for _, prefix := range state.pinnedPrefixes() { + c.cache.Invalidate(prefix) + } + if c.logger != nil { + c.logger.Info("[adaptive-pin] demoted", + "hash", hex.EncodeToString(hash[:]), + "pinned_was", state.pinnedTotal(), + "used_mb_was", state.usedBytes()/(1<<20), + "cold_blocks", state.coldBlocksInARow) + } +} + +// promoteLocked: caller must hold c.mu. On error the partial pin set is rolled back. +func (c *AdaptivePinController) promoteLocked( + ctx context.Context, + hash [32]byte, + blockNum uint64, + parallelResolve BatchBranchResolver, + reader CommitmentReader, +) (*adaptiveContractState, error) { + if parallelResolve != nil { + p, err := NewContractTrunkPreloadParallel(hash[:]) + if err != nil { + return nil, err + } + var dbBranches map[string][]byte + if c.dbBranchesProvider != nil { + dbBranches = c.dbBranchesProvider(hash[:]) + } + if _, _, err := p.Run(c.cfg.InitialViewBudgetBytes, dbBranches, parallelResolve, c.cache, c.logger); err != nil { + for _, prefix := range p.PinnedPrefixes() { + c.cache.Invalidate(prefix) + } + return nil, err + } + return &adaptiveContractState{ + contractHash: hash, + promotedAtBlock: blockNum, + parallel: p, + }, nil + } + p, err := NewContractTrunkPreload(hash[:]) + if err != nil { + return nil, err + } + if _, _, err := p.Run(c.cfg.InitialViewBudgetBytes, reader, c.cache, c.logger); err != nil { + for _, prefix := range p.PinnedPrefixes() { + c.cache.Invalidate(prefix) + } + return nil, err + } + return &adaptiveContractState{ + contractHash: hash, + promotedAtBlock: blockNum, + preload: p, + }, nil +} + +// runExtensionLocked: caller must hold c.mu. Uses the saved state's mode +// (parallel vs serial); a serial state with a parallel resolver available +// keeps using serial — switching mid-contract would lose the queue position. +func (c *AdaptivePinController) runExtensionLocked( + ctx context.Context, + state *adaptiveContractState, + stepBudget int, + parallelResolve BatchBranchResolver, + reader CommitmentReader, +) error { + if state.parallel != nil { + if parallelResolve == nil { + return nil + } + var dbBranches map[string][]byte + if c.dbBranchesProvider != nil { + dbBranches = c.dbBranchesProvider(state.contractHash[:]) + } + _, _, err := state.parallel.Run(stepBudget, dbBranches, parallelResolve, c.cache, c.logger) + return err + } + _, _, err := state.preload.Run(stepBudget, reader, c.cache, c.logger) + return err +} + +func (c *AdaptivePinController) PromotedContracts() [][32]byte { + c.mu.Lock() + defer c.mu.Unlock() + out := make([][32]byte, 0, len(c.states)) + for h := range c.states { + out = append(out, h) + } + return out +} + +func pickPromotionCandidates(misses map[[32]byte]uint64, threshold uint64, maxN int) [][32]byte { + if maxN <= 0 { + return nil + } + type cand struct { + hash [32]byte + n uint64 + } + var pool []cand + for h, n := range misses { + if n >= threshold { + pool = append(pool, cand{h, n}) + } + } + if len(pool) > maxN { + for i := 0; i < maxN; i++ { + best := i + for j := i + 1; j < len(pool); j++ { + if pool[j].n > pool[best].n { + best = j + } + } + pool[i], pool[best] = pool[best], pool[i] + } + pool = pool[:maxN] + } + out := make([][32]byte, len(pool)) + for i, c := range pool { + out[i] = c.hash + } + return out +} + +func (c *AdaptivePinController) warnf(msg string, kv ...any) { + if c.logger != nil { + c.logger.Warn(msg, kv...) + } +} + +var _ = context.Background // reserved for cancellation of in-flight preloads diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index d4f581ff04d..440ea76d508 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -19,12 +19,21 @@ package commitment import ( "bytes" "fmt" + "os" "sync/atomic" + "github.com/elastic/go-freelru" + + "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/maphash" "github.com/erigontech/erigon/execution/cache/coherence" + "github.com/erigontech/erigon/execution/commitment/nibbles" ) +// u64ident is the freelru hash callback for uint64 keys already well-distributed +// by maphash — the low 32 bits suffice for shard routing. +func u64ident(k uint64) uint32 { return uint32(k) } + // KeyCommitmentState is the commitment-domain key under which the trie // checkpoint (txNum / blockNum / encoded root state) is stored. It is NOT a // trie branch: it changes every block, so it must never enter the @@ -37,35 +46,170 @@ func isCommitmentStateKey(prefix []byte) bool { return bytes.Equal(prefix, KeyCommitmentState) } -// BranchCache is the aggregator-scope (one per commitment Domain) cache of -// commitment-trie branch data: a single pinned slot for the always-hot root -// branch plus a bounded LRU tail for the rest. It is a passive store — the -// trie walker/encoder drives all reads and writes; the cache never reaches -// into underlying state. +// BranchCache stores commitment-trie branch data: +// +// - Bounded LRU tail with configurable capacity (eviction is well-defined, +// suitable for long-lived caching across many Process calls without +// unbounded memory growth). +// - Single pinned slot for the root branch (always hottest, always present +// once populated, never subject to LRU eviction). Compact prefix of +// length 0 (or single-byte "no-key" form) targets this slot. +// +// Lifetime: aggregator-scope (one instance per Domain). SharedDomains +// pulls the instance via BranchCacheProvider on the AggregatorRoTx; +// commitment-context plumbs it through to the trie via +// InitializeTrieAndUpdates. The previous WarmupCache type (per-Process, +// duplicating account/storage/branch caching above this layer) was +// deleted in the WarmupCache consolidation; BranchCache is now the +// single branch cache. +// +// # Responsibility split (architectural) +// +// The cache is a passive store. Reads and writes are driven by the +// trie walker / encoder; the cache itself never reaches into the +// underlying state. +// +// - BranchCache: passive store of branch bytes. +// Doesn't fetch anything. +// - Branch warmer (warmuper.go): narrow scope — pre-fetches +// *branches* along touched-key paths via SD.GetLatest. No +// account/storage prefetch — that conflated branch warm-up with +// leaf-data fetch. If a fold needs leaf data the trie walker +// fetches it directly (or it's already in Updates / memoized as +// stateHash). +// - Trie walker, block-processing path: receives Updates from the +// executor, folds them. Memoized stateHashes serve siblings; new +// values come from Updates. Doesn't reach into leaf data via +// prefetch. +// - Trie walker, witness / proof generation path: walks the trie +// structure and *needs* to fetch state to materialize the proof. +// This is the walker's responsibility — it drives its own reads +// against SD. If that path turns out to be cold-bound on real +// workloads it may indicate a need for separate account / storage +// caches (the `add_execution_context_with_caches` work has a +// reference design for these). Treat that as a separate concern +// from this BranchCache — different scope, different lifetime, +// different invalidation. Do not regrow the branch warmer's +// scope to cover it. +// +// The disk_sto / disk_acc counters on the [commitment][cache-fp] log +// line surface any fall-through where the trie compute reaches the +// underlying ctx.Account / ctx.Storage paths. On block-processing +// workloads they should remain zero; non-zero values signal a +// memoization gap or a missing walker-side prefetch. +// +// # Concurrency contract — caller invariants +// +// Internally, the LRU tail is thread-safe (hashicorp/golang-lru/v2) and +// the pinned root slot is an atomic.Pointer. So any combination of +// concurrent Get / Put / Invalidate is mechanically safe — no panics, no +// torn reads. But "mechanically safe" is NOT the same as "logically +// consistent across writers." The cache is designed to be used under one +// caller invariant: +// +// - Single writer per prefix at any moment. The cache does not coordinate +// concurrent writes to the same key — last-Put-wins semantics, with no +// guarantee that the winning value is the one the application wanted. +// +// # Concurrency contract — how the existing concurrent trie satisfies it +// +// The current ConcurrentPatriciaHashed (parallel commitment calculator) +// satisfies that invariant by construction: +// +// - Mounts partition the prefix space by FIRST NIBBLE. Mount N's +// encoder only writes branches whose key starts with [0x0N ...]. +// Different mounts therefore never write to the same prefix. +// (See hex_concurrent_patricia_hashed.go: NewConcurrentPatriciaHashed +// creates 16 mounts via SpawnSubTrie; each mount has its own HPH, +// own BranchEncoder, own PatriciaContext / roTx.) // -// Concurrency: the LRU tail and the atomic-pointer root slot are individually -// thread-safe, but the cache assumes a single writer per prefix (last-Put-wins -// with no cross-writer coordination). The concurrent trie satisfies this by -// partitioning the prefix space by first nibble across mounts and writing the -// root branch only in the sequential post-Wait root fold. Any future design -// that breaks single-writer-per-prefix (e.g. parallel tree-reduce fold, or a -// different prefix partitioning) must add per-prefix coordination at the -// orchestrator layer — do not add internal locking here. +// - Root branch (prefix [0x00]) is written by the single root fold +// that runs SEQUENTIALLY after errgroup.Wait() in ParallelHashSort. +// One writer for the pinned root slot. +// +// - Mount→root grid roll-up is mutex-protected via +// ConcurrentPatriciaHashed.rootMu — but that updates IN-MEMORY grid +// cells, not the cache. The cache only sees the eventual root +// branch when the post-Wait root fold encodes it. +// +// # Concurrency contract — what future parallel fold work must preserve +// +// A future parallel tree-reduce fold would change the picture: the parent +// fold (incl. root) would no longer be a single post-Wait sequential pass. +// Multiple goroutines would compute parent branches in parallel as their +// children complete. This MUST not violate "single writer per prefix" — +// any future Stage F design needs an explicit per-prefix coordination layer +// (atomic counter on parent "children remaining"; only the last-decrementer +// writes the parent). That coordination belongs at the orchestrator layer; +// the cache itself does NOT add per-prefix locking because that would be +// wasted work for the current architecture. +// +// If you are implementing parallel fold (or any other architecture that +// breaks the "single writer per prefix" invariant), do NOT relax the +// invariant by adding internal locking to the cache. Add the +// coordination at the orchestrator layer where the partitioning logic +// lives. The cache stays simple; the orchestrator owns the discipline. +// +// Likewise if you change the prefix partitioning (e.g. by-second-nibble +// mounts, depth-based partitioning, anything other than first-nibble), +// re-validate that distinct workers continue to write disjoint prefix +// spaces. Re-read the partitioning code in +// hex_concurrent_patricia_hashed.go and confirm. type BranchCache struct { // Root tier — single slot for the root branch (always hottest, always // present). Atomic-pointer access so no lock is needed for the hot // read path. root atomic.Pointer[branchCacheEntry] - // LRU tail — bounded entries with per-shard LRU eviction (no global recency - // order across shards). See branchCacheTailShards for why it's sharded. - tail *maphash.ShardedLRU[*branchCacheEntry] + // accountTrunk — the resident upper-account-trie trunk: account-trie + // branches at nibble depths 1-4, held in fixed arrays indexed directly by + // the compact-hex prefix (no hashing, no eviction). The global root + // (depth 0) is the dedicated root slot above; depth 5+ goes to the LRU + // tail. Each slot is an independent atomic.Pointer, so reads and writes + // take no mutex and don't serialize through a shared lock the way the LRU + // tail (and a storage trunk's deep overflow map) do. + accountTrunk *trunk + + // Pinned tier — one storageTrunk per hot contract, keyed by the 32-byte + // account hash. Entries never LRU-evict (sized by the residency policy) + // but still honor the (txN, epoch) unwind model. Lookup checks this tier + // between the account trunk and the tail. pinnedEntries counts filled + // storage slots across all storageTrunks. + pinned *maphash.Map[*trunk] + pinnedEntries atomic.Int64 + + // LRU tail — bounded entries, evicts oldest when full. freelru.ShardedLRU + // keyed by the maphash of the prefix (single-alloc, thread-safe per shard). + tail *freelru.ShardedLRU[uint64, *branchCacheEntry] + + // trunkDisabled (env BRANCH_CACHE_TRUNK_DISABLE) routes depth-1-4 account + // branches back to the LRU tail instead of the resident account trunk — a + // runtime A/B switch to isolate whether the resident trunk is the source + // of a data discrepancy (the LRU self-heals stale entries via eviction; + // the trunk does not). + trunkDisabled bool // Stats — atomic counters surfaced via Stats(). - rootHits, rootMisses atomic.Uint64 - tailHits, tailMisses atomic.Uint64 - bytesServed atomic.Uint64 - staleEvicted atomic.Uint64 // entries dropped lazily on read after an unwind + rootHits, rootMisses atomic.Uint64 + trunkHits, trunkMisses atomic.Uint64 + pinnedHits, pinnedMisses atomic.Uint64 + tailHits, tailMisses atomic.Uint64 + bytesServed atomic.Uint64 + staleEvicted atomic.Uint64 // entries dropped lazily on read after an unwind + + // onMiss fires when lookup misses all tiers. The residency/adaptive layer + // (added separately) registers here to attribute miss pressure per + // contract; nil hot path is one atomic load + nil check. + onMiss atomic.Pointer[MissCallback] + + // preloadClaimed gates the one-shot residency preload trigger. + preloadClaimed atomic.Bool + + // last-published pinned counter snapshots — PublishMetrics emits the delta + // since the previous publish so the Prometheus counters track per-Flush + // activity, not snapshot absolutes. + lastPublishedPinnedHits atomic.Uint64 + lastPublishedPinnedMisses atomic.Uint64 // coh is the (epoch, floor) unwind-coherence primitive shared with the state // and code caches: an entry is valid iff written in the current epoch OR its @@ -96,43 +240,218 @@ type branchCacheEntry struct { epoch uint32 } +// MissCallback is invoked when lookup misses ALL tiers (root, account trunk, +// pinned storage trunk, LRU tail). Called on the hot read path; the residency +// layer registers it. Implementations must be lock-free / non-blocking. +type MissCallback func(prefix []byte) + +// trunk is a resident, lock-free fixed-array tier shared by both tries: the +// accountTrunk holds account-trie branches at nibble depths 1-4 (d4 allocated, +// deep nil); each per-contract storageTrunk holds storage branches at storage +// depths 0-3 with depth 4+ in deep (d4 nil, deep allocated). Slots are +// atomic.Pointer: under the single-writer-per-prefix invariant readers/writers +// take no mutex (just an atomic load/store per slot); only deep (a maphash.Map) +// locks. +type trunk struct { + d0 atomic.Pointer[branchCacheEntry] + d1 [16]atomic.Pointer[branchCacheEntry] + d2 [256]atomic.Pointer[branchCacheEntry] + d3 [4096]atomic.Pointer[branchCacheEntry] + d4 *[65536]atomic.Pointer[branchCacheEntry] + deep *maphash.Map[*branchCacheEntry] +} + +// newAccountTrunk builds the global account trunk: dense depth-4 fixed array, +// no deep overflow (account depth 5+ uses the LRU tail). +func newAccountTrunk() *trunk { + return &trunk{d4: &[65536]atomic.Pointer[branchCacheEntry]{}} +} + +// newStorageTrunk builds a per-contract storage trunk: deep overflow for +// storage depth 4+, no depth-4 fixed array. +func newStorageTrunk() *trunk { + return &trunk{deep: maphash.NewMap[*branchCacheEntry]()} +} + +// slot returns the fixed-array slot for a nibble path of length 0-3 (and length +// 4 when the depth-4 array is present, i.e. the account trunk), or nil when the +// path is deeper — the caller then uses deep (storage) or the tail (account). +func (t *trunk) slot(path []byte) *atomic.Pointer[branchCacheEntry] { + switch len(path) { + case 0: + return &t.d0 + case 1: + return &t.d1[path[0]] + case 2: + return &t.d2[uint16(path[0])<<4|uint16(path[1])] + case 3: + return &t.d3[uint16(path[0])<<8|uint16(path[1])<<4|uint16(path[2])] + case 4: + if t.d4 != nil { + return &t.d4[uint32(path[0])<<12|uint32(path[1])<<8|uint32(path[2])<<4|uint32(path[3])] + } + } + return nil +} + // DefaultBranchCacheTailCapacity is the LRU tail size used when no // explicit capacity is given. ~50k entries × ~500 bytes = ~25 MB // at typical mainnet branch sizes. const DefaultBranchCacheTailCapacity = 50000 -// branchCacheTailShards splits the tail into this many independently-locked -// shards so concurrent warmup workers don't serialize on a single LRU mutex. -// Fixed value sized for the default warmup pool (dbg.TipTrieWarmupers ≈ NumCPU*8). -const branchCacheTailShards = 256 - // BranchCacheProvider exposes the long-lived BranchCache attached to the -// commitment domain. Implemented by *db/state.AggregatorRoTx via duck typing to -// avoid an execctx→db/state import cycle. Nil means caching is disabled — callers -// MUST treat nil as "behave as if disabled" rather than panic. +// commitment domain. Implemented by *db/state.AggregatorRoTx (via duck +// typing) so callers in the SharedDomains construction path can fetch the +// cache without forcing db/state/execctx to import db/state — that import +// would create a cycle since db/state imports execctx (squeeze.go, +// trie_reader_integration_test.go, …). +// +// Returning nil is permitted; callers MUST treat nil as "no shared cache, +// behave as if disabled" rather than panic. type BranchCacheProvider interface { BranchCache() *BranchCache } +// branchCacheTailShards splits the LRU tail into independently-locked shards so +// concurrent commitment mounts / warmup workers don't serialize on one mutex. +const branchCacheTailShards = 256 + // NewBranchCache constructs a BranchCache with the given LRU tail capacity. // Capacity <= 0 panics — pass a positive value or DefaultBranchCacheTailCapacity. func NewBranchCache(tailCapacity int) *BranchCache { if tailCapacity <= 0 { panic(fmt.Sprintf("BranchCache: tailCapacity must be positive, got %d", tailCapacity)) } - tail, err := maphash.NewShardedLRU[*branchCacheEntry](tailCapacity, branchCacheTailShards) + tailCap := uint32(tailCapacity) + tail, err := freelru.NewShardedWithSize[uint64, *branchCacheEntry](branchCacheTailShards, tailCap, tailCap+tailCap/4, u64ident) if err != nil { - panic(fmt.Sprintf("BranchCache: NewShardedLRU: %s", err)) + panic(fmt.Sprintf("BranchCache: NewShardedWithSize: %s", err)) } bc := &BranchCache{ - tail: tail, + tail: tail, + accountTrunk: newAccountTrunk(), + pinned: maphash.NewMap[*trunk](), + trunkDisabled: os.Getenv("BRANCH_CACHE_TRUNK_DISABLE") != "", } // Before any unwind every entry's txN is at/below the floor, so the epoch // check never strands a valid entry. bc.coh.Init() + log.Info("[branch-cache] init", "trunkEnabled", !bc.trunkDisabled, "tailCap", tailCapacity) return bc } +// trunkSlot returns the resident account-trunk slot for an account-trie branch +// at nibble depth 1-4, or nil if the prefix is the root (depth 0), a storage +// trunk, or depth >= 5 (served by the LRU tail). The compact-hex prefix maps +// directly to an array index, no hashing. Bit 4 of byte 0 is the odd-length +// flag; the low nibble of byte 0 is the first nibble when odd. +func (c *BranchCache) trunkSlot(prefix []byte) *atomic.Pointer[branchCacheEntry] { + if c.trunkDisabled { + return nil + } + switch len(prefix) { + case 1: + if prefix[0]&0x10 != 0 { // 1 nibble + return &c.accountTrunk.d1[prefix[0]&0x0f] + } + case 2: + if prefix[0]&0x10 == 0 { // 2 nibbles + return &c.accountTrunk.d2[prefix[1]] + } + return &c.accountTrunk.d3[uint16(prefix[0]&0x0f)<<8|uint16(prefix[1])] // 3 nibbles + case 3: + if prefix[0]&0x10 == 0 { // 4 nibbles + return &c.accountTrunk.d4[uint16(prefix[1])<<8|uint16(prefix[2])] + } + // 5 nibbles (odd, 3 bytes) -> LRU tail + } + return nil +} + +// storageRoute decodes a storage-trunk prefix (compact-hex of 64 account +// nibbles + S storage nibbles) into its contract storageTrunk and the +// storage-nibble path. Returns ok=false for non-storage prefixes (< 64 nibbles) +// so the caller falls through to the LRU tail. When create is true the +// contract's storageTrunk is allocated on demand (PinEntry path). acct is the +// 32-byte packed account hash (the map key). +func (c *BranchCache) storageRoute(prefix []byte, create bool) (st *trunk, acct []byte, stor []byte, ok bool) { + if len(prefix) < 33 { + return nil, nil, nil, false + } + nib := nibbles.CompactToHex(prefix) + if len(nib) < 64 { + return nil, nil, nil, false + } + packed := make([]byte, 32) + for i := 0; i < 32; i++ { + packed[i] = nib[2*i]<<4 | nib[2*i+1] + } + stor = nib[64:] + st, found := c.pinned.Get(packed) + if !found { + if !create { + return nil, packed, stor, false + } + st = newStorageTrunk() + c.pinned.Set(packed, st) + } + return st, packed, stor, true +} + +// ContractHashFromPrefix extracts the 32-byte contract (account) hash from a +// storage-trunk prefix (compact-hex of >= 64 account nibbles + storage +// nibbles). ok=false for non-storage prefixes. Used by the residency layer to +// attribute per-contract miss pressure. +func ContractHashFromPrefix(prefix []byte) (hash [32]byte, ok bool) { + if len(prefix) < 33 { + return hash, false + } + nib := nibbles.CompactToHex(prefix) + if len(nib) < 64 { + return hash, false + } + for i := 0; i < 32; i++ { + hash[i] = nib[2*i]<<4 | nib[2*i+1] + } + return hash, true +} + +// clearTrunk resets every resident account-trunk slot (depths 0-4) to nil in +// place (atomic per-slot stores, not a pointer swap — lock-free readers deref +// c.accountTrunk concurrently). +func (c *BranchCache) clearTrunk() { + t := c.accountTrunk + t.d0.Store(nil) + for i := range t.d1 { + t.d1[i].Store(nil) + } + for i := range t.d2 { + t.d2[i].Store(nil) + } + for i := range t.d3 { + t.d3[i].Store(nil) + } + for i := range t.d4 { + t.d4[i].Store(nil) + } +} + +func (c *BranchCache) fireOnMiss(prefix []byte) { + if cb := c.onMiss.Load(); cb != nil { + (*cb)(prefix) + } +} + +// SetMissCallback installs a hook fired on every all-tier miss. Pass nil to +// clear. Used by the residency/adaptive layer (added separately). +func (c *BranchCache) SetMissCallback(cb MissCallback) { + if cb == nil { + c.onMiss.Store(nil) + return + } + c.onMiss.Store(&cb) +} + // isRootPrefix reports whether prefix targets the pinned root slot. The // commitment-trie compact encoding uses a 1-byte even-length flag (0x00) // to represent the empty nibble path (root branch). Anything longer goes @@ -141,32 +460,46 @@ func isRootPrefix(prefix []byte) bool { return len(prefix) == 1 && prefix[0] == 0x00 } -// tailHash maps prefix to its LRU-tail key, returning ok=false for a prefix that -// must never be cached (the commitment state checkpoint key). -func tailHash(prefix []byte) (uint64, bool) { - if isCommitmentStateKey(prefix) { - return 0, false - } - return maphash.Hash(prefix), true -} - func (c *BranchCache) lookup(prefix []byte) (*branchCacheEntry, bool) { if isRootPrefix(prefix) { entry := c.root.Load() if entry == nil { c.rootMisses.Add(1) + c.fireOnMiss(prefix) return nil, false } c.rootHits.Add(1) return entry, true } - h, ok := tailHash(prefix) - if !ok { + // Resident account trunk (fixed arrays, depths 1-4). Disjoint from the + // storage trunks (depth >= 64) and tail, so a miss here is genuine. + if slot := c.trunkSlot(prefix); slot != nil { + if entry := slot.Load(); entry != nil { + c.trunkHits.Add(1) + return entry, true + } + c.trunkMisses.Add(1) + c.fireOnMiss(prefix) return nil, false } - entry, ok := c.tail.GetByHash(h) + // Pinned tier: per-contract storage trunk (fixed skeleton + deep overflow). + if st, _, stor, ok := c.storageRoute(prefix, false); ok { + var entry *branchCacheEntry + if slot := st.slot(stor); slot != nil { + entry = slot.Load() + } else { + entry, _ = st.deep.Get(prefix) + } + if entry != nil { + c.pinnedHits.Add(1) + return entry, true + } + } + c.pinnedMisses.Add(1) + entry, ok := c.tail.Get(maphash.Hash(prefix)) if !ok { c.tailMisses.Add(1) + c.fireOnMiss(prefix) return nil, false } c.tailHits.Add(1) @@ -178,21 +511,79 @@ func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { c.root.Store(entry) return } - h, ok := tailHash(prefix) + if slot := c.trunkSlot(prefix); slot != nil { + slot.Store(entry) + return + } + // Keep a prefix already pinned in a storage trunk in place across the + // per-block invalidate+Put refresh rather than dropping it to the tail. + if st, _, stor, ok := c.storageRoute(prefix, false); ok { + if slot := st.slot(stor); slot != nil { + if slot.Load() != nil { + slot.Store(entry) + return + } + } else if _, exists := st.deep.Get(prefix); exists { + st.deep.Set(prefix, entry) + return + } + } + c.tail.Add(maphash.Hash(prefix), entry) +} + +// PinEntry inserts or replaces a pinned cache entry for prefix in its contract's +// storage trunk (allocated on demand). Pinned entries never LRU-evict but still +// honor the (txN, epoch) unwind model. Data is copied; safe to mutate the input +// after the call. Non-storage prefixes (< 64 nibbles) fall through to the tail. +func (c *BranchCache) PinEntry(prefix []byte, data []byte, step, txN uint64) { + if isCommitmentStateKey(prefix) { + return + } + dataCopy := make([]byte, len(data)) + copy(dataCopy, data) + entry := &branchCacheEntry{data: dataCopy, step: step, txN: txN, epoch: c.coh.Epoch()} + st, _, stor, ok := c.storageRoute(prefix, true) if !ok { + c.tail.Add(maphash.Hash(prefix), entry) return } - c.tail.SetByHash(h, entry) + if slot := st.slot(stor); slot != nil { + if slot.Load() == nil { + c.pinnedEntries.Add(1) + } + slot.Store(entry) + return + } + if _, exists := st.deep.Get(prefix); !exists { + c.pinnedEntries.Add(1) + } + st.deep.Set(prefix, entry) +} + +// PinnedCount returns the number of currently pinned storage-trunk entries. +func (c *BranchCache) PinnedCount() int { + return int(c.pinnedEntries.Load()) +} + +// PinnedStats returns the pinned-tier hit/miss/entries counters. +func (c *BranchCache) PinnedStats() (hits, misses uint64, entries int) { + return c.pinnedHits.Load(), c.pinnedMisses.Load(), int(c.pinnedEntries.Load()) +} + +// TryClaimPreload returns true exactly once per cache lifetime — the residency +// preload trigger uses it so the preload runs once regardless of how many +// SharedDomains instances are constructed. +func (c *BranchCache) TryClaimPreload() bool { + return c.preloadClaimed.CompareAndSwap(false, true) } // Get retrieves branch data from the cache. Returns the canonical encoded // bytes (with the leading 2-byte touch-map prefix) plus the on-disk file // step the bytes came from (0 if not tracked). -// -// The returned slice is cache-owned and shared across callers — it MUST NOT -// be mutated. Callers needing to modify the bytes must copy first (the -// trie-context Branch() boundary already does, via common.Copy). func (c *BranchCache) Get(prefix []byte) ([]byte, uint64, bool) { + if isCommitmentStateKey(prefix) { + return nil, 0, false + } entry, ok := c.lookup(prefix) if !ok { return nil, 0, false @@ -228,29 +619,39 @@ func (c *BranchCache) Put(prefix []byte, data []byte, step, txN uint64) { }) } -// Invalidate removes the entry at prefix entirely from whichever tier holds -// it. Use when the caller knows the canonical store has changed and the cached -// entry should not be served at all. +// Invalidate removes the entry at prefix entirely from whichever tier +// holds it. Use when the caller knows the canonical store has changed +// and the cached entry should not be served at all (vs MarkDirty which +// keeps the entry but blocks PutIfClean overwrites). func (c *BranchCache) Invalidate(prefix []byte) { if isRootPrefix(prefix) { c.root.Store(nil) return } - h, ok := tailHash(prefix) - if !ok { + if slot := c.trunkSlot(prefix); slot != nil { + slot.Store(nil) return } - c.tail.DeleteByHash(h) + if st, _, stor, ok := c.storageRoute(prefix, false); ok { + if slot := st.slot(stor); slot != nil { + if slot.Swap(nil) != nil { + c.pinnedEntries.Add(-1) + } + } else if _, exists := st.deep.Get(prefix); exists { + st.deep.Delete(prefix) + c.pinnedEntries.Add(-1) + } + } + c.tail.Remove(maphash.Hash(prefix)) } // Unwind invalidates entries that reflect dead-fork state. unwindToTxN is the -// unwind floor — the first rolled-back txNum (SharedDomains passes -// Min(unwindPoint+1)), not the rewind target — because the stale check is -// txN >= floor. O(1) and scan-free: bump the epoch (so entries written in the -// new, live epoch stay valid) and lower the unwind floor to unwindToTxN (so -// old-epoch entries at or above it are dropped lazily on their next Get). The -// floor only ever decreases, so a shallow unwind cannot resurrect entries a -// deeper one invalidated. See coherence.Gen.Unwind. +// txN the chain is rewound to. O(1) and scan-free: bump the epoch (so entries +// written in the new, live epoch stay valid) and lower the unwind floor to +// unwindToTxN (so old-epoch entries at or above it are dropped lazily on their +// next Get). The floor only ever decreases, so a shallow unwind cannot +// resurrect entries a deeper one invalidated. Mirrors GenericCache.Unwind so +// branch and state caches honor one (txN, epoch) model (#21752). func (c *BranchCache) Unwind(unwindToTxN uint64) { c.coh.Unwind(unwindToTxN) } @@ -261,9 +662,16 @@ func (c *BranchCache) Unwind(unwindToTxN uint64) { // different root. func (c *BranchCache) Clear() { c.root.Store(nil) + c.clearTrunk() + c.pinned = maphash.NewMap[*trunk]() + c.pinnedEntries.Store(0) c.tail.Purge() c.rootHits.Store(0) c.rootMisses.Store(0) + c.trunkHits.Store(0) + c.trunkMisses.Store(0) + c.pinnedHits.Store(0) + c.pinnedMisses.Store(0) c.tailHits.Store(0) c.tailMisses.Store(0) c.bytesServed.Store(0) @@ -271,11 +679,13 @@ func (c *BranchCache) Clear() { c.coh.Init() } -// Stats returns a one-line summary of root-tier and tail-tier hit/miss -// counters plus bytes served. Format mirrors WarmupCache.Stats() so -// per-Process log lines can compose them. +// Stats returns a one-line summary of the cache tiers' hit/miss counters plus +// bytes served. Format mirrors WarmupCache.Stats() so per-Process log lines can +// compose them. func (c *BranchCache) Stats() string { rh, rm := c.rootHits.Load(), c.rootMisses.Load() + kh, km := c.trunkHits.Load(), c.trunkMisses.Load() + ph, pm := c.pinnedHits.Load(), c.pinnedMisses.Load() th, tm := c.tailHits.Load(), c.tailMisses.Load() bb := c.bytesServed.Load() pct := func(hit, miss uint64) float64 { @@ -286,8 +696,10 @@ func (c *BranchCache) Stats() string { return 100.0 * float64(hit) / float64(total) } return fmt.Sprintf( - "branch-cache root hit=%d miss=%d (%.1f%%) | tail hit=%d miss=%d (%.1f%%) entries=%d | served %.1f MiB | staleEvicted=%d", + "branch-cache root hit=%d miss=%d (%.1f%%) | trunk hit=%d miss=%d (%.1f%%) | pin hit=%d miss=%d (%.1f%%) entries=%d | tail hit=%d miss=%d (%.1f%%) entries=%d | served %.1f MiB | staleEvicted=%d", rh, rm, pct(rh, rm), + kh, km, pct(kh, km), + ph, pm, pct(ph, pm), int(c.pinnedEntries.Load()), th, tm, pct(th, tm), c.tail.Len(), float64(bb)/1024/1024, c.staleEvicted.Load(), ) diff --git a/execution/commitment/branch_cache_test.go b/execution/commitment/branch_cache_test.go index de044dcc3b4..97ce5b8cae8 100644 --- a/execution/commitment/branch_cache_test.go +++ b/execution/commitment/branch_cache_test.go @@ -17,13 +17,68 @@ package commitment import ( - "runtime" "strings" "testing" "github.com/stretchr/testify/require" ) +// TestBranchCache_AccountTrunkRouting verifies account-trie branches at nibble +// depths 1-4 land in the resident fixed-array trunk (counted as trunk hits), +// survive LRU tail-eviction pressure, and are invalidated lazily by an unwind +// (the trunk honors the same (txN, epoch) model as the tail). +func TestBranchCache_AccountTrunkRouting(t *testing.T) { + c := NewBranchCache(10) // small tail + + trunkKey := []byte{0xa0, 0xb0} // 2 nibbles (even flag) → accountTrunk.d2 + c.Put(trunkKey, []byte("trunk-data"), 0, 100) + + got, _, ok := c.Get(trunkKey) + require.True(t, ok) + require.Equal(t, []byte("trunk-data"), got) + require.Equal(t, uint64(1), c.trunkHits.Load()) + require.Equal(t, uint64(0), c.tailHits.Load(), "depth-2 account branch must not land in the tail") + + // Flood the tail well past capacity with deep (5-nibble) keys; the resident + // trunk entry must not be evicted. + for i := 0; i < 100; i++ { + c.Put([]byte{0x10, byte(i), byte(i)}, []byte{byte(i)}, 0, 100) // odd flag, 5 nibbles → tail + } + got, _, ok = c.Get(trunkKey) + require.True(t, ok, "resident trunk entry must survive tail eviction pressure") + require.Equal(t, []byte("trunk-data"), got) + + // An unwind below the entry's txN invalidates it lazily on next Get. + c.Unwind(60) + _, _, ok = c.Get(trunkKey) + require.False(t, ok, "trunk entry with txN=100 must drop at unwind floor 60") +} + +// TestBranchCache_StorageTrunkPin verifies PinEntry routes a storage-trunk +// prefix (>= 64 nibbles) into its per-contract storage trunk, is served from +// the pinned tier, counts toward PinnedCount, and honors the unwind model. +func TestBranchCache_StorageTrunkPin(t *testing.T) { + c := NewBranchCache(100) + + // 33-byte compact prefix: even flag (0x00) + 32-byte account hash = 64 + // nibbles exactly → the storage trunk's depth-0 slot for that contract. + prefix := make([]byte, 33) + for i := 1; i < 33; i++ { + prefix[i] = byte(i) + } + c.PinEntry(prefix, []byte("storage-root"), 0, 100) + require.Equal(t, 1, c.PinnedCount()) + + got, _, ok := c.Get(prefix) + require.True(t, ok) + require.Equal(t, []byte("storage-root"), got) + require.Equal(t, uint64(1), c.pinnedHits.Load()) + + c.Unwind(60) + _, _, ok = c.Get(prefix) + require.False(t, ok, "pinned storage-trunk entry with txN=100 must drop at unwind floor 60") +} + // TestBranchCache_RootPinning verifies the root branch lands in the pinned // slot (counted as root-hit) and tail entries land in the LRU tier // (counted as tail-hit). @@ -92,10 +147,11 @@ func TestBranchCache_Invalidate(t *testing.T) { // TestBranchCache_Clear empties everything and resets stats. func TestBranchCache_Clear(t *testing.T) { c := NewBranchCache(100) + deepKey := []byte{0x12, 0x34, 0x56} // 5 nibbles → LRU tail c.Put([]byte{0x00}, []byte("r"), 0, 0) - c.Put([]byte{0x12}, []byte("d"), 0, 0) + c.Put(deepKey, []byte("d"), 0, 0) _, _, _ = c.Get([]byte{0x00}) - _, _, _ = c.Get([]byte{0x12}) + _, _, _ = c.Get(deepKey) require.Equal(t, uint64(1), c.rootHits.Load()) require.Equal(t, uint64(1), c.tailHits.Load()) @@ -105,38 +161,23 @@ func TestBranchCache_Clear(t *testing.T) { require.Equal(t, uint64(0), c.tailHits.Load()) _, _, ok := c.Get([]byte{0x00}) require.False(t, ok) - _, _, ok = c.Get([]byte{0x12}) + _, _, ok = c.Get(deepKey) require.False(t, ok) } -// TestBranchCache_StateKeyNeverCached pins the invariant that the commitment -// state checkpoint key bypasses every tier: Put is a no-op, Get always misses, -// and Invalidate neither panics nor disturbs real entries. -func TestBranchCache_StateKeyNeverCached(t *testing.T) { - c := NewBranchCache(100) - - c.Put(KeyCommitmentState, []byte("checkpoint"), 1, 1) - _, _, ok := c.Get(KeyCommitmentState) - require.False(t, ok, "state key must never be served from the cache") - require.Equal(t, 0, c.tail.Len(), "state key must not occupy a tail slot") - - deepKey := []byte{0x12, 0x34} - c.Put(deepKey, []byte("d"), 0, 0) - c.Invalidate(KeyCommitmentState) - got, _, ok := c.Get(deepKey) - require.True(t, ok, "invalidating the state key must not evict real entries") - require.Equal(t, []byte("d"), got) -} - // TestBranchCache_Stats verifies the format of the stats string is // deterministic and contains the expected per-tier counts. func TestBranchCache_Stats(t *testing.T) { c := NewBranchCache(100) + // 3-byte odd-flag prefixes are 5 nibbles deep → LRU tail (the account + // trunk only holds depths 1-4). + tailHit := []byte{0x12, 0x34, 0x56} + tailMiss := []byte{0x12, 0x34, 0x57} c.Put([]byte{0x00}, []byte("rrr"), 0, 0) - c.Put([]byte{0x12, 0x34}, []byte("ddd"), 0, 0) + c.Put(tailHit, []byte("ddd"), 0, 0) _, _, _ = c.Get([]byte{0x00}) - _, _, _ = c.Get([]byte{0x12, 0x34}) - _, _, _ = c.Get([]byte{0xff}) // tail miss + _, _, _ = c.Get(tailHit) + _, _, _ = c.Get(tailMiss) // tail miss s := c.Stats() for _, want := range []string{ @@ -145,6 +186,9 @@ func TestBranchCache_Stats(t *testing.T) { } { require.Contains(t, s, want, "Stats output: %s", s) } + // New format carries the trunk and pin tiers. + require.Contains(t, s, "trunk hit=", "Stats output: %s", s) + require.Contains(t, s, "pin hit=", "Stats output: %s", s) // Sanity: format doesn't blow up if we read it require.True(t, strings.HasPrefix(s, "branch-cache ")) } @@ -240,58 +284,3 @@ func TestBranchCache_Unwind_FrozenSurvives(t *testing.T) { _, _, ok := c.Get(key) require.True(t, ok, "frozen txN=0 entry must survive any positive-txN unwind") } - -// TestBranchCache_ShardedTailUnwindAcrossShards drives a lazy Unwind over many -// tail entries spread across the sharded tail, pinning that invalidation by txN -// floor works at scale: every stale entry (txN >= floor) drops on its next Get -// and every fresh one survives, regardless of which shard it landed in. -func TestBranchCache_ShardedTailUnwindAcrossShards(t *testing.T) { - c := NewBranchCache(DefaultBranchCacheTailCapacity) - - const n = 2000 - const watermark = 1000 - for i := 0; i < n; i++ { - prefix := []byte{0x01, byte(i), byte(i >> 8)} - c.Put(prefix, []byte{byte(i)}, 0, uint64(i)) - } - - // Lazy unwind: bump the epoch and lower the floor to watermark (the first - // unwound txN). Stale entries (old epoch, txN >= floor) are dropped on their - // next Get, across all tail shards — no eager scan. - c.Unwind(watermark) - - for i := 0; i < n; i++ { - prefix := []byte{0x01, byte(i), byte(i >> 8)} - _, _, ok := c.Get(prefix) - if uint64(i) >= watermark { - require.False(t, ok, "entry txN=%d must be dropped by floor=%d", i, watermark) - } else { - require.True(t, ok, "entry txN=%d must survive floor=%d", i, watermark) - } - } -} - -// TestBranchCache_BaselineFootprint pins that a freshly constructed cache is -// cheap. One is allocated per aggregator and may linger after Close, so an -// empty cache must not carry a multi-megabyte fixed backing for its LRU tail. -func TestBranchCache_BaselineFootprint(t *testing.T) { - const ( - n = 128 - maxBytesPerCache = 256 * 1024 - ) - caches := make([]*BranchCache, n) - - var before, after runtime.MemStats - runtime.GC() - runtime.ReadMemStats(&before) - for i := range caches { - caches[i] = NewBranchCache(DefaultBranchCacheTailCapacity) - } - runtime.GC() - runtime.ReadMemStats(&after) - runtime.KeepAlive(caches) - - perCache := (after.HeapAlloc - before.HeapAlloc) / n - require.Less(t, perCache, uint64(maxBytesPerCache), - "fresh BranchCache baseline is %d KiB/cache, want < %d KiB", perCache/1024, maxBytesPerCache/1024) -} diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index f824a301855..3a77454fda9 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -219,6 +219,8 @@ func (sdc *SharedDomainsCommitmentContext) trieContext(tx kv.TemporalTx, blockNu stepSize: sdc.sharedDomains.StepSize(), txNum: txNum, blockNum: blockNum, + probeSd: sdc.sharedDomains, + probeTx: tx, } if sdc.stateReader != nil { mainTtx.stateReader = sdc.stateReader.CloneForWorker(readCtx, tx) @@ -830,6 +832,10 @@ type TrieContext struct { trace bool stateReader StateReader localCollector *etl.Collector // per-goroutine collector for concurrent PutBranch + + // Diagnostics-only — both nil for read-only / test contexts. + probeSd sd + probeTx kv.TemporalTx } // NewTrieContextRo creates a read-only TrieContext suitable for TrieReader lookups. @@ -851,6 +857,22 @@ func (sdc *TrieContext) Branch(pref []byte) ([]byte, kv.Step, error) { return common.Copy(enc), step, nil } +// ProbeStateLayers samples sd.mem, parent.mem and tx-direct (MDBX) for one +// key — divergence diagnostics. Returns empty / not-ok when constructed +// without a probe-capable SharedDomains (e.g. NewTrieContextRo). +func (sdc *TrieContext) ProbeStateLayers(domain kv.Domain, key []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) { + if sdc.probeSd == nil { + return + } + return sdc.probeSd.ProbeReadLayers(domain, sdc.probeTx, key) +} + +// SiteIdentity tags cache entries with the SD lineage that produced them so +// divergence diagnostics can tell parent-SD writes from fork-SD writes. +func (sdc *TrieContext) SiteIdentity() string { + return fmt.Sprintf("sd=%p", sdc.probeSd) +} + func (sdc *TrieContext) PutBranch(prefix []byte, data []byte, prevData []byte) error { if sdc.stateReader.WithHistory() { // do not store branches if explicitly operate on history return nil diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 7ca0f3716a7..72e1c84fa7e 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -893,6 +893,7 @@ func (hph *HexPatriciaHashed) witnessComputeCellHashWithStorage(cell *cell, dept } else { if !cell.loaded.storage() { hph.metrics.StorageLoad(cell.storageAddr[:cell.storageAddrLen]) + diskLoadStorage.Add(1) update, err := hph.storageFromCacheOrDB(cell.storageAddr[:cell.storageAddrLen]) if err != nil { return nil, storageRootHashIsSet, nil, err @@ -978,6 +979,7 @@ func (hph *HexPatriciaHashed) witnessComputeCellHashWithStorage(cell *cell, dept } // storage root update or extension update could invalidate older stateHash, so we need to reload state hph.metrics.AccountLoad(cell.accountAddr[:cell.accountAddrLen]) + diskLoadAccount.Add(1) update, err := hph.accountFromCacheOrDB(cell.accountAddr[:cell.accountAddrLen]) if err != nil { return nil, storageRootHashIsSet, storageRootHash[:], err @@ -1130,6 +1132,7 @@ func (hph *HexPatriciaHashed) computeCellHash(cell *cell, depth int16, buf []byt } // storage root update or extension update could invalidate older stateHash, so we need to reload state hph.metrics.AccountLoad(cell.accountAddr[:cell.accountAddrLen]) + diskLoadAccount.Add(1) update, err := hph.accountFromCacheOrDB(cell.accountAddr[:cell.accountAddrLen]) if err != nil { return nil, err @@ -1826,9 +1829,11 @@ func (hph *HexPatriciaHashed) needFolding(hashedKey []byte) bool { // Process-cumulative trie-compute counters feeding the KVReadLevelledMetrics // "skip ratio"/"reset ratio" Debug log at the end of ComputeCommitment. var ( - hadToLoad atomic.Uint64 - skippedLoad atomic.Uint64 - hadToReset atomic.Uint64 + hadToLoad atomic.Uint64 + skippedLoad atomic.Uint64 + hadToReset atomic.Uint64 + diskLoadStorage atomic.Uint64 + diskLoadAccount atomic.Uint64 ) var ( diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go new file mode 100644 index 00000000000..dfc2152c26b --- /dev/null +++ b/execution/commitment/preload.go @@ -0,0 +1,175 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package commitment + +import ( + "encoding/binary" + "fmt" + + "github.com/erigontech/erigon/common/log/v3" + "github.com/erigontech/erigon/execution/commitment/nibbles" +) + +// CommitmentReader does GetLatest on the CommitmentDomain. Decoupled from +// tx/aggregator types to keep this package free of db/state imports. +type CommitmentReader func(prefix []byte) (v []byte, step uint64, found bool, err error) + +// estimatedEntryOverheadBytes is the per-entry RAM cost beyond the encoded +// value itself: branchCacheEntry (~80 B), maphash slot + hash (~40 B), +// prefix slice (~24 B header + content), value slice header (~24 B). +const estimatedEntryOverheadBytes = 168 + +type pathDepth struct { + path []byte + depth int +} + +// ContractTrunkPreload holds the resumable state of a BFS preload for one +// contract's storage subtree. Not goroutine-safe. +type ContractTrunkPreload struct { + contractHash []byte + queue []pathDepth + pinnedPrefixes [][]byte + pinned int + usedBytes int + maxDepthReached int +} + +// NewContractTrunkPreload seeds a preload state at depth 64 (storage +// subtree root: keccak256(address), 32 bytes / 64 nibbles). +func NewContractTrunkPreload(contractHash []byte) (*ContractTrunkPreload, error) { + if len(contractHash) != 32 { + return nil, fmt.Errorf("NewContractTrunkPreload: contractHash must be 32 bytes, got %d", len(contractHash)) + } + contractNibbles := make([]byte, 64) + for i, b := range contractHash { + contractNibbles[2*i] = b >> 4 + contractNibbles[2*i+1] = b & 0x0f + } + return &ContractTrunkPreload{ + contractHash: contractHash, + queue: []pathDepth{{path: contractNibbles, depth: 64}}, + maxDepthReached: 64, + }, nil +} + +// Run advances the BFS one chunk, pinning branches until additionalBudgetBytes +// is exhausted or the queue is empty. Returns entries pinned THIS call, whether +// the preload is now complete, and any reader error. On error the partial pin +// set and queue position are preserved for a retry on the next Run. +func (p *ContractTrunkPreload) Run( + additionalBudgetBytes int, + reader CommitmentReader, + cache *BranchCache, + logger log.Logger, +) (newlyPinned int, queueEmpty bool, err error) { + if cache == nil { + return 0, false, fmt.Errorf("ContractTrunkPreload.Run: cache is nil") + } + if additionalBudgetBytes <= 0 { + return 0, len(p.queue) == 0, nil + } + + chunkUsedBytes := 0 + chunkPinned := 0 + + for len(p.queue) > 0 { + head := p.queue[0] + p.queue = p.queue[1:] + + prefix := nibbles.HexToCompact(head.path) + v, step, found, rerr := reader(prefix) + if rerr != nil { + return chunkPinned, false, fmt.Errorf("preload at depth %d: %w", head.depth, rerr) + } + if !found { + continue + } + + entryCost := estimatedEntryOverheadBytes + len(prefix) + len(v) + if chunkUsedBytes+entryCost > additionalBudgetBytes { + p.queue = append([]pathDepth{head}, p.queue...) + break + } + + cache.PinEntry(prefix, v, step, 0) + // HexToCompact may alias a reused buffer; copy for a stable Invalidate handle. + prefixCopy := make([]byte, len(prefix)) + copy(prefixCopy, prefix) + p.pinnedPrefixes = append(p.pinnedPrefixes, prefixCopy) + chunkUsedBytes += entryCost + chunkPinned++ + if head.depth > p.maxDepthReached { + p.maxDepthReached = head.depth + } + if logger != nil && (p.pinned+chunkPinned)%5000 == 0 { + logger.Info("[trunk-preload] progress", + "pinned", p.pinned+chunkPinned, "depth", head.depth, + "used_mb", (p.usedBytes+chunkUsedBytes)/(1<<20)) + } + + // Branch encoding: 2-byte touchMap || 2-byte bitmap || per-child data. + if len(v) < 4 { + continue + } + bitmap := binary.BigEndian.Uint16(v[2:4]) + for n := 0; n < 16; n++ { + if bitmap&(1<= 64; value may be empty). +// Bounds a wave's file fetch so the budget is guaranteed exhausted inside it. +const minEntryBytes = estimatedEntryOverheadBytes + 33 + +// maxStorageTrunkDepth: 64 (account path) + 64 (keccak256(slot)) = 128. +const maxStorageTrunkDepth = 128 + +type pathKey struct { + path []byte // nibble path (1 byte / nibble) + key []byte // HexToCompact(path) +} + +func toPathKey(path []byte) pathKey { + k := nibbles.HexToCompact(path) + kc := make([]byte, len(k)) + copy(kc, k) // HexToCompact result may alias a reused buffer + return pathKey{path: path, key: kc} +} + +// ContractTrunkPreloadParallel is the wave-BFS analogue of ContractTrunkPreload. +// It walks one depth-level per wave and resolves missing branches through a +// batched, file-only BatchBranchResolver (no MDBX in the hot path). Each Run +// advances zero or more waves bounded by stepBudgetBytes; partial waves are +// truncated to fit the budget and resumed on the next Run. +// +// dbBranches shadows file values for the same key — DB is authoritative for +// steps not yet flushed to files. Pass nil for cold-snapshot / file-only mode. +// +// Not goroutine-safe. The resolver is passed per-Run (not held) so callers +// can supply a fresh tx-scoped resolver each block. +type ContractTrunkPreloadParallel struct { + contractHash []byte + frontier []pathKey // paths to process at depth = nextDepth + pendingChildren []pathKey // accumulated children of pinned items at depth = nextDepth+1 + nextDepth int // depth of the next wave (starts at 64) + pinnedPrefixes [][]byte + pinned int + usedBytes int + maxDepthReached int + dbHitsPinned int +} + +// NewContractTrunkPreloadParallel seeds a preload at depth 64 (storage subtree root). +func NewContractTrunkPreloadParallel(contractHash []byte) (*ContractTrunkPreloadParallel, error) { + if len(contractHash) != 32 { + return nil, fmt.Errorf("NewContractTrunkPreloadParallel: contractHash must be 32 bytes, got %d", len(contractHash)) + } + contractHashCopy := make([]byte, len(contractHash)) + copy(contractHashCopy, contractHash) + return &ContractTrunkPreloadParallel{ + contractHash: contractHashCopy, + frontier: []pathKey{toPathKey(ContractNibbles(contractHashCopy))}, + nextDepth: 64, + maxDepthReached: 64, + }, nil +} + +// Run advances the wave-BFS until stepBudgetBytes is exhausted, the frontier +// is empty, or maxStorageTrunkDepth is reached. On resolver error the partial +// pin set and wave position survive for retry on the next Run. +func (p *ContractTrunkPreloadParallel) Run( + stepBudgetBytes int, + dbBranches map[string][]byte, + resolve BatchBranchResolver, + cache *BranchCache, + logger log.Logger, +) (newlyPinned int, queueEmpty bool, err error) { + if cache == nil { + return 0, false, fmt.Errorf("ContractTrunkPreloadParallel.Run: cache is nil") + } + if resolve == nil { + return 0, false, fmt.Errorf("ContractTrunkPreloadParallel.Run: resolver is nil") + } + if stepBudgetBytes <= 0 { + return 0, len(p.frontier) == 0, nil + } + + stepCap := p.usedBytes + stepBudgetBytes + chunkPinned := 0 + budgetHit := false + + // pin records the entry and queues its children. Returns false on budget hit. + pin := func(pk pathKey, v []byte, depth int, next *[]pathKey) bool { + cost := estimatedEntryCost(pk.key, v) + if p.usedBytes+cost > stepCap { + budgetHit = true + return false + } + cache.PinEntry(pk.key, v, 0, 0) + kc := make([]byte, len(pk.key)) + copy(kc, pk.key) + p.pinnedPrefixes = append(p.pinnedPrefixes, kc) + p.usedBytes += cost + p.pinned++ + chunkPinned++ + if depth > p.maxDepthReached { + p.maxDepthReached = depth + } + if logger != nil && p.pinned%5000 == 0 { + logger.Info("[trunk-preload-parallel] progress", + "pinned", p.pinned, "depth", depth, "used_mb", p.usedBytes/(1<<20)) + } + if len(v) >= 4 { // 2-byte touchMap || 2-byte afterMap || per-child data + bitmap := binary.BigEndian.Uint16(v[2:4]) + for n := 0; n < 16; n++ { + if bitmap&(1< 0 { + depth := p.nextDepth + // Ascending key order so the file-batch partition is contiguous-in-file. + sort.Slice(p.frontier, func(i, j int) bool { return bytes.Compare(p.frontier[i].key, p.frontier[j].key) < 0 }) + + var dbHits []pathKey + var dbVals [][]byte + var fileMiss []pathKey + dbHitsBytes := 0 + for _, pk := range p.frontier { + if v, ok := dbBranches[string(pk.key)]; ok { + dbHits = append(dbHits, pk) + dbVals = append(dbVals, v) + dbHitsBytes += estimatedEntryCost(pk.key, v) + } else { + fileMiss = append(fileMiss, pk) + } + } + + // Cap the file fetch by what the budget can absorb after dbHits. + var fileMissDeferred []pathKey + if fileBudget := stepCap - p.usedBytes - dbHitsBytes; fileBudget <= 0 { + fileMissDeferred = fileMiss + fileMiss = nil + } else if maxFileFetch := fileBudget/minEntryBytes + 1; maxFileFetch < len(fileMiss) { + fileMissDeferred = fileMiss[maxFileFetch:] + fileMiss = fileMiss[:maxFileFetch] + } + + var fileVals [][]byte + if len(fileMiss) > 0 { + keys := make([][]byte, len(fileMiss)) + for i := range fileMiss { + keys[i] = fileMiss[i].key + } + fileVals, err = resolve(keys) + if err != nil { + return chunkPinned, false, fmt.Errorf("preload at depth %d: %w", depth, err) + } + if len(fileVals) != len(keys) { + return chunkPinned, false, fmt.Errorf("preload at depth %d: resolver returned %d vals for %d keys", depth, len(fileVals), len(keys)) + } + } + + dbHitStop := len(dbHits) + for i, pk := range dbHits { + if !pin(pk, dbVals[i], depth, &p.pendingChildren) { + dbHitStop = i + break + } + p.dbHitsPinned++ + } + fileMissStop := len(fileMiss) + if !budgetHit { + for i, pk := range fileMiss { + v := fileVals[i] + if v == nil { + continue + } + if !pin(pk, v, depth, &p.pendingChildren) { + fileMissStop = i + break + } + } + } + + if budgetHit { + // Preserve un-pinned items at current depth; pendingChildren stays + // at depth+1 for when this depth is drained on a future Run. + rest := make([]pathKey, 0, len(dbHits)-dbHitStop+len(fileMiss)-fileMissStop+len(fileMissDeferred)) + rest = append(rest, dbHits[dbHitStop:]...) + rest = append(rest, fileMiss[fileMissStop:]...) + rest = append(rest, fileMissDeferred...) + p.frontier = rest + break + } + + if len(fileMissDeferred) > 0 { + // Defensive: !budgetHit should mean no truncation. Re-queue at current depth. + p.frontier = fileMissDeferred + } else { + p.frontier = p.pendingChildren + p.pendingChildren = nil + p.nextDepth++ + } + } + + queueEmpty = (len(p.frontier) == 0 && len(p.pendingChildren) == 0) || p.nextDepth > maxStorageTrunkDepth + if logger != nil && (chunkPinned > 0 || queueEmpty) { + logger.Info("[trunk-preload-parallel] step", + "contract_hash", fmt.Sprintf("%x", p.contractHash), + "step_budget_mb", stepBudgetBytes/(1<<20), + "used_mb_total", p.usedBytes/(1<<20), + "pinned_this_step", chunkPinned, + "pinned_total", p.pinned, + "db_hits_total", p.dbHitsPinned, + "max_depth_reached", p.maxDepthReached, + "queue_empty", queueEmpty, + "next_depth", p.nextDepth, + "frontier_size", len(p.frontier)) + } + return chunkPinned, queueEmpty, nil +} + +func (p *ContractTrunkPreloadParallel) PinnedTotal() int { return p.pinned } +func (p *ContractTrunkPreloadParallel) UsedBytes() int { return p.usedBytes } +func (p *ContractTrunkPreloadParallel) MaxDepthReached() int { return p.maxDepthReached } +func (p *ContractTrunkPreloadParallel) DbHitsPinned() int { return p.dbHitsPinned } +func (p *ContractTrunkPreloadParallel) ContractHash() []byte { return p.contractHash } + +func (p *ContractTrunkPreloadParallel) QueueRemaining() int { + return len(p.frontier) + len(p.pendingChildren) +} + +// PinnedPrefixes returns slices aliasing internal storage — do not mutate. +func (p *ContractTrunkPreloadParallel) PinnedPrefixes() [][]byte { return p.pinnedPrefixes } + +// PreloadContractTrunkParallel is the one-shot wrapper around +// NewContractTrunkPreloadParallel + Run. +func PreloadContractTrunkParallel( + contractHash []byte, + ramBudgetBytes int, + dbBranches map[string][]byte, + resolve BatchBranchResolver, + cache *BranchCache, + logger log.Logger, +) (pinned int, err error) { + if ramBudgetBytes <= 0 { + return 0, fmt.Errorf("PreloadContractTrunkParallel: ramBudgetBytes must be positive, got %d", ramBudgetBytes) + } + p, err := NewContractTrunkPreloadParallel(contractHash) + if err != nil { + return 0, err + } + if resolve == nil { + return 0, fmt.Errorf("PreloadContractTrunkParallel: resolver is nil") + } + pinned, queueEmpty, err := p.Run(ramBudgetBytes, dbBranches, resolve, cache, logger) + if logger != nil { + logger.Info("[trunk-preload-parallel] complete", + "contract_hash", fmt.Sprintf("%x", contractHash), + "ram_budget_mb", ramBudgetBytes/(1<<20), + "used_mb", p.UsedBytes()/(1<<20), + "pinned", pinned, + "db_hits_pinned", p.DbHitsPinned(), + "max_depth_reached", p.MaxDepthReached(), + "budget_exhausted", !queueEmpty, + "cache_pinned_total", cache.PinnedCount()) + } + return pinned, err +} diff --git a/execution/commitment/preload_parallel_test.go b/execution/commitment/preload_parallel_test.go new file mode 100644 index 00000000000..6cd3fe75e77 --- /dev/null +++ b/execution/commitment/preload_parallel_test.go @@ -0,0 +1,735 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package commitment + +import ( + "bytes" + "encoding/binary" + "errors" + "sort" + "testing" + + "github.com/erigontech/erigon/execution/commitment/nibbles" +) + +func hexNibbles(b []byte) []byte { + out := make([]byte, len(b)*2) + for i, x := range b { + out[2*i] = x >> 4 + out[2*i+1] = x & 0x0f + } + return out +} + +// branchVal builds a synthetic branch-node value: 2-byte touchMap (0) || +// 2-byte afterMap (the child bitmap) || zero-padding to size sz (>= 4). +func branchVal(afterMap uint16, sz int) []byte { + if sz < 4 { + sz = 4 + } + v := make([]byte, sz) + binary.BigEndian.PutUint16(v[2:4], afterMap) + return v +} + +// syntheticTree describes a contract storage subtree: path-string -> afterMap. +// Root is the 64-nibble path of the contract hash; a node R is present iff R is +// a key here, and R's children are R||n for each set bit n in afterMap[R]. +type syntheticTree map[string]uint16 + +func buildSyntheticTree(t *testing.T) (hash []byte, tree syntheticTree, allPaths [][]byte) { + t.Helper() + hash = make([]byte, 32) + for i := range hash { + hash[i] = 0x42 + } + root := string(hexNibbles(hash)) + // R(64) -> {1,2} ; R1(65) -> {3} ; R2(65) -> {4,5} ; + // R1.3(66) leaf ; R2.4(66) leaf ; R2.5(66) -> {6} ; R2.5.6(67) leaf. + r := []byte(root) + p := func(suffix ...byte) []byte { return append(append([]byte{}, r...), suffix...) } + tree = syntheticTree{ + string(p()): 0b110, // bits 1,2 + string(p(1)): 0b1000, // bit 3 + string(p(2)): 0b110000, // bits 4,5 + string(p(1, 3)): 0, + string(p(2, 4)): 0, + string(p(2, 5)): 0b1000000, // bit 6 + string(p(2, 5, 6)): 0, + } + for k := range tree { + allPaths = append(allPaths, []byte(k)) + } + return hash, tree, allPaths +} + +// fakeResolver returns a BatchBranchResolver backed by the synthetic tree. +// notFound (path-strings) are treated as absent from the file layer. +// valSz is the branch value size. If failOnKey is non-empty, the resolver +// returns an error when that key is requested. +func fakeResolver(tree syntheticTree, notFound map[string]bool, valSz int, failOnPath string) BatchBranchResolver { + return func(keys [][]byte) ([][]byte, error) { + // keys must be sorted ascending (the contract of BatchBranchResolver). + for i := 1; i < len(keys); i++ { + if bytes.Compare(keys[i-1], keys[i]) >= 0 { + return nil, errors.New("resolver got unsorted keys") + } + } + vals := make([][]byte, len(keys)) + for i, k := range keys { + path := string(nibbles.CompactToHex(k)) + if failOnPath != "" && path == failOnPath { + return nil, errors.New("synthetic resolver failure") + } + am, ok := tree[path] + if !ok || notFound[path] { + continue // nil + } + vals[i] = branchVal(am, valSz) + } + return vals, nil + } +} + +// breadthFirstOrder returns the synthetic tree's paths in the order +// PreloadContractTrunkParallel pins them: by depth, then by compact-key. +func breadthFirstOrder(tree syntheticTree, exclude map[string]bool) []string { + type pk struct { + path string + key []byte + } + var pks []pk + // reachability from root, honoring exclude (an excluded node stops descent — + // it and its subtree become unreachable) + root := "" + for p := range tree { + if root == "" || len(p) < len(root) { + root = p + } + } + reach := map[string]bool{} + var dfs func(p string) + dfs = func(p string) { + if exclude[p] { + return + } + am, ok := tree[p] + if !ok { + return + } + reach[p] = true + for n := 0; n < 16; n++ { + if am&(1< maxBatch { + maxBatch = len(keys) + } + return base(keys) + } + c := NewBranchCache(64) + n, err := PreloadContractTrunkParallel(hash, budget, nil, resolve, c, nil) + if err != nil { + t.Fatal(err) + } + if n < 1 || n > 3 { + t.Fatalf("pinned %d, expected 1..3 for a ~3-entry budget", n) + } + if maxBatch > 6 { + t.Fatalf("depth-65 wave (width 16) should have been capped to ~remaining/minEntryBytes; resolver saw a batch of %d", maxBatch) + } +} + +func TestPreloadParallel_DbHitsShadowFiles(t *testing.T) { + // A branch present in both dbBranches (fresh) and the file layer (stale) + // must resolve to the DB value — and the DB value's child bitmap must drive + // the descent. Tree: as buildSyntheticTree but with an extra leaf R1.7; the + // file value for R1 has bitmap {3} (so R1.7 unreachable via files), the DB + // value for R1 has bitmap {3,7} — so R1.7 should get pinned iff the DB value + // is the one used. + hash, tree, _ := buildSyntheticTree(t) + root := "" + for p := range tree { + if root == "" || len(p) < len(root) { + root = p + } + } + r1 := root + string([]byte{1}) + tree[r1+string([]byte{7})] = 0 // R1.7 leaf, present in the file layer + const valSz = 100 + + freshR1 := branchVal(0b10001000, valSz) // bits 3 and 7 + freshR1[4] = 0xAB // a marker so we can assert the exact bytes were pinned + dbBranches := map[string][]byte{string(nibbles.HexToCompact([]byte(r1))): freshR1} + + c := NewBranchCache(64) + n, err := PreloadContractTrunkParallel(hash, 1<<20, dbBranches, fakeResolver(tree, nil, valSz, ""), c, nil) + if err != nil { + t.Fatal(err) + } + if n != len(tree) { + t.Fatalf("pinned %d, want %d (whole tree reachable via the fresh R1 bitmap)", n, len(tree)) + } + // R1's cached value is the DB one, not the stale file one. + gotR1, _, ok := c.Get(nibbles.HexToCompact([]byte(r1))) + if !ok { + t.Fatal("R1 not pinned") + } + if !bytes.Equal(gotR1, freshR1) { + t.Fatalf("R1 cached value is not the DB value: got %x want %x", gotR1, freshR1) + } + // R1.7 is reachable only because the DB bitmap has bit 7 — it must be pinned. + if _, _, ok := c.Get(nibbles.HexToCompact([]byte(r1 + string([]byte{7})))); !ok { + t.Fatal("R1.7 should be pinned (the DB value of R1 has it as a child); the stale file bitmap was used instead") + } +} + +func TestNextSubtree(t *testing.T) { + cases := []struct{ in, want []byte }{ + {[]byte{0x01, 0x02}, []byte{0x01, 0x03}}, + {[]byte{0x01, 0xff}, []byte{0x02}}, + {[]byte{0x00}, []byte{0x01}}, + } + for _, c := range cases { + if got := NextSubtree(c.in); !bytes.Equal(got, c.want) { + t.Fatalf("NextSubtree(%x) = %x, want %x", c.in, got, c.want) + } + } + if NextSubtree([]byte{0xff, 0xff}) != nil { + t.Fatalf("NextSubtree(0xffff) should be nil") + } +} + +func TestContractTrunkKeyRanges(t *testing.T) { + hashA := make([]byte, 32) + for i := range hashA { + hashA[i] = byte(7*i + 3) + } + hashB := make([]byte, 32) + for i := range hashB { + hashB[i] = byte(251 - 3*i) + } + nibA := ContractNibbles(hashA) + nibB := ContractNibbles(hashB) + evenFrom, evenTo, oddFrom, oddTo := ContractTrunkKeyRanges(nibA) + inRange := func(k, from, to []byte) bool { + return bytes.Compare(k, from) >= 0 && (to == nil || bytes.Compare(k, to) < 0) + } + keyOf := func(contractNibbles, slotPath []byte) []byte { + return nibbles.HexToCompact(append(append([]byte{}, contractNibbles...), slotPath...)) + } + slotPaths := [][]byte{ + {}, // 64 — subtree root (even) + {0x0}, {0xf}, // 65 (odd) + {0x1, 0x2}, {0xf, 0xf}, // 66 (even) + {0x3, 0x4, 0x5}, // 67 (odd) + {0x6, 0x7, 0x8, 0x9}, // 68 (even) + {0xa, 0xb, 0xc, 0xd, 0xe}, // 69 (odd) + make([]byte, 64), // 128 (even) — deepest + } + for _, sp := range slotPaths { + k := keyOf(nibA, sp) + total := 64 + len(sp) + if total%2 == 0 { + if !inRange(k, evenFrom, evenTo) || inRange(k, oddFrom, oddTo) { + t.Fatalf("depth %d (even) branch %x: must be in [%x,%x), not in [%x,%x)", total, k, evenFrom, evenTo, oddFrom, oddTo) + } + } else { + if !inRange(k, oddFrom, oddTo) || inRange(k, evenFrom, evenTo) { + t.Fatalf("depth %d (odd) branch %x: must be in [%x,%x), not in [%x,%x)", total, k, oddFrom, oddTo, evenFrom, evenTo) + } + } + if got := nibbles.CompactToHex(k); !bytes.Equal(got, append(append([]byte{}, nibA...), sp...)) { + t.Fatalf("CompactToHex round-trip mismatch for slot %x", sp) + } + } + // A different contract's branches must be in neither of A's ranges. + for _, sp := range slotPaths[:6] { + k := keyOf(nibB, sp) + if inRange(k, evenFrom, evenTo) || inRange(k, oddFrom, oddTo) { + t.Fatalf("foreign-contract branch %x leaked into A's ranges", k) + } + } +} + +func TestPreloadParallel_ResolverError(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + root := "" + for p := range tree { + if root == "" || len(p) < len(root) { + root = p + } + } + c := NewBranchCache(64) + // Fail when R||1 (depth 65) is requested -> root pinned at depth 64, then error. + _, err := PreloadContractTrunkParallel(hash, 1<<20, nil, fakeResolver(tree, nil, 100, root+string([]byte{1})), c, nil) + if err == nil { + t.Fatal("expected error from the resolver") + } + if c.PinnedCount() == 0 { + t.Fatal("the depth-64 root should have been pinned before the depth-65 failure") + } +} + +// --- Resumable ContractTrunkPreloadParallel tests (Run-by-Run) --- + +// TestContractTrunkPreloadParallel_ResumeAcrossSteps confirms that splitting a +// full preload into multiple Run calls yields the same pinned set as a +// one-shot run with the equivalent total budget. +func TestContractTrunkPreloadParallel_ResumeAcrossSteps(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + const valSz = 100 + resolve := fakeResolver(tree, nil, valSz, "") + + // Reference: one-shot. + cRef := NewBranchCache(64) + if _, err := PreloadContractTrunkParallel(hash, 1<<20, nil, resolve, cRef, nil); err != nil { + t.Fatal(err) + } + + // Step-by-step: budget exactly one entry per Run (the budget is checked + // before the entry is pinned; with overhead we need at least one entry's + // worth per step to make progress). + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + // Each entry is ~estimatedEntryOverheadBytes + 33 + valSz; over-allocate a + // bit per step so we always pin at least one new entry. + perStep := 2 * (estimatedEntryOverheadBytes + 33 + valSz) + const maxSteps = 50 + var steps int + for ; steps < maxSteps; steps++ { + _, done, err := p.Run(perStep, nil, resolve, c, nil) + if err != nil { + t.Fatalf("step %d: %v", steps, err) + } + if done { + break + } + } + if steps >= maxSteps { + t.Fatalf("preload did not complete in %d steps; pinned=%d", maxSteps, p.PinnedTotal()) + } + if p.PinnedTotal() != len(tree) { + t.Fatalf("step-by-step pinned %d, want %d", p.PinnedTotal(), len(tree)) + } + if c.PinnedCount() != cRef.PinnedCount() { + t.Fatalf("step-by-step cache pinned %d != one-shot cache pinned %d", c.PinnedCount(), cRef.PinnedCount()) + } + // Spot-check: every path in the reference is in the step-by-step cache. + for path := range tree { + key := nibbles.HexToCompact([]byte(path)) + vRef, _, okRef := cRef.Get(key) + v, _, ok := c.Get(key) + if !okRef || !ok { + t.Fatalf("path %x: ref ok=%v, step ok=%v", path, okRef, ok) + } + if !bytes.Equal(v, vRef) { + t.Fatalf("path %x: step value differs from ref", path) + } + } +} + +// TestContractTrunkPreloadParallel_RunAfterCompleteIsNoOp confirms that once +// the BFS reaches an empty frontier, further Run calls are no-ops. +func TestContractTrunkPreloadParallel_RunAfterCompleteIsNoOp(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + const valSz = 100 + resolve := fakeResolver(tree, nil, valSz, "") + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + n1, done1, err := p.Run(1<<20, nil, resolve, c, nil) + if err != nil { + t.Fatal(err) + } + if !done1 { + t.Fatalf("expected done after full budget, got done=false (queue=%d)", p.QueueRemaining()) + } + if n1 != len(tree) { + t.Fatalf("first Run pinned %d, want %d", n1, len(tree)) + } + prevPinned := c.PinnedCount() + n2, done2, err := p.Run(1<<20, nil, resolve, c, nil) + if err != nil { + t.Fatal(err) + } + if !done2 { + t.Fatal("expected done on second Run") + } + if n2 != 0 { + t.Fatalf("second Run pinned %d new entries, want 0", n2) + } + if c.PinnedCount() != prevPinned { + t.Fatalf("cache pinned count changed across no-op Run: %d -> %d", prevPinned, c.PinnedCount()) + } +} + +// TestContractTrunkPreloadParallel_StepBudgetCaps confirms that a small step +// budget stops the BFS even when the frontier has more work — and the saved +// state has the queue position preserved for the next call. +func TestContractTrunkPreloadParallel_StepBudgetCaps(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + const valSz = 100 + resolve := fakeResolver(tree, nil, valSz, "") + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + // Budget for ~3 entries (depth-64 root + 2 depth-65 children). + rootKey := nibbles.HexToCompact(hexNibbles(hash)) + entry := estimatedEntryOverheadBytes + len(rootKey) + valSz + smallBudget := 3*entry + 10 + n1, done1, err := p.Run(smallBudget, nil, resolve, c, nil) + if err != nil { + t.Fatal(err) + } + if done1 { + t.Fatalf("expected NOT done after a 3-entry budget; got done=true (pinned=%d)", n1) + } + if n1 < 1 || n1 > 5 { + t.Fatalf("expected ~3 pinned this step, got %d", n1) + } + if p.QueueRemaining() == 0 { + t.Fatal("expected frontier to be non-empty after small-budget step") + } + // Now exhaust with a full follow-on budget. + _, done2, err := p.Run(1<<20, nil, resolve, c, nil) + if err != nil { + t.Fatal(err) + } + if !done2 { + t.Fatalf("expected done after large follow-on budget; queue=%d", p.QueueRemaining()) + } + // In our synthetic tree only 3 of 7 paths sit at depths 64-65; the rest + // require descending past the truncated wave. The cap-per-wave logic + // drops the truncated wave's tail (BFS-wise) but the children of the + // pinned ones progress on the next call. Cumulative pinned should be + // >= the step-1 count. + if p.PinnedTotal() <= n1 { + t.Fatalf("follow-on Run made no progress: pinned still %d (step-1 was %d)", p.PinnedTotal(), n1) + } +} + +// TestContractTrunkPreloadParallel_ResumeAfterResolverError confirms that a +// resolver error preserves the partial state — a retry on the next Run picks +// up from the same wave once the resolver is healthy again. +func TestContractTrunkPreloadParallel_ResumeAfterResolverError(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + root := "" + for p := range tree { + if root == "" || len(p) < len(root) { + root = p + } + } + const valSz = 100 + // Fail when R||1 is requested (a depth-65 key) — depth-64 wave succeeds. + failingResolve := fakeResolver(tree, nil, valSz, root+string([]byte{1})) + healthyResolve := fakeResolver(tree, nil, valSz, "") + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + _, done, err := p.Run(1<<20, nil, failingResolve, c, nil) + if err == nil { + t.Fatal("expected resolver error") + } + if done { + t.Fatal("Run with resolver error should return done=false") + } + // Depth-64 root should still be pinned (it was the previous wave). + if c.PinnedCount() == 0 { + t.Fatal("expected the depth-64 root pinned before the depth-65 wave failed") + } + preErrPinned := p.PinnedTotal() + // Retry with a healthy resolver — should pick up where we left off and + // finish. The previous partial wave will be re-attempted in the new + // call (the failing wave's frontier WAS advanced past the depth where + // the error fired — the error path returns before updating + // p.frontier/p.nextDepth, so retry sees the same depth's frontier). + n, done, err := p.Run(1<<20, nil, healthyResolve, c, nil) + if err != nil { + t.Fatalf("retry failed: %v", err) + } + if !done { + t.Fatalf("retry should complete the preload; queue=%d", p.QueueRemaining()) + } + if p.PinnedTotal()-preErrPinned != n { + t.Fatalf("PinnedTotal delta %d != Run pinned %d", p.PinnedTotal()-preErrPinned, n) + } + // Whole tree should be pinned by now. + if p.PinnedTotal() != len(tree) { + t.Fatalf("after retry pinned %d, want %d", p.PinnedTotal(), len(tree)) + } +} + +// TestContractTrunkPreloadParallel_DbBranchesPerStep confirms that dbBranches +// can change between Run calls (caller may pass a freshly-prefetched overlay +// per block) and that the freshest values shadow file values per call. +func TestContractTrunkPreloadParallel_DbBranchesPerStep(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + root := "" + for p := range tree { + if root == "" || len(p) < len(root) { + root = p + } + } + const valSz = 100 + resolve := fakeResolver(tree, nil, valSz, "") + + // On the first wave (depth 64) supply a fresh R value via dbBranches. + freshRoot := branchVal(tree[root], valSz) + freshRoot[4] = 0xAB + dbWave0 := map[string][]byte{string(nibbles.HexToCompact([]byte(root))): freshRoot} + + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + // Wave 0: pin the root using dbBranches (one entry budget). + rootKey := nibbles.HexToCompact([]byte(root)) + stepBudget := estimatedEntryOverheadBytes + len(rootKey) + valSz + 10 + if _, _, err := p.Run(stepBudget, dbWave0, resolve, c, nil); err != nil { + t.Fatal(err) + } + if p.DbHitsPinned() != 1 { + t.Fatalf("wave 0: expected 1 db-hit pinned, got %d", p.DbHitsPinned()) + } + gotRoot, _, ok := c.Get(rootKey) + if !ok { + t.Fatal("root not pinned after wave 0") + } + if !bytes.Equal(gotRoot, freshRoot) { + t.Fatalf("wave 0: root pinned with stale file value, expected fresh dbBranches value") + } + + // Wave 1: depth 65. Pass an empty dbBranches (file-only); resolver + // supplies stale-bitmap values. + if _, done, err := p.Run(1<<20, nil, resolve, c, nil); err != nil { + t.Fatal(err) + } else if !done { + t.Fatalf("expected done after large budget; queue=%d", p.QueueRemaining()) + } + if p.DbHitsPinned() != 1 { + t.Fatalf("expected db-hit count to remain 1, got %d", p.DbHitsPinned()) + } + if p.PinnedTotal() != len(tree) { + t.Fatalf("after wave 1 pinned %d, want %d", p.PinnedTotal(), len(tree)) + } +} + +// TestContractTrunkPreloadParallel_PinnedPrefixesAccumulate confirms that +// PinnedPrefixes() accumulates across Run calls (needed for demote-time +// cache invalidation in the adaptive controller). +func TestContractTrunkPreloadParallel_PinnedPrefixesAccumulate(t *testing.T) { + hash, tree, _ := buildSyntheticTree(t) + const valSz = 100 + resolve := fakeResolver(tree, nil, valSz, "") + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + rootKey := nibbles.HexToCompact(hexNibbles(hash)) + entry := estimatedEntryOverheadBytes + len(rootKey) + valSz + // Two small steps then one big step. + for i := 0; i < 2; i++ { + if _, _, err := p.Run(2*entry+10, nil, resolve, c, nil); err != nil { + t.Fatal(err) + } + } + if _, done, err := p.Run(1<<20, nil, resolve, c, nil); err != nil { + t.Fatal(err) + } else if !done { + t.Fatal("expected done after large step") + } + prefixes := p.PinnedPrefixes() + if len(prefixes) != p.PinnedTotal() { + t.Fatalf("PinnedPrefixes len %d != PinnedTotal %d", len(prefixes), p.PinnedTotal()) + } + // Every prefix must be in the cache. + for _, pf := range prefixes { + if _, _, ok := c.Get(pf); !ok { + t.Fatalf("prefix %x in PinnedPrefixes but not in cache", pf) + } + } + // All prefixes are unique. + seen := map[string]bool{} + for _, pf := range prefixes { + if seen[string(pf)] { + t.Fatalf("duplicate prefix %x in PinnedPrefixes", pf) + } + seen[string(pf)] = true + } +} + +// TestContractTrunkPreloadParallel_NilCacheError + NilResolverError confirm +// the input-validation guards. +func TestContractTrunkPreloadParallel_NilCacheError(t *testing.T) { + hash := make([]byte, 32) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + resolve := func(keys [][]byte) ([][]byte, error) { return make([][]byte, len(keys)), nil } + if _, _, err := p.Run(1<<20, nil, resolve, nil, nil); err == nil { + t.Fatal("expected error when cache is nil") + } +} + +func TestContractTrunkPreloadParallel_NilResolverError(t *testing.T) { + hash := make([]byte, 32) + c := NewBranchCache(64) + p, err := NewContractTrunkPreloadParallel(hash) + if err != nil { + t.Fatal(err) + } + if _, _, err := p.Run(1<<20, nil, nil, c, nil); err == nil { + t.Fatal("expected error when resolver is nil") + } +} + +func TestContractTrunkPreloadParallel_BadHashLengthError(t *testing.T) { + if _, err := NewContractTrunkPreloadParallel(make([]byte, 31)); err == nil { + t.Fatal("expected error for 31-byte hash") + } + if _, err := NewContractTrunkPreloadParallel(make([]byte, 33)); err == nil { + t.Fatal("expected error for 33-byte hash") + } +} diff --git a/execution/commitment/preload_ranges.go b/execution/commitment/preload_ranges.go new file mode 100644 index 00000000000..3e2f9b8406d --- /dev/null +++ b/execution/commitment/preload_ranges.go @@ -0,0 +1,59 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package commitment + +import "github.com/erigontech/erigon/execution/commitment/nibbles" + +// NextSubtree: exclusive upper bound of a prefix-range scan over `in`. Returns +// nil if `in` is all 0xff. Mirrors db/kv.NextSubtree (inlined to keep imports minimal). +func NextSubtree(in []byte) []byte { + r := make([]byte, len(in)) + copy(r, in) + for i := len(r) - 1; i >= 0; i-- { + if r[i] != 0xff { + r[i]++ + return r[:i+1] + } + } + return nil +} + +// ContractTrunkKeyRanges returns the two CommitmentDomain key ranges that +// together cover every branch node of a contract's storage subtree. +// +// The commitment domain keys branches by HexToCompact(nibblePath); HexToCompact's +// flag byte differs by the parity of the path length, so a contract's storage +// branches (path = keccak256(addr)'s 64 nibbles ++ k slot-path nibbles, total +// 64+k) split into two non-adjacent byte ranges: +// - even total length (k even, incl. the depth-64 subtree root): +// key = 0x00 || H || ⇒ prefix 0x00||H (33 bytes) +// - odd total length (k odd): +// all such keys lie in [HexToCompact(H||0), NextSubtree(HexToCompact(H||15))). +func ContractTrunkKeyRanges(contractNibbles []byte) (evenFrom, evenTo, oddFrom, oddTo []byte) { + evenFrom = nibbles.HexToCompact(contractNibbles) // 0x00 || H, 33 bytes + evenTo = NextSubtree(evenFrom) + + odd0 := make([]byte, 0, len(contractNibbles)+1) + odd0 = append(append(odd0, contractNibbles...), 0) + oddF := make([]byte, 0, len(contractNibbles)+1) + oddF = append(append(oddF, contractNibbles...), 15) + oddFrom = nibbles.HexToCompact(odd0) + oddTo = NextSubtree(nibbles.HexToCompact(oddF)) + return evenFrom, evenTo, oddFrom, oddTo +} + +// ContractNibbles expands a 32-byte hash to its 64-nibble path (high nibble first). +func ContractNibbles(contractHash []byte) []byte { + out := make([]byte, len(contractHash)*2) + for i, b := range contractHash { + out[2*i] = b >> 4 + out[2*i+1] = b & 0x0f + } + return out +} diff --git a/execution/commitment/trunk_pin_metrics.go b/execution/commitment/trunk_pin_metrics.go new file mode 100644 index 00000000000..b4d3000eb43 --- /dev/null +++ b/execution/commitment/trunk_pin_metrics.go @@ -0,0 +1,44 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +package commitment + +import ( + "github.com/erigontech/erigon/diagnostics/metrics" +) + +// Per-contract labels omitted to keep cardinality bounded; per-contract +// detail is in the [adaptive-pin] structured log line. + +var ( + mxPinnedHits = metrics.GetOrCreateCounter("commitment_branchcache_pinned_hits_total") + mxPinnedMisses = metrics.GetOrCreateCounter("commitment_branchcache_pinned_misses_total") + mxPinnedEntries = metrics.GetOrCreateGauge("commitment_branchcache_pinned_entries") + + mxAdaptivePromoted = metrics.GetOrCreateCounter("commitment_adaptive_pin_promoted_total") + mxAdaptiveExtended = metrics.GetOrCreateCounter("commitment_adaptive_pin_extended_total") + mxAdaptiveDemoted = metrics.GetOrCreateCounter("commitment_adaptive_pin_demoted_total") + mxAdaptiveActive = metrics.GetOrCreateGauge("commitment_adaptive_pin_active_contracts") + + mxPreloadDurationSecondsTotal = metrics.GetOrCreateCounter("commitment_trunk_preload_duration_seconds_total") + mxPreloadBytesTotal = metrics.GetOrCreateCounter("commitment_trunk_preload_bytes_total") +) + +// PublishMetrics emits counter deltas (last-published tracked internally) and +// sets gauges absolute. Call once per SD.Flush — once-per-batch avoids hot-path cost. +func (c *BranchCache) PublishMetrics() { + hits := c.pinnedHits.Load() + misses := c.pinnedMisses.Load() + if delta := hits - c.lastPublishedPinnedHits.Swap(hits); delta > 0 { + mxPinnedHits.AddUint64(delta) + } + if delta := misses - c.lastPublishedPinnedMisses.Swap(misses); delta > 0 { + mxPinnedMisses.AddUint64(delta) + } + mxPinnedEntries.SetUint64(uint64(c.pinnedEntries.Load())) +} diff --git a/execution/commitment/warmuper.go b/execution/commitment/warmuper.go index abec0bb2f0c..d65a5a7dadf 100644 --- a/execution/commitment/warmuper.go +++ b/execution/commitment/warmuper.go @@ -31,6 +31,20 @@ import ( "github.com/erigontech/erigon/execution/commitment/nibbles" ) +// Warmer branch-read outcome counters. Hit: the branch read returned +// >= 4 bytes; Empty: returned nothing or unparseable. Used to size the +// value of bypassing the xorfilter in this call path. +var ( + warmerBranchHitCount atomic.Uint64 + warmerBranchEmptyCount atomic.Uint64 +) + +// WarmerBranchOutcomeStats returns process-cumulative counts. Snapshot +// before/after for per-block deltas. +func WarmerBranchOutcomeStats() (hit, empty uint64) { + return warmerBranchHitCount.Load(), warmerBranchEmptyCount.Load() +} + // TrieContextFactory creates new PatriciaContext instances for parallel warmup. type TrieContextFactory func() (PatriciaContext, func()) @@ -163,8 +177,10 @@ func (w *Warmuper) warmupKey(trieCtx PatriciaContext, hashedKey []byte, startDep // Branch data format: 2-byte touch map + 2-byte bitmap + per-child data if len(branchData) < 4 { + warmerBranchEmptyCount.Add(1) break } + warmerBranchHitCount.Add(1) if depth >= len(hashedKey) { break diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 988dde84ae5..4020dbb621b 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -231,7 +231,9 @@ type ExecModule struct { publishedSD func() *execctx.SharedDomains // fallback for background commit // stateCache is a cache for state data (accounts, storage, code) - stateCache *cache.StateCache + stateCache *cache.StateCache + // codeStore is the persistent codehash-keyed code cache (in-mem + MDBX backing). + codeStore *cache.CodeStore readAheader *exec.BlockReadAheader stopNode func() error @@ -268,6 +270,10 @@ func NewExecModule( if domainCache == nil { domainCache = cache.NewDefaultStateCache() } + var codeStore *cache.CodeStore + if dbg.UseCodeStore { + codeStore = cache.NewCodeStore(cache.DefaultCodeStoreMemBytes, cache.DefaultCodeStoreTableBytes) + } forkValidator := newForkValidator(ctx, currentBlockNumber, pipelineExecutor, blockReader, syncCfg.MaxReorgDepth) em := &ExecModule{ @@ -289,6 +295,7 @@ func NewExecModule( fcuBackgroundCommit: fcuBackgroundCommit, onlySnapDownloadOnStart: onlySnapDownloadOnStart, stateCache: domainCache, + codeStore: codeStore, readAheader: readAheader, stopNode: stopNode, } @@ -549,7 +556,40 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b } var tx kv.TemporalRwTx = doms.BlockOverlay() - // Chain to the canonical generation so head-extending reads and fork unwind sets resolve via the parent link. + // DO NOT CHANGE THIS WITHOUT WORKING THROUGH THE UNWIND CACHING SCENARIOS. + // The earlier `header.ParentHash == ReadHeadBlockHash(tx)` head-extending- + // only gate has been intentionally widened back to "chain whenever a + // currentContext exists" because fork-payload caching needs the parent + // link too — see the two-role breakdown below. The narrower gate was + // merged from main during the post-#21017 rebase and is the WRONG choice + // for this branch; keep the wider gate. + // + // Chain the validation SD to the latest in-memory canonical generation: + // e.currentContext when present, otherwise the newest in-flight commit + // generation (gate item 2 — the prior FCU cleared currentContext and + // handed its SD to the background commit). + // + // The parent link serves two roles: + // + // 1. Head-extending payloads read the canonical generation's + // not-yet-committed domain state instead of stale MDBX. + // + // 2. Fork payloads: unwindToCommonCanonical below must build an unwind + // set, and the diffsets of the canonical blocks it unwinds live in + // the canonical generation's pastChangesAccumulator — reachable only + // through this parent link (GetDiffset chains to the parent). Without + // it the unwind silently runs with no unwind set, leaving the + // BranchCache unmasked and corrupting the computed root. + // + // For a fork payload the parent does NOT shadow the unwound base: once + // unwindToCommonCanonical has run, doms.mem.unwindChangeset holds every + // key the unwound canonical blocks touched, and TemporalMemBatch.getLatest + // resolves those from the unwind set before ever consulting the parent. + // + // Cherry-pick note: the upstream commit also chained to e.latestGen() + // (the gate-2 in-flight commit generation) when currentContext is nil; + // that generation chain is not on this branch, so currentContext is the + // only canonical generation here. if e.currentContext != nil { doms.SetParent(e.currentContext) } @@ -570,6 +610,7 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b // Set state cache in SharedDomains for use during state reading doms.SetStateCache(e.stateCache) + doms.SetCodeStore(e.codeStore) if err = e.unwindToCommonCanonical(doms, tx, header); err != nil { doms.Close() return ValidationResult{}, err diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index 3cd70394975..437a7428547 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -224,6 +224,7 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa // ValidateChain (fork validation, exec_module.go) set this, leaving // the canonical execution path running uncached against the aggTx. currentContext.SetStateCache(e.stateCache) + currentContext.SetCodeStore(e.codeStore) } // Clear the published overlay before closing the SD, so concurrent @@ -566,6 +567,7 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa } freshSD.SetInMemHistoryReads(inMemHistoryReads) freshSD.SetStateCache(e.stateCache) + freshSD.SetCodeStore(e.codeStore) if err := freshSD.InitBlockOverlay(roTx, roTx.Debug().Dirs().Tmp); err != nil { roTx.Rollback() freshSD.Close() @@ -892,6 +894,11 @@ func (e *ExecModule) runForkchoicePrune(initialCycle bool) ([]any, error) { pruneTimeout = maxTimeout } if err := agg.CollateAndPrune(e.bacgroundCtx, e.db, func(tx kv.TemporalRwTx) error { + if e.codeStore != nil { + if err := e.codeStore.Evict(tx); err != nil { + return err + } + } return e.pipelineExecutor.RunPrune(e.bacgroundCtx, tx, initialCycle, pruneTimeout) }, e.logger); err != nil { return nil, err diff --git a/execution/execmodule/set_head.go b/execution/execmodule/set_head.go index 36b245ac47d..9a9c11c2004 100644 --- a/execution/execmodule/set_head.go +++ b/execution/execmodule/set_head.go @@ -109,6 +109,7 @@ func (e *ExecModule) SetHead(ctx context.Context, targetBlock uint64) error { // and the next FCU re-execution reads them and computes a stale state root // (BadBlock). Mirrors ValidateChain/forkchoice. sd.SetStateCache(e.stateCache) + sd.SetCodeStore(e.codeStore) // Drain in-flight warmup before the unwind bumps the cache epoch, so a // fire-and-forget warmup can't Put a dead-fork value stamped with the new diff --git a/execution/stagedsync/rawdbreset/reset_stages.go b/execution/stagedsync/rawdbreset/reset_stages.go index b796fc79894..1889d4ddac2 100644 --- a/execution/stagedsync/rawdbreset/reset_stages.go +++ b/execution/stagedsync/rawdbreset/reset_stages.go @@ -187,7 +187,12 @@ func ResetExec(ctx context.Context, db kv.TemporalRwDB) (err error) { return err } - // Wiping the commitment table makes branchCache entries stale; drop it so it repopulates from the wiped table. + // Wiping the commitment table leaves the aggregator's in-memory branchCache + // referencing trie nodes that no longer exist on disk. A subsequent from-0 + // re-exec then reads those stale nodes when computing block 0's commitment + // and produces a wrong trie root (parallel-exec failure mode of #21138). + // Drop the cache so it repopulates from the freshly-wiped table. + branchCacheCleared := false if hasAgg, ok := db.(dbstate.HasAgg); ok { if agg, ok := hasAgg.Agg().(*dbstate.Aggregator); ok { aggTx := agg.BeginFilesRo() @@ -195,8 +200,12 @@ func ResetExec(ctx context.Context, db kv.TemporalRwDB) (err error) { if bc := aggTx.BranchCache(); bc != nil { bc.Clear() } + branchCacheCleared = true } } + if !branchCacheCleared { + log.Warn("[reset] commitment branch cache not cleared after wiping the table (no *state.Aggregator); a from-0 re-exec may read stale commitment nodes and produce a wrong trie root") + } return nil } diff --git a/execution/state/rw_v3.go b/execution/state/rw_v3.go index 7af3f2c1612..69c1521e21a 100644 --- a/execution/state/rw_v3.go +++ b/execution/state/rw_v3.go @@ -35,6 +35,7 @@ import ( "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/db/rawdb/rawtemporaldb" "github.com/erigontech/erigon/db/state/execctx" + "github.com/erigontech/erigon/execution/cache" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/types" "github.com/erigontech/erigon/execution/types/accounts" @@ -1010,6 +1011,18 @@ func NewReaderV3(getter kv.TemporalGetter) *ReaderV3 { } } +// CodeStore returns the codehash-keyed code cache + backing tx when the reader's +// getter exposes one, so callers holding an authoritative codehash can serve a +// code read without the addr-keyed CodeDomain decompression. +func (r *ReaderV3) CodeStore() (*cache.CodeStore, kv.TemporalTx) { + if g, ok := r.getter.(interface { + CodeStore() (*cache.CodeStore, kv.TemporalTx) + }); ok { + return g.CodeStore() + } + return nil, nil +} + func (r *ReaderV3) DiscardReadList() {} func (r *ReaderV3) SetTxNum(txNum uint64) { r.txNum = txNum } func (r *ReaderV3) SetGetter(getter kv.TemporalGetter) { r.getter = getter } diff --git a/execution/state/state_object.go b/execution/state/state_object.go index ea7b0b9ee21..8093ac725d2 100644 --- a/execution/state/state_object.go +++ b/execution/state/state_object.go @@ -35,6 +35,8 @@ import ( "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/empty" "github.com/erigontech/erigon/common/u256" + "github.com/erigontech/erigon/db/kv" + "github.com/erigontech/erigon/execution/cache" "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/execution/tracing" "github.com/erigontech/erigon/execution/types/accounts" @@ -426,6 +428,21 @@ func (so *stateObject) Code() ([]byte, error) { } } } + + ch := so.data.CodeHash.Value() + var codeStore *cache.CodeStore + if cs, ok := so.db.stateReader.(interface { + CodeStore() (*cache.CodeStore, kv.TemporalTx) + }); ok { + var tx kv.TemporalTx + if codeStore, tx = cs.CodeStore(); codeStore != nil { + if code, ok := codeStore.GetByHash(tx, ch[:]); ok { + so.code = code + return code, nil + } + } + } + if dbg.TraceDomainIO || (dbg.TraceTransactionIO && (so.db.trace || dbg.TraceAccount(so.address.Handle()))) { so.db.stateReader.SetTrace(true, fmt.Sprintf("%d (%d.%d)", so.db.blockNum, so.db.txIndex, so.db.version)) } @@ -444,6 +461,9 @@ func (so *stateObject) Code() ([]byte, error) { return nil, fmt.Errorf("can't read code for %x: %w", so.Address(), err) } so.code = code + if codeStore != nil && len(code) > 0 { + codeStore.SetMem(ch[:], code) + } return code, nil } diff --git a/go.mod b/go.mod index 807b44d5bdd..531fefdfadf 100644 --- a/go.mod +++ b/go.mod @@ -79,6 +79,7 @@ require ( github.com/mark3labs/mcp-go v0.55.0 github.com/mattn/go-colorable v0.1.15 github.com/mattn/go-isatty v0.0.22 + github.com/maypok86/otter/v2 v2.3.0 github.com/miekg/dns v1.1.72 github.com/multiformats/go-multiaddr v0.16.1 github.com/nyaosorg/go-windows-shortcut v0.0.0-20220529122037-8b0c89bca4c4 diff --git a/go.sum b/go.sum index 20f4483575f..ce072f799c2 100644 --- a/go.sum +++ b/go.sum @@ -732,6 +732,8 @@ github.com/mattn/go-isatty v0.0.22/go.mod h1:ZXfXG4SQHsB/w3ZeOYbR0PrPwLy+n6xiMrJ github.com/mattn/go-runewidth v0.0.23 h1:7ykA0T0jkPpzSvMS5i9uoNn2Xy3R383f9HDx3RybWcw= github.com/mattn/go-runewidth v0.0.23/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/maypok86/otter/v2 v2.3.0 h1:8H8AVVFUSzJwIegKwv1uF5aGitTY+AIrtktg7OcLs8w= +github.com/maypok86/otter/v2 v2.3.0/go.mod h1:XgIdlpmL6jYz882/CAx1E4C1ukfgDKSaw4mWq59+7l8= github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U= From b5c5261831a36e49921fd16f8383d126fe397179 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 2 Jul 2026 09:42:03 +0000 Subject: [PATCH 02/18] execution, db: trim comments to project comment policy Comment-only cleanup (no behavior change) across the files this branch touches, per CLAUDE.md / .claude/rules/comments.md: drop issue/PR-reference narration, incident/rebase history, client name-drops, "NOTE:"/"WRONG choice" shouting, and "safety net" scope narration, keeping each comment's one-line technical "why". Pre-existing bare TODOs are left for a dedicated pass. --- execution/cache/cache_test.go | 5 ++--- execution/cache/code_cache_codehash_test.go | 2 +- execution/commitment/branch_cache.go | 2 +- execution/execmodule/exec_module.go | 22 +++++-------------- execution/execmodule/forkchoice.go | 8 +++---- .../stagedsync/rawdbreset/reset_stages.go | 6 ++--- 6 files changed, 17 insertions(+), 28 deletions(-) diff --git a/execution/cache/cache_test.go b/execution/cache/cache_test.go index 1fb00f30165..e3b4cefd096 100644 --- a/execution/cache/cache_test.go +++ b/execution/cache/cache_test.go @@ -273,8 +273,7 @@ func TestCodeCache_CodeDeduplication(t *testing.T) { func TestCodeCache_AddrCapacityLimit(t *testing.T) { // addrToHash is an LRU keyed by 20-byte address. Verify eviction is - // LRU rather than no-op-when-full — fresh-address workloads must - // warm up (geth's lru.Cache pattern, mirroring core/state/database_code.go). + // LRU rather than no-op-when-full so fresh-address workloads warm up. // makeAddr / makeCode wrap at 256, so we generate addrs/codes from // a wider 16-bit space directly. wideAddr := func(i int) []byte { @@ -365,7 +364,7 @@ func TestCodeCache_Clear(t *testing.T) { c.Clear() assert.Equal(t, 0, c.Len()) // Clear hard-resets every layer: unwound/cleared code must not remain - // discoverable (#21752), so the content layer is dropped too. + // discoverable, so the content layer is dropped too. assert.Equal(t, 0, c.CodeLen()) } diff --git a/execution/cache/code_cache_codehash_test.go b/execution/cache/code_cache_codehash_test.go index ccd416fa044..dd7feee771a 100644 --- a/execution/cache/code_cache_codehash_test.go +++ b/execution/cache/code_cache_codehash_test.go @@ -216,7 +216,7 @@ func BenchmarkCodeCache_GetByCodeHash_ManyAddrs_OneCode(b *testing.B) { } // TestCodeCache_Unwind_DropsUnwoundCodeEverywhere verifies the (txNum, epoch) -// model the user requires (#21752): code deployed on a fork that is later +// model: code deployed on a fork that is later // unwound must stop being discoverable on EVERY layer — addr→code, the // content-addressed codeHash→code, and the size layer — not just the addr // layer. The code's value is invariant for a hash, but its existence is not. diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 440ea76d508..51a2a7f9f14 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -651,7 +651,7 @@ func (c *BranchCache) Invalidate(prefix []byte) { // unwindToTxN (so old-epoch entries at or above it are dropped lazily on their // next Get). The floor only ever decreases, so a shallow unwind cannot // resurrect entries a deeper one invalidated. Mirrors GenericCache.Unwind so -// branch and state caches honor one (txN, epoch) model (#21752). +// branch and state caches honor one (txN, epoch) model. func (c *BranchCache) Unwind(unwindToTxN uint64) { c.coh.Unwind(unwindToTxN) } diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index 4020dbb621b..c3a14aba186 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -100,11 +100,6 @@ func GetBlockHashFromMissingSegmentError(err error) (common.Hash, bool) { // OnNewBlock is intentionally a no-op: in the embedded (non-remote) rpcdaemon // the SD is the authoritative source, so the coherent cache's state-tracking // machinery is unnecessary. -// -// This shim predates SharedDomains' current capabilities and will be simplified -// as part of #19623 (2-cache IBS rationalization) once the StateReader/CacheView -// interfaces stabilize. See also #19798 (event stream extraction) and #19855 -// (TransactionState/BlockState separation). type Cache struct { execModule *ExecModule publishedSD func() *execctx.SharedDomains // returns the latest published SD from Events (for background commit) @@ -544,10 +539,9 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b if err != nil { return ValidationResult{}, err } - // NOTE: do NOT defer doms.Close(). On the success path, ownership of - // doms transfers to forkValidator.sharedDom inside ValidatePayload — - // later phases (MergeExtendingFork, NotifyCurrentHeight) close it. - // We Close explicitly only on the early-return error paths below. + // Do not defer doms.Close(): on the success path ownership transfers to + // forkValidator.sharedDom inside ValidatePayload and later phases close it, + // so we Close explicitly only on the early-return error paths below. doms.SetInMemHistoryReads(inMemHistoryReads) if err := doms.InitBlockOverlay(roTx, roTx.Debug().Dirs().Tmp); err != nil { @@ -556,13 +550,9 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b } var tx kv.TemporalRwTx = doms.BlockOverlay() - // DO NOT CHANGE THIS WITHOUT WORKING THROUGH THE UNWIND CACHING SCENARIOS. - // The earlier `header.ParentHash == ReadHeadBlockHash(tx)` head-extending- - // only gate has been intentionally widened back to "chain whenever a - // currentContext exists" because fork-payload caching needs the parent - // link too — see the two-role breakdown below. The narrower gate was - // merged from main during the post-#21017 rebase and is the WRONG choice - // for this branch; keep the wider gate. + // Chain whenever a currentContext exists, not only when head-extending + // (header.ParentHash == head): fork-payload caching needs the parent link + // too — see the two-role breakdown below. // // Chain the validation SD to the latest in-memory canonical generation: // e.currentContext when present, otherwise the newest in-flight commit diff --git a/execution/execmodule/forkchoice.go b/execution/execmodule/forkchoice.go index 437a7428547..776ce232936 100644 --- a/execution/execmodule/forkchoice.go +++ b/execution/execmodule/forkchoice.go @@ -549,7 +549,7 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa if err != nil { return nil, nil, fmt.Errorf("updateForkChoice: begin rw after hasMore: %w", err) } - defer commitRwTx.Rollback() // safety net; idempotent after successful Commit + defer commitRwTx.Rollback() // idempotent after a successful Commit // The committed sd is spent; RunLoop closes it and continues on the // fresh SD built below (no ClearRam reuse). if err := sd.Commit(ctx, commitRwTx); err != nil { @@ -699,8 +699,8 @@ func (e *ExecModule) updateForkChoice(ctx context.Context, originalBlockHash, sa handOffSemaphore(func() error { defer bgSD.Close() // bgRoTx is rolled back inside runForkchoiceFlushCommit between - // Flush and Commit so the commit sees openTxs=1 in MDBX. This - // defer is a safety net — Rollback is idempotent. + // Flush and Commit so the commit sees openTxs=1 in MDBX; this + // defer is redundant (Rollback is idempotent). defer bgRoTx.Rollback() err := e.runPostForkchoice(bgSD, bgRoTx, finishProgressBefore, isSynced, initialCycle) // Signal that the DB commit is done — RPC consumers can @@ -834,7 +834,7 @@ func (e *ExecModule) dispatchNotificationsFromOverlay(sd *execctx.SharedDomains, // pinning them behind the still-open RO reader until the next commit. SD.Flush // only writes in-memory state to rwTx and does not read from the RO tx, so // closing it after Flush is safe. Rollback is idempotent, so callers keep their -// outer `defer roTx.Rollback()` as a safety net. +// outer `defer roTx.Rollback()` unchanged. func (e *ExecModule) runForkchoiceFlushCommit(sd *execctx.SharedDomains, roTxToCloseBeforeCommit kv.TemporalTx, finishProgressBefore uint64, isSynced bool) ([]any, error) { timings := make([]any, 0, 2) diff --git a/execution/stagedsync/rawdbreset/reset_stages.go b/execution/stagedsync/rawdbreset/reset_stages.go index 1889d4ddac2..5bec686ed0b 100644 --- a/execution/stagedsync/rawdbreset/reset_stages.go +++ b/execution/stagedsync/rawdbreset/reset_stages.go @@ -190,7 +190,7 @@ func ResetExec(ctx context.Context, db kv.TemporalRwDB) (err error) { // Wiping the commitment table leaves the aggregator's in-memory branchCache // referencing trie nodes that no longer exist on disk. A subsequent from-0 // re-exec then reads those stale nodes when computing block 0's commitment - // and produces a wrong trie root (parallel-exec failure mode of #21138). + // and produces a wrong trie root under parallel exec. // Drop the cache so it repopulates from the freshly-wiped table. branchCacheCleared := false if hasAgg, ok := db.(dbstate.HasAgg); ok { @@ -257,8 +257,8 @@ func clearStageProgress(tx kv.RwTx, stagesList ...stages.SyncStage) error { // "block 0 already done" — SeekCommitment then returns (1, 0), the exec // loop starts at block 1, and the block-0 init task that re-applies the // genesis allocation never runs. This breaks `stage_exec --reset` → - // `stage_exec` from-0 sync (parallel-exec drops genesis-allocated - // addresses that no subsequent block touches; see #21138). + // `stage_exec` from-0 sync (parallel exec drops genesis-allocated + // addresses that no subsequent block touches). for _, stage := range stagesList { if err := tx.Delete(kv.SyncStageProgress, []byte(stage)); err != nil { return err From fcfc99b9929c4586936cc52fbeda74c09fa56485 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 2 Jul 2026 11:10:44 +0000 Subject: [PATCH 03/18] db/state, execution: fix CodeStore reorg/unwind wrong-root The persistent CodeStore produced INVALID canonical blocks on reorg/unwind (BAL mismatch, wrong trie root) with it enabled. Root causes, all fixed here: - The app-level read-through in stateObject.Code resolved code by the stateObject's snapshot so.data.CodeHash, which can lag an in-block code change. Serve the code store from SD.GetCode instead, keyed off the mem-first codeHashForAddr resolution (reflects in-block writes), so the hash lookup is reorg-safe. The read-through is removed. - The cold-path SetMem stored (codeHash-from-AccountsDomain, code-from- CodeDomain) pairs, which parallel exec can observe momentarily inconsistent, poisoning the store with a hash->wrong-code entry that later flipped a SetCode prev-code comparison and the BAL. The store is now populated only by the flush callback (self-consistent keccak(v)->v). - The flush-callback code-store MDBX write ran mid-flushMem; defer it to after flushMem so it no longer interleaves with the in-progress domain flush. Removes the now-dead SetMem, ReaderV3.CodeStore and temporalGetter.CodeStore plumbing. --- db/state/execctx/domain_shared.go | 40 ++++++++++++++++++++----------- execution/cache/code_store.go | 11 --------- execution/state/rw_v3.go | 13 ---------- execution/state/state_object.go | 19 --------------- 4 files changed, 26 insertions(+), 57 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 5b1bde824af..1fa15b0b27e 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -761,12 +761,6 @@ func (sd *SharedDomains) SetCodeStore(codeStore *cache.CodeStore) { sd.codeStore = codeStore } -// CodeStore exposes the code store + the backing tx so an addr-keyed reader can -// serve a code-by-hash read using the application's authoritative codehash. -func (tg *temporalGetter) CodeStore() (*cache.CodeStore, kv.TemporalTx) { - return tg.sd.codeStore, tg.tx -} - // PrintCacheStats logs the state cache hit/miss counters and resets them. // No-op when the cache is disabled. The cache is an SD-internal detail, so // callers observe it through SD rather than reaching for the cache directly. @@ -980,12 +974,15 @@ func (sd *SharedDomains) Commit(ctx context.Context, tx kv.RwTx, validate ...fun if sd.stateCache != nil { opts = append(opts, stash(kv.AccountsDomain), stash(kv.StorageDomain)) } - // CodeDomain flush both populates the persistent code store (write path, has - // an RwTx) and stashes for the in-mem state cache. + // CodeDomain flush stashes state-cache updates and collects code for the + // persistent store. The code-store MDBX write is deferred to after flushMem — + // an in-callback tx.Put interleaves with the in-progress domain flush and + // corrupts it (reorg/unwind wrong root). + var codeStoreWrites [][2][]byte if sd.stateCache != nil || sd.codeStore != nil { opts = append(opts, kv.WithFlushCallback(kv.CodeDomain, func(k []byte, v []byte, step kv.Step, txNum uint64) { if sd.codeStore != nil && len(v) > 0 { - _ = sd.codeStore.PutByHash(tx, crypto.Keccak256(v), v) + codeStoreWrites = append(codeStoreWrites, [2][]byte{crypto.Keccak256(v), append([]byte(nil), v...)}) } if sd.stateCache != nil { pending = append(pending, cacheUpdate{ @@ -1001,6 +998,11 @@ func (sd *SharedDomains) Commit(ctx context.Context, tx kv.RwTx, validate ...fun if err := sd.flushMem(ctx, tx, opts...); err != nil { return err } + for _, cw := range codeStoreWrites { + if err := sd.codeStore.PutByHash(tx, cw[0], cw[1]); err != nil { + return err + } + } if err := runValidate(); err != nil { return err } @@ -1400,11 +1402,21 @@ func (sd *SharedDomains) GetCode(tx kv.TemporalTx, addr []byte, txNum uint64) ([ } // Fast path: addr → account codeHash → content-addressed bytes, no - // per-address CodeDomain read. - if sd.stateCache != nil { - if codeHash := sd.codeHashForAddr(tx, addr, txNum); len(codeHash) > 0 { - if cv, ok := sd.stateCache.GetCodeByHash(codeHash); ok { - return cv, true, nil + // per-address CodeDomain read. The codeHash is resolved mem-first, so it + // reflects in-block code changes — keying the code store off it (rather than + // a stateObject's stale snapshot) is reorg-safe. + var codeHash []byte + if sd.stateCache != nil || sd.codeStore != nil { + if codeHash = sd.codeHashForAddr(tx, addr, txNum); len(codeHash) > 0 { + if sd.stateCache != nil { + if cv, ok := sd.stateCache.GetCodeByHash(codeHash); ok { + return cv, true, nil + } + } + if sd.codeStore != nil { + if cv, ok := sd.codeStore.GetByHash(tx, codeHash); ok { + return cv, true, nil + } } } } diff --git a/execution/cache/code_store.go b/execution/cache/code_store.go index 92ca1bea951..e1c608e3256 100644 --- a/execution/cache/code_store.go +++ b/execution/cache/code_store.go @@ -121,14 +121,3 @@ func (s *CodeStore) Evict(tx kv.RwTx) error { } return nil } - -// SetMem populates only the in-memory tier — used on a read-path (RoTx) cold -// decompress where the MDBX backing cannot be written. -func (s *CodeStore) SetMem(codeHash, code []byte) { - if s == nil || len(codeHash) != 32 || len(code) == 0 { - return - } - var key [32]byte - copy(key[:], codeHash) - s.mem.Set(key, code) -} diff --git a/execution/state/rw_v3.go b/execution/state/rw_v3.go index 69c1521e21a..7af3f2c1612 100644 --- a/execution/state/rw_v3.go +++ b/execution/state/rw_v3.go @@ -35,7 +35,6 @@ import ( "github.com/erigontech/erigon/db/rawdb" "github.com/erigontech/erigon/db/rawdb/rawtemporaldb" "github.com/erigontech/erigon/db/state/execctx" - "github.com/erigontech/erigon/execution/cache" "github.com/erigontech/erigon/execution/chain" "github.com/erigontech/erigon/execution/types" "github.com/erigontech/erigon/execution/types/accounts" @@ -1011,18 +1010,6 @@ func NewReaderV3(getter kv.TemporalGetter) *ReaderV3 { } } -// CodeStore returns the codehash-keyed code cache + backing tx when the reader's -// getter exposes one, so callers holding an authoritative codehash can serve a -// code read without the addr-keyed CodeDomain decompression. -func (r *ReaderV3) CodeStore() (*cache.CodeStore, kv.TemporalTx) { - if g, ok := r.getter.(interface { - CodeStore() (*cache.CodeStore, kv.TemporalTx) - }); ok { - return g.CodeStore() - } - return nil, nil -} - func (r *ReaderV3) DiscardReadList() {} func (r *ReaderV3) SetTxNum(txNum uint64) { r.txNum = txNum } func (r *ReaderV3) SetGetter(getter kv.TemporalGetter) { r.getter = getter } diff --git a/execution/state/state_object.go b/execution/state/state_object.go index 8093ac725d2..6701ee9b840 100644 --- a/execution/state/state_object.go +++ b/execution/state/state_object.go @@ -35,8 +35,6 @@ import ( "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/empty" "github.com/erigontech/erigon/common/u256" - "github.com/erigontech/erigon/db/kv" - "github.com/erigontech/erigon/execution/cache" "github.com/erigontech/erigon/execution/rlp" "github.com/erigontech/erigon/execution/tracing" "github.com/erigontech/erigon/execution/types/accounts" @@ -429,20 +427,6 @@ func (so *stateObject) Code() ([]byte, error) { } } - ch := so.data.CodeHash.Value() - var codeStore *cache.CodeStore - if cs, ok := so.db.stateReader.(interface { - CodeStore() (*cache.CodeStore, kv.TemporalTx) - }); ok { - var tx kv.TemporalTx - if codeStore, tx = cs.CodeStore(); codeStore != nil { - if code, ok := codeStore.GetByHash(tx, ch[:]); ok { - so.code = code - return code, nil - } - } - } - if dbg.TraceDomainIO || (dbg.TraceTransactionIO && (so.db.trace || dbg.TraceAccount(so.address.Handle()))) { so.db.stateReader.SetTrace(true, fmt.Sprintf("%d (%d.%d)", so.db.blockNum, so.db.txIndex, so.db.version)) } @@ -461,9 +445,6 @@ func (so *stateObject) Code() ([]byte, error) { return nil, fmt.Errorf("can't read code for %x: %w", so.Address(), err) } so.code = code - if codeStore != nil && len(code) > 0 { - codeStore.SetMem(ch[:], code) - } return code, nil } From 0950d780c4cbee7dd1e8441461f8e2d11a693b2e Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 2 Jul 2026 16:14:08 +0000 Subject: [PATCH 04/18] execution/cache, execution/commitment: address #22154 review - CodeStore.Evict: tableSizeBytes starts at 0 each process start, so a persistent TblCodeCache that already exceeds the cap would never be pruned (unbounded growth across restarts). Seed it once from the table via sumTableBytes, in the same key+value byte units the eviction loop decrements. - BranchCache.Get: only count a pinned hit/miss when the prefix actually routes to a pinned storage trunk; account-trie and tail-only lookups no longer inflate pinnedMisses. - AdaptivePinController: the value threaded through OnBlockComplete is a txNum (that is what SharedDomains has at commit), not a block number; rename the parameter, the write-only promotedAtBlock field, and the log keys to txNum so the diagnostics are accurate. No behaviour change. --- execution/cache/code_store.go | 35 +++++++++++++++++++++++++++- execution/cache/code_store_test.go | 9 +++++++ execution/commitment/adaptive_pin.go | 16 ++++++------- execution/commitment/branch_cache.go | 4 +++- 4 files changed, 54 insertions(+), 10 deletions(-) diff --git a/execution/cache/code_store.go b/execution/cache/code_store.go index e1c608e3256..bfc0bcf9dd2 100644 --- a/execution/cache/code_store.go +++ b/execution/cache/code_store.go @@ -21,6 +21,7 @@ type CodeStore struct { // LRU and entries re-derive from CodeDomain. tableCapBytes uint64 tableSizeBytes atomic.Int64 + tableSeeded atomic.Bool memHits atomic.Uint64 tableHits atomic.Uint64 @@ -97,7 +98,21 @@ func (s *CodeStore) PutByHash(tx kv.RwTx, codeHash, code []byte) error { // order when over capacity. Safe: evicted entries are re-derivable from // CodeDomain on miss. Call on a write tx (e.g., at commit), never on reads. func (s *CodeStore) Evict(tx kv.RwTx) error { - if s == nil || s.tableCapBytes == 0 || uint64(s.tableSizeBytes.Load()) <= s.tableCapBytes { + if s == nil || s.tableCapBytes == 0 { + return nil + } + // tableSizeBytes starts at 0 each process start; seed it once from the + // persistent table (in the same key+value byte units the eviction loop + // decrements) so a backing that already exceeds the cap gets pruned rather + // than growing unbounded across restarts. + if s.tableSeeded.CompareAndSwap(false, true) { + total, err := sumTableBytes(tx) + if err != nil { + return err + } + s.tableSizeBytes.Store(total) + } + if uint64(s.tableSizeBytes.Load()) <= s.tableCapBytes { return nil } c, err := tx.RwCursor(kv.TblCodeCache) @@ -121,3 +136,21 @@ func (s *CodeStore) Evict(tx kv.RwTx) error { } return nil } + +// sumTableBytes returns the total key+value byte size of TblCodeCache, in the +// same units Evict tracks and decrements. +func sumTableBytes(tx kv.RwTx) (int64, error) { + c, err := tx.Cursor(kv.TblCodeCache) + if err != nil { + return 0, err + } + defer c.Close() + var total int64 + for k, v, err := c.First(); k != nil; k, v, err = c.Next() { + if err != nil { + return 0, err + } + total += int64(len(k) + len(v)) + } + return total, nil +} diff --git a/execution/cache/code_store_test.go b/execution/cache/code_store_test.go index 59fb5cdb77d..e0f675d05b5 100644 --- a/execution/cache/code_store_test.go +++ b/execution/cache/code_store_test.go @@ -60,4 +60,13 @@ func TestCodeStore_TwoTierAndEvict(t *testing.T) { } require.NoError(t, small.Evict(tx)) require.LessOrEqual(t, small.tableSizeBytes.Load(), int64(128)) + + // Restart scenario: a fresh store (tableSizeBytes=0) over an already-full + // backing must still prune — seed the size from the table, don't grow + // unbounded. Without the seed, Evict's under-cap early return would skip. + restarted := NewCodeStore(1<<20, 128) + require.Zero(t, restarted.tableSizeBytes.Load()) + require.NoError(t, restarted.Evict(tx)) + require.LessOrEqual(t, restarted.tableSizeBytes.Load(), int64(128), + "a fresh store must seed its size from the backing and prune, not grow unbounded across restarts") } diff --git a/execution/commitment/adaptive_pin.go b/execution/commitment/adaptive_pin.go index baff8961978..91a7107adb6 100644 --- a/execution/commitment/adaptive_pin.go +++ b/execution/commitment/adaptive_pin.go @@ -71,7 +71,7 @@ type DbBranchesProvider func(contractHash []byte) map[string][]byte type adaptiveContractState struct { contractHash [32]byte - promotedAtBlock uint64 + promotedAtTxNum uint64 preload *ContractTrunkPreload // serial-BFS path (nil when parallel) parallel *ContractTrunkPreloadParallel // parallel-wave-BFS path (nil when serial) coldBlocksInARow int @@ -161,7 +161,7 @@ func (c *AdaptivePinController) onCacheMiss(prefix []byte) { // OnBlockComplete consumes the per-block miss snapshot and decides // promotions, extensions, and demotions. Synchronous — preloads run // inline so the new pin set is available for the next block's reads. -func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum uint64, reader CommitmentReader) { +func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, txNum uint64, reader CommitmentReader) { misses := c.snapshotMisses() c.mu.Lock() @@ -173,7 +173,7 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui if c.parallelResolverFactory != nil { r, release, err := c.parallelResolverFactory() if err != nil { - c.warnf("[adaptive-pin] parallel resolver factory failed, falling back to serial", "err", err, "block", blockNum) + c.warnf("[adaptive-pin] parallel resolver factory failed, falling back to serial", "err", err, "txNum", txNum) } else { parallelResolve = r releaseParallel = release @@ -215,7 +215,7 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui if len(misses) > 0 && len(c.states) < c.cfg.MaxPromotedContracts { candidates := pickPromotionCandidates(misses, c.cfg.PromoteThresholdMisses, c.cfg.MaxPromotedContracts-len(c.states)) for _, hash := range candidates { - state, err := c.promoteLocked(ctx, hash, blockNum, parallelResolve, reader) + state, err := c.promoteLocked(ctx, hash, txNum, parallelResolve, reader) if err != nil { c.warnf("[adaptive-pin] initial-view failed", "hash", hex.EncodeToString(hash[:]), "err", err) continue @@ -238,7 +238,7 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, blockNum ui if c.logger != nil && (promoted+extended+demoted > 0 || len(c.states) > 0) { c.logger.Info("[adaptive-pin]", - "block", blockNum, + "txNum", txNum, "promoted_total", len(c.states), "promoted_this_block", promoted, "extended_this_block", extended, @@ -278,7 +278,7 @@ func (c *AdaptivePinController) demoteLocked(hash [32]byte, state *adaptiveContr func (c *AdaptivePinController) promoteLocked( ctx context.Context, hash [32]byte, - blockNum uint64, + txNum uint64, parallelResolve BatchBranchResolver, reader CommitmentReader, ) (*adaptiveContractState, error) { @@ -299,7 +299,7 @@ func (c *AdaptivePinController) promoteLocked( } return &adaptiveContractState{ contractHash: hash, - promotedAtBlock: blockNum, + promotedAtTxNum: txNum, parallel: p, }, nil } @@ -315,7 +315,7 @@ func (c *AdaptivePinController) promoteLocked( } return &adaptiveContractState{ contractHash: hash, - promotedAtBlock: blockNum, + promotedAtTxNum: txNum, preload: p, }, nil } diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 51a2a7f9f14..179d2616391 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -483,6 +483,8 @@ func (c *BranchCache) lookup(prefix []byte) (*branchCacheEntry, bool) { return nil, false } // Pinned tier: per-contract storage trunk (fixed skeleton + deep overflow). + // Only a lookup that actually routes to a pinned trunk counts toward the + // pinned hit/miss stats; account-trie and tail-only prefixes are excluded. if st, _, stor, ok := c.storageRoute(prefix, false); ok { var entry *branchCacheEntry if slot := st.slot(stor); slot != nil { @@ -494,8 +496,8 @@ func (c *BranchCache) lookup(prefix []byte) (*branchCacheEntry, bool) { c.pinnedHits.Add(1) return entry, true } + c.pinnedMisses.Add(1) } - c.pinnedMisses.Add(1) entry, ok := c.tail.Get(maphash.Hash(prefix)) if !ok { c.tailMisses.Add(1) From 77c7800c465592ff3f84e8661d2edd6cb017b2d3 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 2 Jul 2026 18:48:28 +0000 Subject: [PATCH 05/18] execution: trim over-long comments to the load-bearing why Address the #22154 review's comment-length notes: condense the BranchCache doc, the reset_stages branchCache-clear note, and the ValidateChain SetParent note to their load-bearing invariants. The design-doc detail (concurrency walkthrough, responsibility split, disk counters, two-role breakdown, cherry- pick note) moves out of source per the comment policy. No behaviour change. --- execution/commitment/branch_cache.go | 119 ++---------------- execution/execmodule/exec_module.go | 36 +----- .../stagedsync/rawdbreset/reset_stages.go | 6 +- 3 files changed, 19 insertions(+), 142 deletions(-) diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 179d2616391..de1d6339f3a 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -46,115 +46,18 @@ func isCommitmentStateKey(prefix []byte) bool { return bytes.Equal(prefix, KeyCommitmentState) } -// BranchCache stores commitment-trie branch data: +// BranchCache stores commitment-trie branch data: a bounded LRU tail plus a +// single never-evicted slot for the root branch (a length-0 / no-key prefix). +// Aggregator-scope (one instance per Domain), pulled via BranchCacheProvider and +// plumbed to the trie through InitializeTrieAndUpdates. It is a passive store — +// the trie walker/encoder drive all reads and writes; the cache never fetches +// state itself. // -// - Bounded LRU tail with configurable capacity (eviction is well-defined, -// suitable for long-lived caching across many Process calls without -// unbounded memory growth). -// - Single pinned slot for the root branch (always hottest, always present -// once populated, never subject to LRU eviction). Compact prefix of -// length 0 (or single-byte "no-key" form) targets this slot. -// -// Lifetime: aggregator-scope (one instance per Domain). SharedDomains -// pulls the instance via BranchCacheProvider on the AggregatorRoTx; -// commitment-context plumbs it through to the trie via -// InitializeTrieAndUpdates. The previous WarmupCache type (per-Process, -// duplicating account/storage/branch caching above this layer) was -// deleted in the WarmupCache consolidation; BranchCache is now the -// single branch cache. -// -// # Responsibility split (architectural) -// -// The cache is a passive store. Reads and writes are driven by the -// trie walker / encoder; the cache itself never reaches into the -// underlying state. -// -// - BranchCache: passive store of branch bytes. -// Doesn't fetch anything. -// - Branch warmer (warmuper.go): narrow scope — pre-fetches -// *branches* along touched-key paths via SD.GetLatest. No -// account/storage prefetch — that conflated branch warm-up with -// leaf-data fetch. If a fold needs leaf data the trie walker -// fetches it directly (or it's already in Updates / memoized as -// stateHash). -// - Trie walker, block-processing path: receives Updates from the -// executor, folds them. Memoized stateHashes serve siblings; new -// values come from Updates. Doesn't reach into leaf data via -// prefetch. -// - Trie walker, witness / proof generation path: walks the trie -// structure and *needs* to fetch state to materialize the proof. -// This is the walker's responsibility — it drives its own reads -// against SD. If that path turns out to be cold-bound on real -// workloads it may indicate a need for separate account / storage -// caches (the `add_execution_context_with_caches` work has a -// reference design for these). Treat that as a separate concern -// from this BranchCache — different scope, different lifetime, -// different invalidation. Do not regrow the branch warmer's -// scope to cover it. -// -// The disk_sto / disk_acc counters on the [commitment][cache-fp] log -// line surface any fall-through where the trie compute reaches the -// underlying ctx.Account / ctx.Storage paths. On block-processing -// workloads they should remain zero; non-zero values signal a -// memoization gap or a missing walker-side prefetch. -// -// # Concurrency contract — caller invariants -// -// Internally, the LRU tail is thread-safe (hashicorp/golang-lru/v2) and -// the pinned root slot is an atomic.Pointer. So any combination of -// concurrent Get / Put / Invalidate is mechanically safe — no panics, no -// torn reads. But "mechanically safe" is NOT the same as "logically -// consistent across writers." The cache is designed to be used under one -// caller invariant: -// -// - Single writer per prefix at any moment. The cache does not coordinate -// concurrent writes to the same key — last-Put-wins semantics, with no -// guarantee that the winning value is the one the application wanted. -// -// # Concurrency contract — how the existing concurrent trie satisfies it -// -// The current ConcurrentPatriciaHashed (parallel commitment calculator) -// satisfies that invariant by construction: -// -// - Mounts partition the prefix space by FIRST NIBBLE. Mount N's -// encoder only writes branches whose key starts with [0x0N ...]. -// Different mounts therefore never write to the same prefix. -// (See hex_concurrent_patricia_hashed.go: NewConcurrentPatriciaHashed -// creates 16 mounts via SpawnSubTrie; each mount has its own HPH, -// own BranchEncoder, own PatriciaContext / roTx.) -// -// - Root branch (prefix [0x00]) is written by the single root fold -// that runs SEQUENTIALLY after errgroup.Wait() in ParallelHashSort. -// One writer for the pinned root slot. -// -// - Mount→root grid roll-up is mutex-protected via -// ConcurrentPatriciaHashed.rootMu — but that updates IN-MEMORY grid -// cells, not the cache. The cache only sees the eventual root -// branch when the post-Wait root fold encodes it. -// -// # Concurrency contract — what future parallel fold work must preserve -// -// A future parallel tree-reduce fold would change the picture: the parent -// fold (incl. root) would no longer be a single post-Wait sequential pass. -// Multiple goroutines would compute parent branches in parallel as their -// children complete. This MUST not violate "single writer per prefix" — -// any future Stage F design needs an explicit per-prefix coordination layer -// (atomic counter on parent "children remaining"; only the last-decrementer -// writes the parent). That coordination belongs at the orchestrator layer; -// the cache itself does NOT add per-prefix locking because that would be -// wasted work for the current architecture. -// -// If you are implementing parallel fold (or any other architecture that -// breaks the "single writer per prefix" invariant), do NOT relax the -// invariant by adding internal locking to the cache. Add the -// coordination at the orchestrator layer where the partitioning logic -// lives. The cache stays simple; the orchestrator owns the discipline. -// -// Likewise if you change the prefix partitioning (e.g. by-second-nibble -// mounts, depth-based partitioning, anything other than first-nibble), -// re-validate that distinct workers continue to write disjoint prefix -// spaces. Re-read the partitioning code in -// hex_concurrent_patricia_hashed.go and confirm. +// Concurrency: the LRU tail and the atomic-pointer root slot make any mix of +// concurrent Get/Put/Invalidate mechanically safe, but the cache does not +// coordinate writers — callers must ensure a single writer per prefix +// (last-Put-wins otherwise); add any such coordination at the orchestrator, not +// by locking the cache. type BranchCache struct { // Root tier — single slot for the root branch (always hottest, always // present). Atomic-pointer access so no lock is needed for the hot diff --git a/execution/execmodule/exec_module.go b/execution/execmodule/exec_module.go index c3a14aba186..921a0ecb038 100644 --- a/execution/execmodule/exec_module.go +++ b/execution/execmodule/exec_module.go @@ -550,36 +550,12 @@ func (e *ExecModule) ValidateChain(ctx context.Context, blockHash common.Hash, b } var tx kv.TemporalRwTx = doms.BlockOverlay() - // Chain whenever a currentContext exists, not only when head-extending - // (header.ParentHash == head): fork-payload caching needs the parent link - // too — see the two-role breakdown below. - // - // Chain the validation SD to the latest in-memory canonical generation: - // e.currentContext when present, otherwise the newest in-flight commit - // generation (gate item 2 — the prior FCU cleared currentContext and - // handed its SD to the background commit). - // - // The parent link serves two roles: - // - // 1. Head-extending payloads read the canonical generation's - // not-yet-committed domain state instead of stale MDBX. - // - // 2. Fork payloads: unwindToCommonCanonical below must build an unwind - // set, and the diffsets of the canonical blocks it unwinds live in - // the canonical generation's pastChangesAccumulator — reachable only - // through this parent link (GetDiffset chains to the parent). Without - // it the unwind silently runs with no unwind set, leaving the - // BranchCache unmasked and corrupting the computed root. - // - // For a fork payload the parent does NOT shadow the unwound base: once - // unwindToCommonCanonical has run, doms.mem.unwindChangeset holds every - // key the unwound canonical blocks touched, and TemporalMemBatch.getLatest - // resolves those from the unwind set before ever consulting the parent. - // - // Cherry-pick note: the upstream commit also chained to e.latestGen() - // (the gate-2 in-flight commit generation) when currentContext is nil; - // that generation chain is not on this branch, so currentContext is the - // only canonical generation here. + // Chain the validation SD to the canonical generation (e.currentContext) for + // any payload with a parent, not just head-extending ones: head-extending + // payloads read its not-yet-committed domain state instead of stale MDBX, and + // fork payloads reach the canonical generation's pastChangesAccumulator (via + // GetDiffset's parent chain) to build the unwind set — without the link the + // unwind runs empty, leaving the BranchCache unmasked and corrupting the root. if e.currentContext != nil { doms.SetParent(e.currentContext) } diff --git a/execution/stagedsync/rawdbreset/reset_stages.go b/execution/stagedsync/rawdbreset/reset_stages.go index 5bec686ed0b..43a5b16a724 100644 --- a/execution/stagedsync/rawdbreset/reset_stages.go +++ b/execution/stagedsync/rawdbreset/reset_stages.go @@ -188,10 +188,8 @@ func ResetExec(ctx context.Context, db kv.TemporalRwDB) (err error) { } // Wiping the commitment table leaves the aggregator's in-memory branchCache - // referencing trie nodes that no longer exist on disk. A subsequent from-0 - // re-exec then reads those stale nodes when computing block 0's commitment - // and produces a wrong trie root under parallel exec. - // Drop the cache so it repopulates from the freshly-wiped table. + // pointing at now-deleted trie nodes; drop it so a from-0 re-exec repopulates + // from the wiped table instead of computing a wrong root off stale nodes. branchCacheCleared := false if hasAgg, ok := db.(dbstate.HasAgg); ok { if agg, ok := hasAgg.Agg().(*dbstate.Aggregator); ok { From 7d467d02506221cdaf7ea0271525ffb0f9d551dd Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 3 Jul 2026 08:19:11 +0000 Subject: [PATCH 06/18] execution/commitment, db/state: lazy trunk d4 + skip BranchCache for genesis The trunk-pin BranchCache eagerly allocated a 65536-entry (~512KB) depth-4 account-trie array per aggregator. In production this is one process-wide cache so the cost is trivial, but execution/tests spins up tens of thousands of ephemeral aggregators via GenesisToBlock, turning that eager alloc into GBs of churn (amplified ~10x under -race), long enough to hit the CI runner-reclaim window. - branch_cache.go: d4 is now an atomic.Pointer allocated lazily via CAS on the first depth-4 write; shallow test tries never pay for it. trunkSlot takes a forWrite flag so reads/invalidations don't allocate. - db/state: add AggOpts.DisableBranchCache(); GenesisToBlock opts out entirely (one-shot, no cross-block reuse). --- db/state/aggregator.go | 11 ++-- db/state/aggregator2.go | 14 ++++-- execution/commitment/branch_cache.go | 50 ++++++++++++------- execution/state/genesiswrite/genesis_write.go | 2 +- 4 files changed, 51 insertions(+), 26 deletions(-) diff --git a/db/state/aggregator.go b/db/state/aggregator.go index 1225e215f05..e9606bb8704 100644 --- a/db/state/aggregator.go +++ b/db/state/aggregator.go @@ -88,8 +88,9 @@ type Aggregator struct { oldestVisible *aggregatorVisible snapshotBuildSema *semaphore.Weighted - disableHistory bool - workers workersCfg + disableHistory bool + branchCacheDisabled bool + workers workersCfg // To keep DB small - need move data to small files ASAP. // It means goroutine which creating small files - can't be locked by merge or indexing. @@ -411,8 +412,10 @@ func (a *Aggregator) ConfigureDomains() error { } a.configured = true - // Attach the aggregator-lifetime BranchCache to the commitment domain; gated by USE_STATE_CACHE, nil = disabled. - if dbg.UseStateCache { + // Attach the aggregator-lifetime BranchCache to the commitment domain; gated + // by USE_STATE_CACHE, nil = disabled. Skipped for ephemeral aggregators that + // opt out (e.g. one-shot genesis processing has no cross-block reuse). + if dbg.UseStateCache && !a.branchCacheDisabled { if cd := a.d[kv.CommitmentDomain]; cd != nil && cd.branchCache == nil { cd.branchCache = commitment.NewBranchCache(commitment.DefaultBranchCacheTailCapacity) } diff --git a/db/state/aggregator2.go b/db/state/aggregator2.go index 703633c30fd..da0bb2e0cf5 100644 --- a/db/state/aggregator2.go +++ b/db/state/aggregator2.go @@ -30,10 +30,11 @@ type AggOpts struct { //nolint:gocritic referencesInCommitmentBranches *bool // nil = leave global schema default untouched - genSaltIfNeed bool - sanityOldNaming bool // prevent start directory with old file names - disableFsync bool // for tests speed - disableHistory bool // for temp/inmem aggregator instances + genSaltIfNeed bool + sanityOldNaming bool // prevent start directory with old file names + disableFsync bool // for tests speed + disableHistory bool // for temp/inmem aggregator instances + disableBranchCache bool // for one-shot aggregators with no cross-block reuse (e.g. genesis) } func New(dirs datadir.Dirs) AggOpts { //nolint:gocritic @@ -74,6 +75,7 @@ func (opts AggOpts) Open(ctx context.Context, db kv.RoDB) (*Aggregator, error) { a.erigondbDomainStepsInFrozenFile = opts.erigondbDomainStepsInFrozenFile a.disableHistory = opts.disableHistory + a.branchCacheDisabled = opts.disableBranchCache a.disableFsync = opts.disableFsync a.savedSalt = salt @@ -121,6 +123,10 @@ func (opts AggOpts) GenSaltIfNeed(v bool) AggOpts { opts.genSaltIfNeed = v; retu func (opts AggOpts) Logger(l log.Logger) AggOpts { opts.logger = l; return opts } //nolint:gocritic func (opts AggOpts) DisableFsync() AggOpts { opts.disableFsync = true; return opts } //nolint:gocritic func (opts AggOpts) DisableHistory() AggOpts { opts.disableHistory = true; return opts } //nolint:gocritic +func (opts AggOpts) DisableBranchCache() AggOpts { //nolint:gocritic + opts.disableBranchCache = true + return opts +} func (opts AggOpts) SanityOldNaming() AggOpts { //nolint:gocritic opts.sanityOldNaming = true return opts diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index de1d6339f3a..3a8512f4091 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -149,25 +149,29 @@ type branchCacheEntry struct { type MissCallback func(prefix []byte) // trunk is a resident, lock-free fixed-array tier shared by both tries: the -// accountTrunk holds account-trie branches at nibble depths 1-4 (d4 allocated, -// deep nil); each per-contract storageTrunk holds storage branches at storage +// accountTrunk holds account-trie branches at nibble depths 1-4 (d4 lazily +// allocated, deep nil); each per-contract storageTrunk holds storage branches at storage // depths 0-3 with depth 4+ in deep (d4 nil, deep allocated). Slots are // atomic.Pointer: under the single-writer-per-prefix invariant readers/writers // take no mutex (just an atomic load/store per slot); only deep (a maphash.Map) // locks. type trunk struct { - d0 atomic.Pointer[branchCacheEntry] - d1 [16]atomic.Pointer[branchCacheEntry] - d2 [256]atomic.Pointer[branchCacheEntry] - d3 [4096]atomic.Pointer[branchCacheEntry] - d4 *[65536]atomic.Pointer[branchCacheEntry] + d0 atomic.Pointer[branchCacheEntry] + d1 [16]atomic.Pointer[branchCacheEntry] + d2 [256]atomic.Pointer[branchCacheEntry] + d3 [4096]atomic.Pointer[branchCacheEntry] + // d4 (the 65536-entry depth-4 array, ~512KB) is allocated lazily on the + // first depth-4 write: prod has one process-wide cache so eager alloc is + // cheap, but tests spin up thousands of ephemeral aggregators whose tries + // rarely reach depth 4, so eager alloc there is pure churn. + d4 atomic.Pointer[[65536]atomic.Pointer[branchCacheEntry]] deep *maphash.Map[*branchCacheEntry] } // newAccountTrunk builds the global account trunk: dense depth-4 fixed array, // no deep overflow (account depth 5+ uses the LRU tail). func newAccountTrunk() *trunk { - return &trunk{d4: &[65536]atomic.Pointer[branchCacheEntry]{}} + return &trunk{} } // newStorageTrunk builds a per-contract storage trunk: deep overflow for @@ -190,8 +194,8 @@ func (t *trunk) slot(path []byte) *atomic.Pointer[branchCacheEntry] { case 3: return &t.d3[uint16(path[0])<<8|uint16(path[1])<<4|uint16(path[2])] case 4: - if t.d4 != nil { - return &t.d4[uint32(path[0])<<12|uint32(path[1])<<8|uint32(path[2])<<4|uint32(path[3])] + if d4 := t.d4.Load(); d4 != nil { + return &d4[uint32(path[0])<<12|uint32(path[1])<<8|uint32(path[2])<<4|uint32(path[3])] } } return nil @@ -248,7 +252,7 @@ func NewBranchCache(tailCapacity int) *BranchCache { // trunk, or depth >= 5 (served by the LRU tail). The compact-hex prefix maps // directly to an array index, no hashing. Bit 4 of byte 0 is the odd-length // flag; the low nibble of byte 0 is the first nibble when odd. -func (c *BranchCache) trunkSlot(prefix []byte) *atomic.Pointer[branchCacheEntry] { +func (c *BranchCache) trunkSlot(prefix []byte, forWrite bool) *atomic.Pointer[branchCacheEntry] { if c.trunkDisabled { return nil } @@ -264,7 +268,17 @@ func (c *BranchCache) trunkSlot(prefix []byte) *atomic.Pointer[branchCacheEntry] return &c.accountTrunk.d3[uint16(prefix[0]&0x0f)<<8|uint16(prefix[1])] // 3 nibbles case 3: if prefix[0]&0x10 == 0 { // 4 nibbles - return &c.accountTrunk.d4[uint16(prefix[1])<<8|uint16(prefix[2])] + d4 := c.accountTrunk.d4.Load() + if d4 == nil { + if !forWrite { + return nil + } + d4 = &[65536]atomic.Pointer[branchCacheEntry]{} + if !c.accountTrunk.d4.CompareAndSwap(nil, d4) { + d4 = c.accountTrunk.d4.Load() // lost the race; use the winner + } + } + return &d4[uint16(prefix[1])<<8|uint16(prefix[2])] } // 5 nibbles (odd, 3 bytes) -> LRU tail } @@ -334,8 +348,10 @@ func (c *BranchCache) clearTrunk() { for i := range t.d3 { t.d3[i].Store(nil) } - for i := range t.d4 { - t.d4[i].Store(nil) + if d4 := t.d4.Load(); d4 != nil { + for i := range d4 { + d4[i].Store(nil) + } } } @@ -376,7 +392,7 @@ func (c *BranchCache) lookup(prefix []byte) (*branchCacheEntry, bool) { } // Resident account trunk (fixed arrays, depths 1-4). Disjoint from the // storage trunks (depth >= 64) and tail, so a miss here is genuine. - if slot := c.trunkSlot(prefix); slot != nil { + if slot := c.trunkSlot(prefix, false); slot != nil { if entry := slot.Load(); entry != nil { c.trunkHits.Add(1) return entry, true @@ -416,7 +432,7 @@ func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { c.root.Store(entry) return } - if slot := c.trunkSlot(prefix); slot != nil { + if slot := c.trunkSlot(prefix, true); slot != nil { slot.Store(entry) return } @@ -533,7 +549,7 @@ func (c *BranchCache) Invalidate(prefix []byte) { c.root.Store(nil) return } - if slot := c.trunkSlot(prefix); slot != nil { + if slot := c.trunkSlot(prefix, false); slot != nil { slot.Store(nil) return } diff --git a/execution/state/genesiswrite/genesis_write.go b/execution/state/genesiswrite/genesis_write.go index c968515af4f..9cb40a86c90 100644 --- a/execution/state/genesiswrite/genesis_write.go +++ b/execution/state/genesiswrite/genesis_write.go @@ -358,7 +358,7 @@ func GenesisToBlock(tb testing.TB, g *types.Genesis, dirs datadir.Dirs, logger l if err != nil { return nil, nil, err } - agg, err := dbstate.New(dirs).Logger(logger).WithErigonDBSettings(erigonDBSettings).Open(ctx, genesisTmpDB) + agg, err := dbstate.New(dirs).Logger(logger).WithErigonDBSettings(erigonDBSettings).DisableBranchCache().Open(ctx, genesisTmpDB) if err != nil { return nil, nil, err } From 88144910cac611f67f276fb3e7688598669d3c0b Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 3 Jul 2026 08:41:09 +0000 Subject: [PATCH 07/18] execution/cache: fix CodeCache size-drift under concurrent same-key Puts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit putContent did a membership check, then accounted bytes, then inserted as three separate steps. freelru has no LoadOrStore, so two goroutines Putting the same cold code both missed the check, both added to the byte counter, and the second Add overwrote the first — leaving one resident entry but a counter inflated by 2x. The drift is permanent and compounds, wedging the byte-based stat. Serialize the check-account-insert per key hash with a striped mutex so distinct keys still put in parallel. --- execution/cache/code_cache.go | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 6cd3556bef6..93de162d616 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -156,6 +156,13 @@ type CodeCache struct { // check+bind is atomic w.r.t. a concurrent authoritative rebind. addrBindMu sync.Mutex + // putStripes serializes putContent's membership-check + size-account + + // insert per key hash: freelru has no LoadOrStore, so without this two + // concurrent Puts of the same cold code both miss the check and both add to + // the byte counter while only one entry survives, drifting the stat upward. + // Striped by key so distinct keys still put in parallel. + putStripes [256]sync.Mutex + // Stats counters (atomic for concurrent access) addrHits atomic.Uint64 addrMisses atomic.Uint64 @@ -195,7 +202,10 @@ func putContent[T any]( coh *coherence.Gen, counter *atomic.Int64, keyCost int64, + stripe *sync.Mutex, ) { + stripe.Lock() + defer stripe.Unlock() if existing, ok := lru.Get(h); ok { if txNum, epoch := stamp(existing); !coh.IsStale(txNum, epoch) { return @@ -361,7 +371,7 @@ func (c *CodeCache) putCode(addr []byte, code []byte, keyHash [32]byte, txNum ui entry := codeEntry{code: code, keyHash: keyHash, txNum: txNum, epoch: ep} // freelru keyed by the codeID (maphash of code) directly; 8-byte key cost. putContent(c.hashToCode, codeID, entry, codeEntryStamp, codeEntryCodeLen, - &c.coh, &c.codeSize, 8) + &c.coh, &c.codeSize, 8, &c.putStripes[uint8(codeID)]) } // GetAddrCodeHash returns the Ethereum codeHash for addr if cached. Lets @@ -460,8 +470,9 @@ func (c *CodeCache) putWithCodeHash(addr []byte, code []byte, codeHash []byte, t entry := codeEntry{code: code, keyHash: kh, txNum: txNum, epoch: ep} // freelru keyed by maphash(codeHash); 32-byte key cost. - putContent(c.codeHashToCode, maphash.Hash(codeHash), entry, codeEntryStamp, codeEntryCodeLen, - &c.coh, &c.codeHashCodeSize, int64(len(codeHash))) + hcc := maphash.Hash(codeHash) + putContent(c.codeHashToCode, hcc, entry, codeEntryStamp, codeEntryCodeLen, + &c.coh, &c.codeHashCodeSize, int64(len(codeHash)), &c.putStripes[uint8(hcc)]) } // GetCodeSizeByCodeHash retrieves the size (in bytes) of a contract by its @@ -502,8 +513,9 @@ func (c *CodeCache) PutCodeSizeByCodeHash(codeHash []byte, size int, txNum uint6 kh := hash32(codeHash) entry := codeSizeEntry{size: size, keyHash: kh, txNum: txNum, epoch: ep} // Entry-counted layer: each entry costs 1 against the entry cap. - putContent(c.codeSizeByCodeHash, maphash.Hash(codeHash), entry, codeSizeEntryStamp, zeroCost, - &c.coh, &c.codeSizeEntries, 1) + hcs := maphash.Hash(codeHash) + putContent(c.codeSizeByCodeHash, hcs, entry, codeSizeEntryStamp, zeroCost, + &c.coh, &c.codeSizeEntries, 1, &c.putStripes[uint8(hcs)]) } // Delete removes the address → code mapping for addr. From 681de6b3718228304d0def895ca67436dc554697 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 3 Jul 2026 09:50:34 +0000 Subject: [PATCH 08/18] execution/commitment, db/state: byte-budget the BranchCache LRU tail MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The BranchCache eagerly allocated a production-sized LRU tail (50k entries, ~3.5MB) at construction. In production there is one process-wide cache so the cost is paid once, but tests build one aggregator per test file for isolation (thousands in execution/tests), turning that eager tail into GBs of allocation churn — enough under -race to OOM the CI runner (peak RSS ~49GB vs ~23GB with the cache stack off). Add AggOpts.BranchCacheTailBudget: a byte budget translated to a tail entry count via TailCapacityForBudget. NewTest uses a small budget (256KB) so test aggregators keep the cache fully functional — the static resident trunk and the LRU tail are both exercised — while the tail stays cheap (test workloads don't rely on tail retention). Production is unchanged (default 50k entries). Measured execution/tests -race peak RSS: 49GB -> 22GB. --- db/state/aggregator.go | 15 +++++++++++---- db/state/aggregator2.go | 27 +++++++++++++++++++++------ execution/commitment/branch_cache.go | 18 ++++++++++++++++++ 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/db/state/aggregator.go b/db/state/aggregator.go index e9606bb8704..31c7eeb6f08 100644 --- a/db/state/aggregator.go +++ b/db/state/aggregator.go @@ -33,6 +33,8 @@ import ( "sync/atomic" "time" + "github.com/c2h5oh/datasize" + "github.com/erigontech/erigon/db/kv/prune" "golang.org/x/sync/errgroup" @@ -88,9 +90,10 @@ type Aggregator struct { oldestVisible *aggregatorVisible snapshotBuildSema *semaphore.Weighted - disableHistory bool - branchCacheDisabled bool - workers workersCfg + disableHistory bool + branchCacheDisabled bool + branchCacheTailBudget datasize.ByteSize + workers workersCfg // To keep DB small - need move data to small files ASAP. // It means goroutine which creating small files - can't be locked by merge or indexing. @@ -417,7 +420,11 @@ func (a *Aggregator) ConfigureDomains() error { // opt out (e.g. one-shot genesis processing has no cross-block reuse). if dbg.UseStateCache && !a.branchCacheDisabled { if cd := a.d[kv.CommitmentDomain]; cd != nil && cd.branchCache == nil { - cd.branchCache = commitment.NewBranchCache(commitment.DefaultBranchCacheTailCapacity) + tailCap := commitment.DefaultBranchCacheTailCapacity + if a.branchCacheTailBudget > 0 { + tailCap = commitment.TailCapacityForBudget(a.branchCacheTailBudget) + } + cd.branchCache = commitment.NewBranchCache(tailCap) } } diff --git a/db/state/aggregator2.go b/db/state/aggregator2.go index da0bb2e0cf5..04c3e650451 100644 --- a/db/state/aggregator2.go +++ b/db/state/aggregator2.go @@ -10,6 +10,8 @@ import ( "strings" "sync" + "github.com/c2h5oh/datasize" + "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/config3" @@ -30,11 +32,12 @@ type AggOpts struct { //nolint:gocritic referencesInCommitmentBranches *bool // nil = leave global schema default untouched - genSaltIfNeed bool - sanityOldNaming bool // prevent start directory with old file names - disableFsync bool // for tests speed - disableHistory bool // for temp/inmem aggregator instances - disableBranchCache bool // for one-shot aggregators with no cross-block reuse (e.g. genesis) + genSaltIfNeed bool + sanityOldNaming bool // prevent start directory with old file names + disableFsync bool // for tests speed + disableHistory bool // for temp/inmem aggregator instances + disableBranchCache bool // for one-shot aggregators with no cross-block reuse (e.g. genesis) + branchCacheTailBudget datasize.ByteSize // 0 = production default; smaller caps the tail footprint for many-instance test aggregators } func New(dirs datadir.Dirs) AggOpts { //nolint:gocritic @@ -48,8 +51,15 @@ func New(dirs datadir.Dirs) AggOpts { //nolint:gocritic } } +// TestBranchCacheTailBudget caps the BranchCache LRU tail for test aggregators. +// Tests create many short-lived aggregators, so the production-sized tail would +// dominate memory; a small budget keeps the cache (and its static resident +// trunk) fully functional while the tail stays cheap — test workloads don't +// rely on tail retention. +const TestBranchCacheTailBudget = 256 * datasize.KB + func NewTest(dirs datadir.Dirs) AggOpts { //nolint:gocritic - return New(dirs).DisableFsync().GenSaltIfNeed(true).ReorgBlockDepth(0).StepSize(config3.DefaultStepSize).StepsInFrozenFile(config3.DefaultStepsInFrozenFile) + return New(dirs).DisableFsync().GenSaltIfNeed(true).ReorgBlockDepth(0).StepSize(config3.DefaultStepSize).StepsInFrozenFile(config3.DefaultStepsInFrozenFile).BranchCacheTailBudget(TestBranchCacheTailBudget) } func (opts AggOpts) Open(ctx context.Context, db kv.RoDB) (*Aggregator, error) { //nolint:gocritic @@ -76,6 +86,7 @@ func (opts AggOpts) Open(ctx context.Context, db kv.RoDB) (*Aggregator, error) { a.disableHistory = opts.disableHistory a.branchCacheDisabled = opts.disableBranchCache + a.branchCacheTailBudget = opts.branchCacheTailBudget a.disableFsync = opts.disableFsync a.savedSalt = salt @@ -127,6 +138,10 @@ func (opts AggOpts) DisableBranchCache() AggOpts { //nolint:gocritic opts.disableBranchCache = true return opts } +func (opts AggOpts) BranchCacheTailBudget(b datasize.ByteSize) AggOpts { //nolint:gocritic + opts.branchCacheTailBudget = b + return opts +} func (opts AggOpts) SanityOldNaming() AggOpts { //nolint:gocritic opts.sanityOldNaming = true return opts diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 3a8512f4091..cd82d80953f 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -22,6 +22,7 @@ import ( "os" "sync/atomic" + "github.com/c2h5oh/datasize" "github.com/elastic/go-freelru" "github.com/erigontech/erigon/common/log/v3" @@ -206,6 +207,23 @@ func (t *trunk) slot(path []byte) *atomic.Pointer[branchCacheEntry] { // at typical mainnet branch sizes. const DefaultBranchCacheTailCapacity = 50000 +// avgBranchEntryBytes is the assumed resident size of one cached branch +// (payload plus freelru/element overhead). Used to translate a byte budget +// into a tail entry count. +const avgBranchEntryBytes = 512 + +// TailCapacityForBudget converts a byte budget into a tail entry count, flooring +// at branchCacheTailShards so every shard keeps at least one slot. Lets callers +// that create many caches (e.g. per-test aggregators) cap the tail's resident +// footprint while keeping the static resident trunk fully functional. +func TailCapacityForBudget(budget datasize.ByteSize) int { + n := int(uint64(budget) / avgBranchEntryBytes) + if n < branchCacheTailShards { + n = branchCacheTailShards + } + return n +} + // BranchCacheProvider exposes the long-lived BranchCache attached to the // commitment domain. Implemented by *db/state.AggregatorRoTx (via duck // typing) so callers in the SharedDomains construction path can fetch the From 94d80952b7690869001dcc44240cfd7a1a636fec Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 3 Jul 2026 13:49:10 +0000 Subject: [PATCH 09/18] execution/commitment: demand-allocate BranchCache tiers to cut alloc churn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The trunk-pin cache stack allocated its per-instance structures eagerly at full production size. In production there is one process-wide cache so the cost is paid once, but the execution-test suite spins up thousands of short-lived caches over shallow tries, turning the eager allocation into churn — which the race detector shadows into an OOM on the 16GB -race runner. The cache is memory-neutral without -race (measured); this makes the footprint demand-driven so it stays small when the workload is small, full when it isn't, with the same code in tests and production. - Trunk tiers d2/d3/d4 (the dense 2KB/32KB/512KB arrays) allocate on first write at that depth, via CompareAndSwap, like d4 already did. A shallow trie pays ~160B instead of 35KB. - The pinned per-contract map allocates on the first pin instead of eagerly. - The LRU tail allocates on first spill and jump-grows toward its cap under a shared, memory-derived budget (estimate.TotalMemory) rather than pre-reserving the full 50k-entry freelru; removes the earlier test-specific tail budget. - onCacheMiss no longer allocates per miss: Load-first (no discarded atomic.Uint64) and ContractHashFromPrefix decodes the leading 64 nibbles straight out of the compact bytes instead of materializing the hex expansion. - HashSort pre-sizes its key arena to min(pending keys, batch) so small commitments (genesis, small blocks) don't reserve the full 3.84MB scratch. --- db/state/aggregator.go | 15 +- db/state/aggregator2.go | 27 +- execution/commitment/adaptive_pin.go | 7 +- execution/commitment/branch_cache.go | 259 ++++++++++++------ execution/commitment/branch_cache_tail.go | 178 ++++++++++++ execution/commitment/branch_cache_test.go | 2 +- execution/commitment/commitment.go | 5 +- .../commitment/contracthash_prefix_test.go | 73 +++++ 8 files changed, 443 insertions(+), 123 deletions(-) create mode 100644 execution/commitment/branch_cache_tail.go create mode 100644 execution/commitment/contracthash_prefix_test.go diff --git a/db/state/aggregator.go b/db/state/aggregator.go index 31c7eeb6f08..e9606bb8704 100644 --- a/db/state/aggregator.go +++ b/db/state/aggregator.go @@ -33,8 +33,6 @@ import ( "sync/atomic" "time" - "github.com/c2h5oh/datasize" - "github.com/erigontech/erigon/db/kv/prune" "golang.org/x/sync/errgroup" @@ -90,10 +88,9 @@ type Aggregator struct { oldestVisible *aggregatorVisible snapshotBuildSema *semaphore.Weighted - disableHistory bool - branchCacheDisabled bool - branchCacheTailBudget datasize.ByteSize - workers workersCfg + disableHistory bool + branchCacheDisabled bool + workers workersCfg // To keep DB small - need move data to small files ASAP. // It means goroutine which creating small files - can't be locked by merge or indexing. @@ -420,11 +417,7 @@ func (a *Aggregator) ConfigureDomains() error { // opt out (e.g. one-shot genesis processing has no cross-block reuse). if dbg.UseStateCache && !a.branchCacheDisabled { if cd := a.d[kv.CommitmentDomain]; cd != nil && cd.branchCache == nil { - tailCap := commitment.DefaultBranchCacheTailCapacity - if a.branchCacheTailBudget > 0 { - tailCap = commitment.TailCapacityForBudget(a.branchCacheTailBudget) - } - cd.branchCache = commitment.NewBranchCache(tailCap) + cd.branchCache = commitment.NewBranchCache(commitment.DefaultBranchCacheTailCapacity) } } diff --git a/db/state/aggregator2.go b/db/state/aggregator2.go index 04c3e650451..da0bb2e0cf5 100644 --- a/db/state/aggregator2.go +++ b/db/state/aggregator2.go @@ -10,8 +10,6 @@ import ( "strings" "sync" - "github.com/c2h5oh/datasize" - "github.com/erigontech/erigon/common/dbg" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/db/config3" @@ -32,12 +30,11 @@ type AggOpts struct { //nolint:gocritic referencesInCommitmentBranches *bool // nil = leave global schema default untouched - genSaltIfNeed bool - sanityOldNaming bool // prevent start directory with old file names - disableFsync bool // for tests speed - disableHistory bool // for temp/inmem aggregator instances - disableBranchCache bool // for one-shot aggregators with no cross-block reuse (e.g. genesis) - branchCacheTailBudget datasize.ByteSize // 0 = production default; smaller caps the tail footprint for many-instance test aggregators + genSaltIfNeed bool + sanityOldNaming bool // prevent start directory with old file names + disableFsync bool // for tests speed + disableHistory bool // for temp/inmem aggregator instances + disableBranchCache bool // for one-shot aggregators with no cross-block reuse (e.g. genesis) } func New(dirs datadir.Dirs) AggOpts { //nolint:gocritic @@ -51,15 +48,8 @@ func New(dirs datadir.Dirs) AggOpts { //nolint:gocritic } } -// TestBranchCacheTailBudget caps the BranchCache LRU tail for test aggregators. -// Tests create many short-lived aggregators, so the production-sized tail would -// dominate memory; a small budget keeps the cache (and its static resident -// trunk) fully functional while the tail stays cheap — test workloads don't -// rely on tail retention. -const TestBranchCacheTailBudget = 256 * datasize.KB - func NewTest(dirs datadir.Dirs) AggOpts { //nolint:gocritic - return New(dirs).DisableFsync().GenSaltIfNeed(true).ReorgBlockDepth(0).StepSize(config3.DefaultStepSize).StepsInFrozenFile(config3.DefaultStepsInFrozenFile).BranchCacheTailBudget(TestBranchCacheTailBudget) + return New(dirs).DisableFsync().GenSaltIfNeed(true).ReorgBlockDepth(0).StepSize(config3.DefaultStepSize).StepsInFrozenFile(config3.DefaultStepsInFrozenFile) } func (opts AggOpts) Open(ctx context.Context, db kv.RoDB) (*Aggregator, error) { //nolint:gocritic @@ -86,7 +76,6 @@ func (opts AggOpts) Open(ctx context.Context, db kv.RoDB) (*Aggregator, error) { a.disableHistory = opts.disableHistory a.branchCacheDisabled = opts.disableBranchCache - a.branchCacheTailBudget = opts.branchCacheTailBudget a.disableFsync = opts.disableFsync a.savedSalt = salt @@ -138,10 +127,6 @@ func (opts AggOpts) DisableBranchCache() AggOpts { //nolint:gocritic opts.disableBranchCache = true return opts } -func (opts AggOpts) BranchCacheTailBudget(b datasize.ByteSize) AggOpts { //nolint:gocritic - opts.branchCacheTailBudget = b - return opts -} func (opts AggOpts) SanityOldNaming() AggOpts { //nolint:gocritic opts.sanityOldNaming = true return opts diff --git a/execution/commitment/adaptive_pin.go b/execution/commitment/adaptive_pin.go index 91a7107adb6..9db4db62d5f 100644 --- a/execution/commitment/adaptive_pin.go +++ b/execution/commitment/adaptive_pin.go @@ -154,6 +154,10 @@ func (c *AdaptivePinController) onCacheMiss(prefix []byte) { if !ok { return } + if v, ok := c.misses.Load(hash); ok { + v.(*atomic.Uint64).Add(1) + return + } v, _ := c.misses.LoadOrStore(hash, new(atomic.Uint64)) v.(*atomic.Uint64).Add(1) } @@ -251,8 +255,7 @@ func (c *AdaptivePinController) snapshotMisses() map[[32]byte]uint64 { out := make(map[[32]byte]uint64) c.misses.Range(func(k, v any) bool { hash := k.([32]byte) - n := v.(*atomic.Uint64).Swap(0) - if n > 0 { + if n := v.(*atomic.Uint64).Swap(0); n > 0 { out[hash] = n } return true diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index cd82d80953f..36c9369b2bd 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -20,11 +20,9 @@ import ( "bytes" "fmt" "os" + "sync" "sync/atomic" - "github.com/c2h5oh/datasize" - "github.com/elastic/go-freelru" - "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/maphash" "github.com/erigontech/erigon/execution/cache/coherence" @@ -79,12 +77,20 @@ type BranchCache struct { // but still honor the (txN, epoch) unwind model. Lookup checks this tier // between the account trunk and the tail. pinnedEntries counts filled // storage slots across all storageTrunks. - pinned *maphash.Map[*trunk] + // Allocated on the first pin (via pinnedForWrite): a cache that never pins a + // contract — the common case for short-lived caches over shallow tries — + // never pays for the (min 32-bucket) concurrent map. + pinned atomic.Pointer[maphash.Map[*trunk]] + pinnedMu sync.Mutex pinnedEntries atomic.Int64 - // LRU tail — bounded entries, evicts oldest when full. freelru.ShardedLRU - // keyed by the maphash of the prefix (single-alloc, thread-safe per shard). - tail *freelru.ShardedLRU[uint64, *branchCacheEntry] + // LRU tail — the memory-adaptive spill tier for prefixes past the resident + // trunk. Allocated on the first tail insert and jump-grown toward tailCap only + // as demand and the shared memory budget allow (see tailLRU), so a cache over + // a shallow trie or in a memory-constrained process stays small. + tail atomic.Pointer[tailLRU] + tailCap uint32 + tailMu sync.Mutex // trunkDisabled (env BRANCH_CACHE_TRUNK_DISABLE) routes depth-1-4 account // branches back to the LRU tail instead of the resident account trunk — a @@ -156,19 +162,62 @@ type MissCallback func(prefix []byte) // atomic.Pointer: under the single-writer-per-prefix invariant readers/writers // take no mutex (just an atomic load/store per slot); only deep (a maphash.Map) // locks. +// The depth tiers d2/d3/d4 are allocated lazily on the first write at that +// depth. A process-wide production cache fills every tier once; the many +// short-lived caches a test suite spins up over shallow tries never reach the +// deeper tiers, so eager allocation of the dense arrays (d3 32KB, d4 512KB) is +// pure churn there. d0/d1 are tiny and always reached, so they stay inline. type trunk struct { - d0 atomic.Pointer[branchCacheEntry] - d1 [16]atomic.Pointer[branchCacheEntry] - d2 [256]atomic.Pointer[branchCacheEntry] - d3 [4096]atomic.Pointer[branchCacheEntry] - // d4 (the 65536-entry depth-4 array, ~512KB) is allocated lazily on the - // first depth-4 write: prod has one process-wide cache so eager alloc is - // cheap, but tests spin up thousands of ephemeral aggregators whose tries - // rarely reach depth 4, so eager alloc there is pure churn. + d0 atomic.Pointer[branchCacheEntry] + d1 [16]atomic.Pointer[branchCacheEntry] + d2 atomic.Pointer[[256]atomic.Pointer[branchCacheEntry]] + d3 atomic.Pointer[[4096]atomic.Pointer[branchCacheEntry]] d4 atomic.Pointer[[65536]atomic.Pointer[branchCacheEntry]] deep *maphash.Map[*branchCacheEntry] } +func (t *trunk) d2For(forWrite bool) *[256]atomic.Pointer[branchCacheEntry] { + if p := t.d2.Load(); p != nil { + return p + } + if !forWrite { + return nil + } + p := &[256]atomic.Pointer[branchCacheEntry]{} + if !t.d2.CompareAndSwap(nil, p) { + p = t.d2.Load() + } + return p +} + +func (t *trunk) d3For(forWrite bool) *[4096]atomic.Pointer[branchCacheEntry] { + if p := t.d3.Load(); p != nil { + return p + } + if !forWrite { + return nil + } + p := &[4096]atomic.Pointer[branchCacheEntry]{} + if !t.d3.CompareAndSwap(nil, p) { + p = t.d3.Load() + } + return p +} + +func (t *trunk) d4For(forWrite bool) *[65536]atomic.Pointer[branchCacheEntry] { + if p := t.d4.Load(); p != nil { + return p + } + if !forWrite { + return nil + } + p := &[65536]atomic.Pointer[branchCacheEntry]{} + if !t.d4.CompareAndSwap(nil, p) { + p = t.d4.Load() + } + return p +} + // newAccountTrunk builds the global account trunk: dense depth-4 fixed array, // no deep overflow (account depth 5+ uses the LRU tail). func newAccountTrunk() *trunk { @@ -184,18 +233,22 @@ func newStorageTrunk() *trunk { // slot returns the fixed-array slot for a nibble path of length 0-3 (and length // 4 when the depth-4 array is present, i.e. the account trunk), or nil when the // path is deeper — the caller then uses deep (storage) or the tail (account). -func (t *trunk) slot(path []byte) *atomic.Pointer[branchCacheEntry] { +func (t *trunk) slot(path []byte, forWrite bool) *atomic.Pointer[branchCacheEntry] { switch len(path) { case 0: return &t.d0 case 1: return &t.d1[path[0]] case 2: - return &t.d2[uint16(path[0])<<4|uint16(path[1])] + if d2 := t.d2For(forWrite); d2 != nil { + return &d2[uint16(path[0])<<4|uint16(path[1])] + } case 3: - return &t.d3[uint16(path[0])<<8|uint16(path[1])<<4|uint16(path[2])] + if d3 := t.d3For(forWrite); d3 != nil { + return &d3[uint16(path[0])<<8|uint16(path[1])<<4|uint16(path[2])] + } case 4: - if d4 := t.d4.Load(); d4 != nil { + if d4 := t.d4For(forWrite); d4 != nil { return &d4[uint32(path[0])<<12|uint32(path[1])<<8|uint32(path[2])<<4|uint32(path[3])] } } @@ -207,23 +260,6 @@ func (t *trunk) slot(path []byte) *atomic.Pointer[branchCacheEntry] { // at typical mainnet branch sizes. const DefaultBranchCacheTailCapacity = 50000 -// avgBranchEntryBytes is the assumed resident size of one cached branch -// (payload plus freelru/element overhead). Used to translate a byte budget -// into a tail entry count. -const avgBranchEntryBytes = 512 - -// TailCapacityForBudget converts a byte budget into a tail entry count, flooring -// at branchCacheTailShards so every shard keeps at least one slot. Lets callers -// that create many caches (e.g. per-test aggregators) cap the tail's resident -// footprint while keeping the static resident trunk fully functional. -func TailCapacityForBudget(budget datasize.ByteSize) int { - n := int(uint64(budget) / avgBranchEntryBytes) - if n < branchCacheTailShards { - n = branchCacheTailShards - } - return n -} - // BranchCacheProvider exposes the long-lived BranchCache attached to the // commitment domain. Implemented by *db/state.AggregatorRoTx (via duck // typing) so callers in the SharedDomains construction path can fetch the @@ -247,15 +283,9 @@ func NewBranchCache(tailCapacity int) *BranchCache { if tailCapacity <= 0 { panic(fmt.Sprintf("BranchCache: tailCapacity must be positive, got %d", tailCapacity)) } - tailCap := uint32(tailCapacity) - tail, err := freelru.NewShardedWithSize[uint64, *branchCacheEntry](branchCacheTailShards, tailCap, tailCap+tailCap/4, u64ident) - if err != nil { - panic(fmt.Sprintf("BranchCache: NewShardedWithSize: %s", err)) - } bc := &BranchCache{ - tail: tail, + tailCap: uint32(tailCapacity), accountTrunk: newAccountTrunk(), - pinned: maphash.NewMap[*trunk](), trunkDisabled: os.Getenv("BRANCH_CACHE_TRUNK_DISABLE") != "", } // Before any unwind every entry's txN is at/below the floor, so the epoch @@ -265,6 +295,31 @@ func NewBranchCache(tailCapacity int) *BranchCache { return bc } +// tailForWrite returns the LRU tail, allocating it on first use so a cache whose +// tries never spill past the resident trunk pays nothing for it. +func (c *BranchCache) tailForWrite() *tailLRU { + if t := c.tail.Load(); t != nil { + return t + } + c.tailMu.Lock() + defer c.tailMu.Unlock() + if t := c.tail.Load(); t != nil { + return t + } + t := newTailLRU(c.tailCap) + c.tail.Store(t) + return t +} + +// tailLen reports the number of resident tail entries, or 0 if the tail has not +// been allocated yet. +func (c *BranchCache) tailLen() int { + if t := c.tail.Load(); t != nil { + return t.Len() + } + return 0 +} + // trunkSlot returns the resident account-trunk slot for an account-trie branch // at nibble depth 1-4, or nil if the prefix is the root (depth 0), a storage // trunk, or depth >= 5 (served by the LRU tail). The compact-hex prefix maps @@ -281,22 +336,21 @@ func (c *BranchCache) trunkSlot(prefix []byte, forWrite bool) *atomic.Pointer[br } case 2: if prefix[0]&0x10 == 0 { // 2 nibbles - return &c.accountTrunk.d2[prefix[1]] + if d2 := c.accountTrunk.d2For(forWrite); d2 != nil { + return &d2[prefix[1]] + } + return nil + } + if d3 := c.accountTrunk.d3For(forWrite); d3 != nil { // 3 nibbles + return &d3[uint16(prefix[0]&0x0f)<<8|uint16(prefix[1])] } - return &c.accountTrunk.d3[uint16(prefix[0]&0x0f)<<8|uint16(prefix[1])] // 3 nibbles + return nil case 3: if prefix[0]&0x10 == 0 { // 4 nibbles - d4 := c.accountTrunk.d4.Load() - if d4 == nil { - if !forWrite { - return nil - } - d4 = &[65536]atomic.Pointer[branchCacheEntry]{} - if !c.accountTrunk.d4.CompareAndSwap(nil, d4) { - d4 = c.accountTrunk.d4.Load() // lost the race; use the winner - } + if d4 := c.accountTrunk.d4For(forWrite); d4 != nil { + return &d4[uint16(prefix[1])<<8|uint16(prefix[2])] } - return &d4[uint16(prefix[1])<<8|uint16(prefix[2])] + return nil } // 5 nibbles (odd, 3 bytes) -> LRU tail } @@ -322,32 +376,51 @@ func (c *BranchCache) storageRoute(prefix []byte, create bool) (st *trunk, acct packed[i] = nib[2*i]<<4 | nib[2*i+1] } stor = nib[64:] - st, found := c.pinned.Get(packed) - if !found { - if !create { - return nil, packed, stor, false + if p := c.pinned.Load(); p != nil { + if st, found := p.Get(packed); found { + return st, packed, stor, true } - st = newStorageTrunk() - c.pinned.Set(packed, st) } + if !create { + return nil, packed, stor, false + } + st = newStorageTrunk() + c.pinnedForWrite().Set(packed, st) return st, packed, stor, true } -// ContractHashFromPrefix extracts the 32-byte contract (account) hash from a -// storage-trunk prefix (compact-hex of >= 64 account nibbles + storage -// nibbles). ok=false for non-storage prefixes. Used by the residency layer to -// attribute per-contract miss pressure. +// pinnedForWrite returns the pinned-contract map, allocating it on first pin. +func (c *BranchCache) pinnedForWrite() *maphash.Map[*trunk] { + if p := c.pinned.Load(); p != nil { + return p + } + c.pinnedMu.Lock() + defer c.pinnedMu.Unlock() + if p := c.pinned.Load(); p != nil { + return p + } + p := maphash.NewMap[*trunk]() + c.pinned.Store(p) + return p +} + +// ContractHashFromPrefix extracts the 32-byte contract (account) hash — keccak +// of the address — from a storage-trunk prefix (compact-hex of >= 64 account +// nibbles + storage nibbles). ok=false for non-storage prefixes. On the +// per-miss hot path, so it decodes the leading 64 nibbles straight out of the +// compact bytes rather than materializing the full hex expansion. func ContractHashFromPrefix(prefix []byte) (hash [32]byte, ok bool) { if len(prefix) < 33 { return hash, false } - nib := nibbles.CompactToHex(prefix) - if len(nib) < 64 { - return hash, false - } - for i := 0; i < 32; i++ { - hash[i] = nib[2*i]<<4 | nib[2*i+1] + if prefix[0]&0x10 != 0 { // odd: first nibble is the low nibble of byte 0 + for i := 0; i < 32; i++ { + hash[i] = prefix[i]&0x0f<<4 | prefix[i+1]>>4 + } + return hash, true } + // even: the account-hash bytes are stored whole starting at byte 1 + copy(hash[:], prefix[1:33]) return hash, true } @@ -360,11 +433,15 @@ func (c *BranchCache) clearTrunk() { for i := range t.d1 { t.d1[i].Store(nil) } - for i := range t.d2 { - t.d2[i].Store(nil) + if d2 := t.d2.Load(); d2 != nil { + for i := range d2 { + d2[i].Store(nil) + } } - for i := range t.d3 { - t.d3[i].Store(nil) + if d3 := t.d3.Load(); d3 != nil { + for i := range d3 { + d3[i].Store(nil) + } } if d4 := t.d4.Load(); d4 != nil { for i := range d4 { @@ -424,7 +501,7 @@ func (c *BranchCache) lookup(prefix []byte) (*branchCacheEntry, bool) { // pinned hit/miss stats; account-trie and tail-only prefixes are excluded. if st, _, stor, ok := c.storageRoute(prefix, false); ok { var entry *branchCacheEntry - if slot := st.slot(stor); slot != nil { + if slot := st.slot(stor, false); slot != nil { entry = slot.Load() } else { entry, _ = st.deep.Get(prefix) @@ -435,7 +512,13 @@ func (c *BranchCache) lookup(prefix []byte) (*branchCacheEntry, bool) { } c.pinnedMisses.Add(1) } - entry, ok := c.tail.Get(maphash.Hash(prefix)) + tail := c.tail.Load() + if tail == nil { + c.tailMisses.Add(1) + c.fireOnMiss(prefix) + return nil, false + } + entry, ok := tail.Get(maphash.Hash(prefix)) if !ok { c.tailMisses.Add(1) c.fireOnMiss(prefix) @@ -457,7 +540,7 @@ func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { // Keep a prefix already pinned in a storage trunk in place across the // per-block invalidate+Put refresh rather than dropping it to the tail. if st, _, stor, ok := c.storageRoute(prefix, false); ok { - if slot := st.slot(stor); slot != nil { + if slot := st.slot(stor, false); slot != nil { if slot.Load() != nil { slot.Store(entry) return @@ -467,7 +550,7 @@ func (c *BranchCache) store(prefix []byte, entry *branchCacheEntry) { return } } - c.tail.Add(maphash.Hash(prefix), entry) + c.tailForWrite().Add(maphash.Hash(prefix), entry) } // PinEntry inserts or replaces a pinned cache entry for prefix in its contract's @@ -483,10 +566,10 @@ func (c *BranchCache) PinEntry(prefix []byte, data []byte, step, txN uint64) { entry := &branchCacheEntry{data: dataCopy, step: step, txN: txN, epoch: c.coh.Epoch()} st, _, stor, ok := c.storageRoute(prefix, true) if !ok { - c.tail.Add(maphash.Hash(prefix), entry) + c.tailForWrite().Add(maphash.Hash(prefix), entry) return } - if slot := st.slot(stor); slot != nil { + if slot := st.slot(stor, true); slot != nil { if slot.Load() == nil { c.pinnedEntries.Add(1) } @@ -572,7 +655,7 @@ func (c *BranchCache) Invalidate(prefix []byte) { return } if st, _, stor, ok := c.storageRoute(prefix, false); ok { - if slot := st.slot(stor); slot != nil { + if slot := st.slot(stor, false); slot != nil { if slot.Swap(nil) != nil { c.pinnedEntries.Add(-1) } @@ -581,7 +664,9 @@ func (c *BranchCache) Invalidate(prefix []byte) { c.pinnedEntries.Add(-1) } } - c.tail.Remove(maphash.Hash(prefix)) + if tail := c.tail.Load(); tail != nil { + tail.Remove(maphash.Hash(prefix)) + } } // Unwind invalidates entries that reflect dead-fork state. unwindToTxN is the @@ -602,9 +687,11 @@ func (c *BranchCache) Unwind(unwindToTxN uint64) { func (c *BranchCache) Clear() { c.root.Store(nil) c.clearTrunk() - c.pinned = maphash.NewMap[*trunk]() + c.pinned.Store(nil) c.pinnedEntries.Store(0) - c.tail.Purge() + if tail := c.tail.Load(); tail != nil { + tail.reset() + } c.rootHits.Store(0) c.rootMisses.Store(0) c.trunkHits.Store(0) @@ -639,7 +726,7 @@ func (c *BranchCache) Stats() string { rh, rm, pct(rh, rm), kh, km, pct(kh, km), ph, pm, pct(ph, pm), int(c.pinnedEntries.Load()), - th, tm, pct(th, tm), c.tail.Len(), + th, tm, pct(th, tm), c.tailLen(), float64(bb)/1024/1024, c.staleEvicted.Load(), ) } diff --git a/execution/commitment/branch_cache_tail.go b/execution/commitment/branch_cache_tail.go new file mode 100644 index 00000000000..69ce1d04cbf --- /dev/null +++ b/execution/commitment/branch_cache_tail.go @@ -0,0 +1,178 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package commitment + +import ( + "fmt" + "sync" + "sync/atomic" + + "github.com/elastic/go-freelru" + + "github.com/erigontech/erigon/common/estimate" +) + +const ( + // tailStartCapacity is the entry count a tail is first sized to. It jump-grows + // (×tailGrowFactor) toward its max only while the shared budget allows, so a + // cache over a shallow trie stays at the start size. + tailStartCapacity = 512 + tailGrowFactor = 4 + // tailEntryBytes is the assumed resident cost of one tail slot (freelru + // element + the branch payload it points at), used only for budget accounting. + tailEntryBytes = 512 +) + +// tailBudget caps the total resident memory of every BranchCache LRU tail in the +// process to a fraction of the memory actually available (system RAM, cgroup +// limit, or GOMEMLIMIT — whichever is lowest). A single production cache draws +// the whole budget and grows to its full cap; a process that spins up thousands +// of ephemeral caches (the execution-test suite) has them share the budget, so +// each stays small — with no test-specific code. +type tailBudget struct { + limit int64 + used atomic.Int64 +} + +var globalTailBudget = &tailBudget{limit: int64(estimate.TotalMemory() / tailBudgetDivisor)} + +// tailBudgetDivisor keeps the whole-process tail budget to a small fraction of +// available memory; the tail is a cache, so undersizing only costs hit rate. +const tailBudgetDivisor = 32 + +// reserve grabs n bytes if the budget has room, returning false when full. +func (b *tailBudget) reserve(n int64) bool { + for { + used := b.used.Load() + if used+n > b.limit { + return false + } + if b.used.CompareAndSwap(used, used+n) { + return true + } + } +} + +func (b *tailBudget) release(n int64) { + if n > 0 { + b.used.Add(-n) + } +} + +// tailLRU is the BranchCache LRU tail. It wraps a sharded freelru that is +// jump-resized (allocate larger, copy the live entries over) as it fills, +// bounded by the shared tailBudget and a per-cache max. Reads and writes take no +// tail-level lock — they load the current freelru atomically and rely on its own +// per-shard locking; the resize mutex is held only during the rare grow. A write +// racing a resize may land in the freelru about to be replaced and be dropped, +// which is a benign cache miss (the branch is re-read from the authoritative DB). +type tailLRU struct { + cur atomic.Pointer[freelru.ShardedLRU[uint64, *branchCacheEntry]] + maxCap uint32 + + resizeMu sync.Mutex + curCap uint32 + reserved int64 +} + +func newTailLRU(maxCapacity uint32) *tailLRU { + start := uint32(tailStartCapacity) + if start > maxCapacity { + start = maxCapacity + } + t := &tailLRU{maxCap: maxCapacity} + t.reserved = int64(start) * tailEntryBytes + globalTailBudget.reserve(t.reserved) // initial slice is small; take it unconditionally + t.curCap = start + t.cur.Store(newTailShards(start)) + return t +} + +func newTailShards(capacity uint32) *freelru.ShardedLRU[uint64, *branchCacheEntry] { + lru, err := freelru.NewShardedWithSize[uint64, *branchCacheEntry]( + branchCacheTailShards, capacity, capacity+capacity/4, u64ident) + if err != nil { + panic(fmt.Sprintf("BranchCache tail: NewShardedWithSize(%d): %s", capacity, err)) + } + return lru +} + +func (t *tailLRU) Get(key uint64) (*branchCacheEntry, bool) { + return t.cur.Load().Get(key) +} + +func (t *tailLRU) Add(key uint64, entry *branchCacheEntry) { + lru := t.cur.Load() + if lru.Len() >= int(t.curCap) { + t.maybeGrow() + lru = t.cur.Load() + } + lru.Add(key, entry) +} + +// maybeGrow jump-resizes the tail one step larger when it is full, the per-cache +// max hasn't been reached, and the shared budget has room. Otherwise the tail +// keeps its size and freelru evicts LRU on the next insert. +func (t *tailLRU) maybeGrow() { + t.resizeMu.Lock() + defer t.resizeMu.Unlock() + + old := t.cur.Load() + if t.curCap >= t.maxCap || old.Len() < int(t.curCap) { + return + } + newCap := t.curCap * tailGrowFactor + if newCap > t.maxCap { + newCap = t.maxCap + } + delta := int64(newCap-t.curCap) * tailEntryBytes + if !globalTailBudget.reserve(delta) { + return + } + next := newTailShards(newCap) + for _, k := range old.Keys() { + if v, ok := old.Get(k); ok { + next.Add(k, v) + } + } + t.cur.Store(next) + t.curCap = newCap + t.reserved += delta +} + +func (t *tailLRU) Remove(key uint64) { + t.cur.Load().Remove(key) +} + +// reset shrinks the tail back to the start size and returns its budget, keeping +// the cache adaptive across unwind/clear (it regrows on demand afterwards). +func (t *tailLRU) reset() { + t.resizeMu.Lock() + defer t.resizeMu.Unlock() + start := uint32(tailStartCapacity) + if start > t.maxCap { + start = t.maxCap + } + globalTailBudget.release(t.reserved - int64(start)*tailEntryBytes) + t.reserved = int64(start) * tailEntryBytes + t.curCap = start + t.cur.Store(newTailShards(start)) +} + +func (t *tailLRU) Len() int { + return t.cur.Load().Len() +} diff --git a/execution/commitment/branch_cache_test.go b/execution/commitment/branch_cache_test.go index 97ce5b8cae8..6541f365c8a 100644 --- a/execution/commitment/branch_cache_test.go +++ b/execution/commitment/branch_cache_test.go @@ -124,7 +124,7 @@ func TestBranchCache_RootSurvivesEvictionPressure(t *testing.T) { require.Equal(t, []byte("ROOT-PERSISTS"), got) // Tail at capacity (10), not 100 - require.LessOrEqual(t, c.tail.Len(), 10, "tail should respect LRU capacity") + require.LessOrEqual(t, c.tailLen(), 10, "tail should respect LRU capacity") } // TestBranchCache_Invalidate removes entries from both tiers. diff --git a/execution/commitment/commitment.go b/execution/commitment/commitment.go index 7f11d71414b..c709cd62aa5 100644 --- a/execution/commitment/commitment.go +++ b/execution/commitment/commitment.go @@ -1871,6 +1871,7 @@ const hashSortBatchSize = 10_000 func (t *Updates) HashSort(ctx context.Context, warmuper *Warmuper, fn func(hk, pk []byte, update *Update) error) error { switch t.mode { case ModeDirect: + cnt := len(t.keys) clear(t.keys) t.batchSlab = t.batchSlab[:0] @@ -1880,7 +1881,7 @@ func (t *Updates) HashSort(ctx context.Context, warmuper *Warmuper, fn func(hk, } } // Pre-size the arena so a mid-batch grow can't reallocate and invalidate live sub-slices (≤180 B/key, 192 with headroom). - t.arenaEnsureCap(hashSortBatchSize * 192) + t.arenaEnsureCap(min(cnt, hashSortBatchSize) * 192) t.arenas[t.curArena] = t.arenas[t.curArena][:0] var prevKey []byte @@ -1955,7 +1956,7 @@ func (t *Updates) HashSort(ctx context.Context, warmuper *Warmuper, fn func(hk, return err } } - t.arenaEnsureCap(hashSortBatchSize * 144) + t.arenaEnsureCap(min(t.tree.Len(), hashSortBatchSize) * 144) t.arenas[t.curArena] = t.arenas[t.curArena][:0] var prevKey []byte var processErr error diff --git a/execution/commitment/contracthash_prefix_test.go b/execution/commitment/contracthash_prefix_test.go new file mode 100644 index 00000000000..42988b3463d --- /dev/null +++ b/execution/commitment/contracthash_prefix_test.go @@ -0,0 +1,73 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package commitment + +import ( + "math/rand" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/erigontech/erigon/execution/commitment/nibbles" +) + +// reference is the pre-optimization implementation kept as the oracle: it +// materializes the full hex expansion via CompactToHex, then repacks the first +// 32 bytes. The zero-alloc ContractHashFromPrefix must agree with it exactly. +func contractHashFromPrefixReference(prefix []byte) (hash [32]byte, ok bool) { + if len(prefix) < 33 { + return hash, false + } + nib := nibbles.CompactToHex(prefix) + if len(nib) < 64 { + return hash, false + } + for i := 0; i < 32; i++ { + hash[i] = nib[2*i]<<4 | nib[2*i+1] + } + return hash, true +} + +func TestContractHashFromPrefix_MatchesReference(t *testing.T) { + rng := rand.New(rand.NewSource(1)) + for n := 0; n < 5000; n++ { + l := 30 + rng.Intn(40) // spans below and above the 33-byte minimum + prefix := make([]byte, l) + rng.Read(prefix) + wantHash, wantOK := contractHashFromPrefixReference(prefix) + gotHash, gotOK := ContractHashFromPrefix(prefix) + require.Equalf(t, wantOK, gotOK, "ok mismatch len=%d prefix0=%#x", l, prefixByte0(prefix)) + require.Equalf(t, wantHash, gotHash, "hash mismatch len=%d prefix0=%#x", l, prefixByte0(prefix)) + } +} + +func prefixByte0(p []byte) byte { + if len(p) == 0 { + return 0 + } + return p[0] +} + +func TestContractHashFromPrefix_ZeroAlloc(t *testing.T) { + prefix := make([]byte, 40) + prefix[0] = 0x10 // odd flag set, to exercise the shifting branch + allocs := testing.AllocsPerRun(1000, func() { _, _ = ContractHashFromPrefix(prefix) }) + require.Zero(t, allocs, "ContractHashFromPrefix must not allocate") + prefix[0] = 0x00 // even branch + allocs = testing.AllocsPerRun(1000, func() { _, _ = ContractHashFromPrefix(prefix) }) + require.Zero(t, allocs, "ContractHashFromPrefix (even) must not allocate") +} From ee9e2dbb3b2760d91223b0a4f99f92bfed503a0b Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 3 Jul 2026 16:44:21 +0000 Subject: [PATCH 10/18] execution/cache, execution/commitment, common/cachebudget, db/state: bound app caches with one shared memory envelope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The cache stack pre-allocated each cache to its full configured byte budget at construction, so a process that builds many cache instances (the execution-test suite) committed hundreds of MB per instance and could exhaust a constrained runner. Make the app caches adaptive to their environment instead of turning them off for tests. - common/cachebudget: one process-wide envelope = TotalMemory/32 (cgroup- and GOMEMLIMIT-aware), the single knob for aggregate cache residency. Reserve is all-or-nothing per growth step; Take is the unconditional initial slice; Release returns bytes on teardown/clear. - execution/cache: GenericCache jump-grows — starts at 1024 slots and resizes ×4 toward the byte-budget ceiling, funding each step from the envelope; a cache with a small working set never pre-commits its budget. Account/storage (NewGenericCacheWithAvg, per-domain avg) and the code cache route through the envelope; Close returns the reservation, Clear shrinks back to the start size. - execution/commitment: the BranchCache LRU tail draws from the same envelope (dropping the separate tail budget). The trunk is fixed-size by design, so it adapts by active-instance count instead — full depth-4 residency for a handful of caches (production), shallow (depth-2, deeper branches spill to the tail) once many are live (the test suite). Close decrements the instance count. - db/state, execmoduletester: wire Close so the envelope and instance count track real concurrency; the test harness now uses the production default cache (the jump-grow keeps it small) rather than a hand-tuned test-only size. --- common/cachebudget/budget.go | 92 ++++++++ common/cachebudget/budget_test.go | 64 ++++++ db/state/aggregator.go | 5 +- execution/cache/cache.go | 3 + execution/cache/code_cache.go | 19 ++ execution/cache/generic_cache.go | 209 ++++++++++++++---- execution/cache/state_cache.go | 28 +-- execution/commitment/branch_cache.go | 67 +++++- execution/commitment/branch_cache_tail.go | 46 +--- .../execmoduletester/exec_module_tester.go | 14 +- 10 files changed, 434 insertions(+), 113 deletions(-) create mode 100644 common/cachebudget/budget.go create mode 100644 common/cachebudget/budget_test.go diff --git a/common/cachebudget/budget.go b/common/cachebudget/budget.go new file mode 100644 index 00000000000..fa1c724191a --- /dev/null +++ b/common/cachebudget/budget.go @@ -0,0 +1,92 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +// Package cachebudget bounds the total resident memory of the process-wide +// application caches (state, code) to a fraction of the memory actually +// available — system RAM, cgroup limit, or GOMEMLIMIT, whichever is lowest. +// +// Caches do not pre-commit their full configured size. They start small and +// grow in steps, reserving each step's bytes from one shared envelope; a step +// that would overflow the envelope is refused, so the cache stops growing and +// evicts within its current size instead. A cache with a small working set (a +// test fixture) therefore stays small regardless of its configured budget, +// while a busy production cache grows into it — and the sum across every cache +// instance in the process stays within the envelope. Release returns a cache's +// reserved bytes when it is torn down or cleared. No cache is ever disabled and +// there is no test-specific sizing. +package cachebudget + +import ( + "sync/atomic" + + "github.com/erigontech/erigon/common/estimate" +) + +// Divisor sets the single shared envelope — covering every application cache +// (state, code, and the commitment-branch LRU tail) — to this fraction of total +// available memory. It is the one knob governing aggregate cache residency: +// larger (e.g. 16) buys hit-rate on big nodes; 32 keeps a constrained 16GB CI +// runner well within bounds even under the race detector's memory multiplier. +const Divisor = 32 + +// Budget is a shared byte allowance drawn down by Reserve and returned by +// Release. Safe for concurrent use. +type Budget struct { + limit int64 + used atomic.Int64 +} + +func New(limit int64) *Budget { return &Budget{limit: limit} } + +// Global is the process-wide application-cache envelope. +var Global = New(int64(estimate.TotalMemory() / Divisor)) + +// Reserve takes exactly n bytes if the envelope has room, returning true; it +// takes nothing and returns false when full. A grow step calls this and stops +// growing on false. +func (b *Budget) Reserve(n int64) bool { + if n <= 0 { + return true + } + for { + used := b.used.Load() + if used+n > b.limit { + return false + } + if b.used.CompareAndSwap(used, used+n) { + return true + } + } +} + +// Take reserves n bytes unconditionally (may push used past limit). Used for a +// cache's initial small allocation, which must always succeed so no cache is +// born disabled. +func (b *Budget) Take(n int64) { + if n > 0 { + b.used.Add(n) + } +} + +// Release returns n bytes to the envelope. +func (b *Budget) Release(n int64) { + if n > 0 { + b.used.Add(-n) + } +} + +func (b *Budget) Limit() int64 { return b.limit } +func (b *Budget) Used() int64 { return b.used.Load() } diff --git a/common/cachebudget/budget_test.go b/common/cachebudget/budget_test.go new file mode 100644 index 00000000000..1eef73eeae8 --- /dev/null +++ b/common/cachebudget/budget_test.go @@ -0,0 +1,64 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package cachebudget + +import "testing" + +func TestReserveStopsAtLimit(t *testing.T) { + b := New(1000) + if !b.Reserve(600) { + t.Fatal("first Reserve(600) should fit") + } + if !b.Reserve(400) { + t.Fatal("second Reserve(400) should exactly fill") + } + if b.Reserve(1) { + t.Fatal("Reserve past the limit must fail and take nothing") + } + if b.Used() != 1000 { + t.Fatalf("used: got %d want 1000", b.Used()) + } +} + +func TestReleaseReopensRoom(t *testing.T) { + b := New(1000) + b.Reserve(1000) + b.Release(400) + if !b.Reserve(400) { + t.Fatal("after Release(400) a Reserve(400) should fit") + } + if b.Reserve(1) { + t.Fatal("still full after regrow") + } +} + +func TestTakeIsUnconditional(t *testing.T) { + b := New(100) + b.Take(500) // initial small allocation always succeeds even past limit + if b.Used() != 500 { + t.Fatalf("used: got %d want 500", b.Used()) + } + if b.Reserve(1) { + t.Fatal("over-committed envelope refuses further Reserve") + } +} + +func TestGlobalSizedFromMemory(t *testing.T) { + if Global.Limit() <= 0 { + t.Fatalf("Global envelope must be positive, got %d", Global.Limit()) + } +} diff --git a/db/state/aggregator.go b/db/state/aggregator.go index e9606bb8704..f5af079befb 100644 --- a/db/state/aggregator.go +++ b/db/state/aggregator.go @@ -632,9 +632,12 @@ func (a *Aggregator) Close() { } a.wg.Wait() - // A closed Aggregator may linger referenced; release the cached branch data eagerly. + // A closed Aggregator may linger referenced; release the cached branch data + // eagerly and drop this cache from the active-instance count so later + // BranchCaches size their trunk depth against real concurrency. if cd := a.d[kv.CommitmentDomain]; cd != nil && cd.branchCache != nil { cd.branchCache.Clear() + cd.branchCache.Close() } a.dirtyFilesLock.Lock() diff --git a/execution/cache/cache.go b/execution/cache/cache.go index 5fa38a6f4b8..fd59c16f2d5 100644 --- a/execution/cache/cache.go +++ b/execution/cache/cache.go @@ -47,6 +47,9 @@ type Cache interface { // size) are evicted on the next read rather than walked eagerly. Unwind(unwindToTxNum uint64) + // Close drops the cache's slot in the shared memory envelope. Idempotent. + Close() + // Len returns the number of entries in the cache. Len() int } diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 93de162d616..06b436dedee 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -26,6 +26,7 @@ import ( lru "github.com/hashicorp/golang-lru/v2" "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/common/cachebudget" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/maphash" "github.com/erigontech/erigon/execution/cache/coherence" @@ -175,6 +176,11 @@ type CodeCache struct { addrCapacityB datasize.ByteSize // capacity in bytes codeCapacityB datasize.ByteSize // capacity in bytes + + // reservedBytes is what NewCodeCache took from the shared envelope; closed + // guards the single paired Release so a double Close can't over-return it. + reservedBytes int64 + closed atomic.Bool } // isStale reports whether an entry stamped (txNum, epoch) reflects dead-fork @@ -238,6 +244,11 @@ func NewCodeCache(codeCapacityBytes, addrCapacityBytes datasize.ByteSize) *CodeC if err != nil { panic(err) } + // Account the code-residency budget in the shared envelope so the code cache + // draws down the same pool as the state caches. Take is unconditional (the + // code slot array is modest and the cache must never be born unusable); + // returned by Close. + cachebudget.Global.Take(int64(codeCapacityBytes)) // Byte budget → entry-count cap for the two bytes layers; the size-only // layer is entry-counted directly. Floor at 1 so tiny (test) budgets still // construct a valid, evicting LRU. @@ -267,6 +278,7 @@ func NewCodeCache(codeCapacityBytes, addrCapacityBytes datasize.ByteSize) *CodeC codeSizeCapEntries: DefaultCodeSizeCacheEntries, addrCapacityB: addrCapacityBytes, codeCapacityB: codeCapacityBytes, + reservedBytes: int64(codeCapacityBytes), } // OnEvict fires on capacity-driven LRU eviction and on explicit Remove, so // the byte/entry counters follow residency without a separate scan. @@ -537,6 +549,13 @@ func (c *CodeCache) Clear() { c.coh.Init() } +// Close returns this cache's reservation to the shared envelope. Idempotent. +func (c *CodeCache) Close() { + if c.closed.CompareAndSwap(false, true) { + cachebudget.Global.Release(c.reservedBytes) + } +} + // Unwind invalidates entries reflecting dead-fork state. Code deployed on the // rolled-back fork must stop being discoverable — even by codeHash — because // although a hash → bytes value is invariant, the code's EXISTENCE is not. diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index d1f6e20b42f..fb2c890260d 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -25,6 +25,7 @@ import ( "github.com/elastic/go-freelru" "github.com/erigontech/erigon/common" + "github.com/erigontech/erigon/common/cachebudget" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/maphash" "github.com/erigontech/erigon/execution/cache/coherence" @@ -58,11 +59,34 @@ type entry[T any] struct { // GenericCache is a sharded, LRU-evicting bounded cache for key-value // data. Eviction mode is fixed at construction (see policy.go). type GenericCache[T any] struct { - data *freelru.ShardedLRU[uint64, entry[T]] - capacityB datasize.ByteSize - capacityEntries uint32 // freelru's slot cap; ModeNoOp refuses inserts at this count - currentSize atomic.Int64 - mode Mode + // data is the sharded LRU, replaced wholesale on a jump-grow. Load it once per + // operation; a write racing a resize may land in the LRU about to be replaced + // and be dropped — a benign miss (the value is re-read from the domain). + data atomic.Pointer[freelru.ShardedLRU[uint64, entry[T]]] + capacityB datasize.ByteSize + mode Mode + + // Jump-grow: the LRU starts at startCap slots and resizes ×genericCacheGrowFactor + // toward maxCap as it fills, reserving each step's bytes from the shared + // envelope; a step the envelope can't fund stops the growth (freelru then + // evicts within the current size). A cache with a small working set never + // grows past startCap, so it costs a few KB regardless of its configured + // budget. resizeMu guards curCap/reservedBytes and the resize itself. + startCap uint32 + maxCap uint32 + curCap uint32 + avgEntryBytes int64 // per-domain byte estimate; maps slot count ↔ envelope bytes + resizeMu sync.Mutex + reservedBytes int64 + + currentSize atomic.Int64 + + // enveloped is set only when the cache draws from the shared envelope (via + // NewGenericCache); closed guards the single paired Release, so neither a test + // cache built with an explicit fixed size nor a double Close mis-accounts the + // envelope. + enveloped bool + closed atomic.Bool // coh is the shared (epoch, floor) unwind-coherence primitive: an entry is // valid iff written in the current epoch OR its txNum is below the unwind @@ -85,51 +109,118 @@ type GenericCache[T any] struct { func u64identity(k uint64) uint32 { return uint32(k) } -// NewGenericCache creates a new GenericCache with the specified byte -// capacity. mode selects ModeEvictLRU (default in this tree) or ModeNoOp -// (kept for diagnostic baselines). +const ( + // genericCacheStartCapacity is the slot count a jump-grow cache is born with. + // A cache whose working set never exceeds it (a test fixture) stays this small + // regardless of its configured byte budget. + genericCacheStartCapacity = 1024 + genericCacheGrowFactor = 4 +) + +// NewGenericCache creates a jump-grow cache with the specified byte capacity as +// its growth ceiling, using the generic per-entry estimate. mode selects +// ModeEvictLRU (default in this tree) or ModeNoOp (diagnostic baseline). func NewGenericCache[T any](capacityBytes datasize.ByteSize, sizeFunc func(T) int, mode Mode) *GenericCache[T] { - capacityEntries := uint32(uint64(capacityBytes) / avgBytesPerEntry) - if capacityEntries < 1024 { - capacityEntries = 1024 + return NewGenericCacheWithAvg(capacityBytes, avgBytesPerEntry, sizeFunc, mode) +} + +// NewGenericCacheWithAvg is NewGenericCache with an explicit per-domain average +// entry size, so the byte-budget ceiling and the envelope accounting reflect the +// domain's real entry cost (accounts ≈ 96 B, storage ≈ 88 B) rather than the +// generic default. It starts small and jump-grows toward the ceiling on demand, +// funding each step from the shared envelope. +func NewGenericCacheWithAvg[T any](capacityBytes datasize.ByteSize, avgBytes uint32, sizeFunc func(T) int, mode Mode) *GenericCache[T] { + if avgBytes == 0 { + avgBytes = avgBytesPerEntry + } + maxCap := uint32(uint64(capacityBytes) / uint64(avgBytes)) + if maxCap < genericCacheStartCapacity { + maxCap = genericCacheStartCapacity } - // Absolute safety ceiling on the eagerly-allocated slot array; kept above the - // configured byte budgets' entry counts so it never caps residency below the - // budget (see newDomainCacheBytes). - if capacityEntries > 1<<24 { - capacityEntries = 1 << 24 + // Absolute safety ceiling on the slot array. + if maxCap > 1<<24 { + maxCap = 1 << 24 } - return newGenericCacheEntries(capacityBytes, capacityEntries, sizeFunc, mode) + start := uint32(genericCacheStartCapacity) + if start > maxCap { + start = maxCap + } + c := newGenericCacheEntries[T](capacityBytes, start, sizeFunc, mode) + c.maxCap = maxCap + c.avgEntryBytes = int64(avgBytes) + c.enveloped = true + // The initial slot array is small; take it unconditionally so no cache is + // born unable to hold anything. + c.reservedBytes = int64(start) * c.avgEntryBytes + cachebudget.Global.Take(c.reservedBytes) + return c } -// newGenericCacheEntries builds a cache against an explicit entry-count -// cap. Used by tests that want to exercise eviction with small capacities; -// production constructs via NewGenericCache. +// newGenericCacheEntries builds a cache against an explicit fixed entry-count +// cap (no jump-grow, no envelope). Used by tests that want to exercise eviction +// with small capacities; production constructs via NewGenericCache. func newGenericCacheEntries[T any](capacityBytes datasize.ByteSize, capacityEntries uint32, sizeFunc func(T) int, mode Mode) *GenericCache[T] { if capacityEntries == 0 { capacityEntries = 1 } - lru, err := freelru.NewSharded[uint64, entry[T]](capacityEntries, u64identity) - if err != nil { - panic(err) - } c := &GenericCache[T]{ - data: lru, - capacityB: capacityBytes, - capacityEntries: capacityEntries, - mode: mode, - sizeFunc: sizeFunc, + capacityB: capacityBytes, + startCap: capacityEntries, + curCap: capacityEntries, + maxCap: capacityEntries, + avgEntryBytes: avgBytesPerEntry, + mode: mode, + sizeFunc: sizeFunc, } // Before any unwind every entry predates the (nonexistent) floor, so all // reads are valid; the floor only drops once an unwind happens. c.coh.Init() - // OnEvict fires for capacity-driven LRU eviction (ModeEvictLRU) and - // for explicit Remove(). In both cases we want currentSize to follow. + c.data.Store(c.newShards(capacityEntries)) + return c +} + +// newShards builds a sharded LRU of the given capacity with this cache's evict +// callback wired, so currentSize follows capacity-driven eviction and Remove. +func (c *GenericCache[T]) newShards(capacity uint32) *freelru.ShardedLRU[uint64, entry[T]] { + lru, err := freelru.NewSharded[uint64, entry[T]](capacity, u64identity) + if err != nil { + panic(err) + } lru.SetOnEvict(func(_ uint64, e entry[T]) { c.currentSize.Add(-int64(e.size)) c.evictions.Add(1) }) - return c + return lru +} + +// maybeGrow jump-resizes the LRU one step larger when it is full, the ceiling +// hasn't been reached, and the shared envelope can fund the step. Otherwise the +// LRU keeps its size and freelru evicts within it. Called with no lock held. +func (c *GenericCache[T]) maybeGrow() { + c.resizeMu.Lock() + defer c.resizeMu.Unlock() + + old := c.data.Load() + if c.curCap >= c.maxCap || old.Len() < int(c.curCap) { + return + } + newCap := c.curCap * genericCacheGrowFactor + if newCap > c.maxCap { + newCap = c.maxCap + } + delta := int64(newCap-c.curCap) * c.avgEntryBytes + if !cachebudget.Global.Reserve(delta) { + return + } + next := c.newShards(newCap) + for _, k := range old.Keys() { + if v, ok := old.Get(k); ok { + next.Add(k, v) + } + } + c.data.Store(next) + c.curCap = newCap + c.reservedBytes += delta } // DomainCache wraps GenericCache[[]byte] to implement the Cache interface. @@ -174,7 +265,8 @@ func (c *GenericCache[T]) Get(key []byte) (T, bool) { // maxStep — the same coherence the BranchCache read applies for commitment. func (c *GenericCache[T]) GetWithTxNum(key []byte) (T, uint64, bool) { h := maphash.Hash(key) - e, ok := c.data.Get(h) + lru := c.data.Load() + e, ok := lru.Get(h) if !ok || !bytes.Equal(e.key, key) { c.misses.Add(1) var zero T @@ -189,7 +281,7 @@ func (c *GenericCache[T]) GetWithTxNum(key []byte) (T, uint64, bool) { // tx — and must be dropped; >= not > (the surviving block's last txNum is // floor-1, so this never drops a live entry). if c.coh.IsStale(e.txNum, e.epoch) { - c.data.Remove(h) + lru.Remove(h) c.staleEvicted.Add(1) c.misses.Add(1) var zero T @@ -223,7 +315,8 @@ func (c *GenericCache[T]) put(key []byte, value T, txNum uint64, overwrite bool) mu.Lock() defer mu.Unlock() - existing, hasExisting := c.data.Get(h) + lru := c.data.Load() + existing, hasExisting := lru.Get(h) // Existing key — update in place. Reuse the stored key buffer to // avoid an extra allocation; the freshly-decoded value replaces the @@ -232,7 +325,7 @@ func (c *GenericCache[T]) put(key []byte, value T, txNum uint64, overwrite bool) if !overwrite && !c.coh.IsStale(existing.txNum, existing.epoch) { return } - c.data.Add(h, entry[T]{key: existing.key, val: value, size: newSize, txNum: txNum, epoch: ep}) + lru.Add(h, entry[T]{key: existing.key, val: value, size: newSize, txNum: txNum, epoch: ep}) c.currentSize.Add(int64(newSize - existing.size)) return } @@ -240,11 +333,18 @@ func (c *GenericCache[T]) put(key []byte, value T, txNum uint64, overwrite bool) if c.mode == ModeNoOp { // Refuse once full by either bound — freelru would otherwise evict at the // entry-count cap, which ModeNoOp ("drop new keys when full") must not do. - if c.currentSize.Load()+int64(newSize) > int64(c.capacityB) || c.data.Len() >= int(c.capacityEntries) { + if c.currentSize.Load()+int64(newSize) > int64(c.capacityB) || lru.Len() >= int(c.maxCap) { c.dropped.Add(1) return } } + + // ModeEvictLRU: grow toward the ceiling before inserting into a full LRU, so a + // busy cache expands into its budget rather than evicting at the start size. + if c.mode != ModeNoOp && lru.Len() >= int(c.curCap) && c.curCap < c.maxCap { + c.maybeGrow() + lru = c.data.Load() + } // In ModeEvictLRU the byte budget is enforced through the entry-count cap, // not a separate currentSize check: capacityEntries is derived from // capacityB (capacityB/avgBytesPerEntry, see NewGenericCache / @@ -265,7 +365,7 @@ func (c *GenericCache[T]) put(key []byte, value T, txNum uint64, overwrite bool) c.currentSize.Add(-int64(existing.size)) } keyCopy := common.Copy(key) - c.data.Add(h, entry[T]{key: keyCopy, val: value, size: newSize, txNum: txNum, epoch: ep}) + lru.Add(h, entry[T]{key: keyCopy, val: value, size: newSize, txNum: txNum, epoch: ep}) c.currentSize.Add(int64(newSize)) c.inserts.Add(1) } @@ -273,8 +373,9 @@ func (c *GenericCache[T]) put(key []byte, value T, txNum uint64, overwrite bool) // Delete removes the data for the given key. func (c *GenericCache[T]) Delete(key []byte) { h := maphash.Hash(key) - if existing, ok := c.data.Get(h); ok && bytes.Equal(existing.key, key) { - c.data.Remove(h) + lru := c.data.Load() + if existing, ok := lru.Get(h); ok && bytes.Equal(existing.key, key) { + lru.Remove(h) } } @@ -284,9 +385,31 @@ func (c *GenericCache[T]) Delete(key []byte) { // serviceable. Mirrors CodeCache.Clear (which already did this — the two had // drifted). func (c *GenericCache[T]) Clear() { - c.data.Purge() c.currentSize.Store(0) c.coh.Init() + // Shrink back to the start size and return the grown budget to the envelope, + // keeping the cache adaptive across fork-validation/reset (it regrows on + // demand). A no-op Purge would leave the grown slot array resident. + c.resizeMu.Lock() + defer c.resizeMu.Unlock() + if c.enveloped { + cachebudget.Global.Release(c.reservedBytes - int64(c.startCap)*c.avgEntryBytes) + c.reservedBytes = int64(c.startCap) * c.avgEntryBytes + } + c.curCap = c.startCap + c.data.Store(c.newShards(c.startCap)) +} + +// Close returns this cache's envelope reservation so later caches can grow into +// the freed budget. Idempotent. +func (c *GenericCache[T]) Close() { + if c.enveloped && c.closed.CompareAndSwap(false, true) { + c.resizeMu.Lock() + reserved := c.reservedBytes + c.reservedBytes = 0 + c.resizeMu.Unlock() + cachebudget.Global.Release(reserved) + } } // Unwind invalidates entries that reflect dead-fork state. unwindToTxNum is the @@ -299,7 +422,7 @@ func (c *GenericCache[T]) Unwind(unwindToTxNum uint64) { // Len returns the number of entries in the cache. func (c *GenericCache[T]) Len() int { - return int(c.data.Len()) + return c.data.Load().Len() } // SizeBytes returns the current size of the cache in bytes. @@ -332,7 +455,7 @@ func (c *GenericCache[T]) PrintStatsAndReset(name string) { "hits", hits, "misses", misses, "hit_rate", hitRate, "inserts", inserts, "evictions", evictions, "dropped", dropped, "stale_evicted", staleEvicted, "epoch", c.coh.Epoch(), - "entries", c.data.Len(), "size_mb", sizeBytes/(1024*1024), + "entries", c.data.Load().Len(), "size_mb", sizeBytes/(1024*1024), "capacity_mb", int64(c.capacityB/datasize.MB), "usage_pct", usagePct, ) } diff --git a/execution/cache/state_cache.go b/execution/cache/state_cache.go index 929a9367e07..1d215a350e4 100644 --- a/execution/cache/state_cache.go +++ b/execution/cache/state_cache.go @@ -85,21 +85,13 @@ func stateCacheModeFromEnv() Mode { } } -// newDomainCacheBytes constructs a DomainCache where the entry-count cap -// is derived from the byte budget using the supplied per-domain avg. +// newDomainCacheBytes constructs a DomainCache whose growth ceiling is derived +// from the byte budget using the supplied per-domain avg. It jump-grows from a +// small start into the shared envelope on demand, so a domain with a small +// working set (a test fixture) never pre-commits the full budget. func newDomainCacheBytes(capacityBytes datasize.ByteSize, avgBytes uint32, mode Mode) *DomainCache { - capacityEntries := uint32(uint64(capacityBytes) / uint64(avgBytes)) - if capacityEntries < 1024 { - capacityEntries = 1024 - } - // Absolute safety ceiling on the eagerly-allocated slot array; must stay - // above the configured byte budgets' entry counts (Account 1 GB / ~96 B ≈ - // 11.2M) or it silently caps residency below the budget. - if capacityEntries > 1<<24 { - capacityEntries = 1 << 24 - } return &DomainCache{ - GenericCache: newGenericCacheEntries(capacityBytes, capacityEntries, func(v []byte) int { return len(v) }, mode), + GenericCache: NewGenericCacheWithAvg(capacityBytes, avgBytes, func(v []byte) int { return len(v) }, mode), } } @@ -277,6 +269,16 @@ func (c *StateCache) Clear() { } } +// Close releases every sub-cache's slot in the shared memory envelope so later +// caches size against real concurrency. Idempotent. +func (c *StateCache) Close() { + for _, cache := range c.caches { + if cache != nil { + cache.Close() + } + } +} + // Unwind invalidates, across all caches, entries reflecting state above // unwindToTxNum on a now-dead fork. Diffset-free and O(1): every cache (the // GenericCaches and the CodeCache, all layers) bumps an epoch + lowers a floor diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 36c9369b2bd..27a237c3100 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -92,6 +92,12 @@ type BranchCache struct { tailCap uint32 tailMu sync.Mutex + // maxDepth is the resident trunk depth for this cache (both the account trunk + // and every pinned storage trunk), chosen from the active-instance count at + // construction. closed guards the single paired active-count decrement. + maxDepth uint8 + closed atomic.Bool + // trunkDisabled (env BRANCH_CACHE_TRUNK_DISABLE) routes depth-1-4 account // branches back to the LRU tail instead of the resident account trunk — a // runtime A/B switch to isolate whether the resident trunk is the source @@ -174,6 +180,13 @@ type trunk struct { d3 atomic.Pointer[[4096]atomic.Pointer[branchCacheEntry]] d4 atomic.Pointer[[65536]atomic.Pointer[branchCacheEntry]] deep *maphash.Map[*branchCacheEntry] + + // maxDepth caps which dense tiers this trunk will allocate: branches deeper + // than it route to deep (storage) or the LRU tail (account) instead. Set from + // the active-BranchCache count so a process with a few caches (production) + // keeps the full depth-4 residency while one with many (the test suite) keeps + // each trunk shallow. d0/d1 are always resident (tiny, always reached). + maxDepth uint8 } func (t *trunk) d2For(forWrite bool) *[256]atomic.Pointer[branchCacheEntry] { @@ -194,7 +207,7 @@ func (t *trunk) d3For(forWrite bool) *[4096]atomic.Pointer[branchCacheEntry] { if p := t.d3.Load(); p != nil { return p } - if !forWrite { + if !forWrite || t.maxDepth < 3 { return nil } p := &[4096]atomic.Pointer[branchCacheEntry]{} @@ -208,7 +221,7 @@ func (t *trunk) d4For(forWrite bool) *[65536]atomic.Pointer[branchCacheEntry] { if p := t.d4.Load(); p != nil { return p } - if !forWrite { + if !forWrite || t.maxDepth < 4 { return nil } p := &[65536]atomic.Pointer[branchCacheEntry]{} @@ -218,16 +231,36 @@ func (t *trunk) d4For(forWrite bool) *[65536]atomic.Pointer[branchCacheEntry] { return p } -// newAccountTrunk builds the global account trunk: dense depth-4 fixed array, -// no deep overflow (account depth 5+ uses the LRU tail). -func newAccountTrunk() *trunk { - return &trunk{} +// newAccountTrunk builds the global account trunk: dense fixed arrays up to +// maxDepth, no deep overflow (account depth past the resident tiers uses the +// LRU tail). +func newAccountTrunk(maxDepth uint8) *trunk { + return &trunk{maxDepth: maxDepth} } // newStorageTrunk builds a per-contract storage trunk: deep overflow for -// storage depth 4+, no depth-4 fixed array. -func newStorageTrunk() *trunk { - return &trunk{deep: maphash.NewMap[*branchCacheEntry]()} +// storage depth past the resident tiers, no depth-4 fixed array. +func newStorageTrunk(maxDepth uint8) *trunk { + return &trunk{maxDepth: maxDepth, deep: maphash.NewMap[*branchCacheEntry]()} +} + +// Adaptive trunk depth: a process with a handful of BranchCaches (production) +// keeps full depth-4 residency; one that spins up many (the test suite) keeps +// each trunk shallow so their fixed-array tiers don't sum past the memory +// envelope. Depth is chosen once per cache from the live instance count. +const ( + trunkDepthFull = 4 + trunkDepthShallow = 2 + trunkInstanceDepthThreshold = 10 +) + +var activeBranchCaches atomic.Int64 + +func adaptiveTrunkDepth(active int64) uint8 { + if active <= trunkInstanceDepthThreshold { + return trunkDepthFull + } + return trunkDepthShallow } // slot returns the fixed-array slot for a nibble path of length 0-3 (and length @@ -283,18 +316,28 @@ func NewBranchCache(tailCapacity int) *BranchCache { if tailCapacity <= 0 { panic(fmt.Sprintf("BranchCache: tailCapacity must be positive, got %d", tailCapacity)) } + maxDepth := adaptiveTrunkDepth(activeBranchCaches.Add(1)) bc := &BranchCache{ tailCap: uint32(tailCapacity), - accountTrunk: newAccountTrunk(), + maxDepth: maxDepth, + accountTrunk: newAccountTrunk(maxDepth), trunkDisabled: os.Getenv("BRANCH_CACHE_TRUNK_DISABLE") != "", } // Before any unwind every entry's txN is at/below the floor, so the epoch // check never strands a valid entry. bc.coh.Init() - log.Info("[branch-cache] init", "trunkEnabled", !bc.trunkDisabled, "tailCap", tailCapacity) + log.Info("[branch-cache] init", "trunkEnabled", !bc.trunkDisabled, "tailCap", tailCapacity, "trunkDepth", maxDepth) return bc } +// Close drops this cache from the active-instance count so later BranchCaches +// size their trunk depth against real concurrency. Idempotent. +func (c *BranchCache) Close() { + if c.closed.CompareAndSwap(false, true) { + activeBranchCaches.Add(-1) + } +} + // tailForWrite returns the LRU tail, allocating it on first use so a cache whose // tries never spill past the resident trunk pays nothing for it. func (c *BranchCache) tailForWrite() *tailLRU { @@ -384,7 +427,7 @@ func (c *BranchCache) storageRoute(prefix []byte, create bool) (st *trunk, acct if !create { return nil, packed, stor, false } - st = newStorageTrunk() + st = newStorageTrunk(c.maxDepth) c.pinnedForWrite().Set(packed, st) return st, packed, stor, true } diff --git a/execution/commitment/branch_cache_tail.go b/execution/commitment/branch_cache_tail.go index 69ce1d04cbf..4bb64199c26 100644 --- a/execution/commitment/branch_cache_tail.go +++ b/execution/commitment/branch_cache_tail.go @@ -23,7 +23,7 @@ import ( "github.com/elastic/go-freelru" - "github.com/erigontech/erigon/common/estimate" + "github.com/erigontech/erigon/common/cachebudget" ) const ( @@ -37,45 +37,9 @@ const ( tailEntryBytes = 512 ) -// tailBudget caps the total resident memory of every BranchCache LRU tail in the -// process to a fraction of the memory actually available (system RAM, cgroup -// limit, or GOMEMLIMIT — whichever is lowest). A single production cache draws -// the whole budget and grows to its full cap; a process that spins up thousands -// of ephemeral caches (the execution-test suite) has them share the budget, so -// each stays small — with no test-specific code. -type tailBudget struct { - limit int64 - used atomic.Int64 -} - -var globalTailBudget = &tailBudget{limit: int64(estimate.TotalMemory() / tailBudgetDivisor)} - -// tailBudgetDivisor keeps the whole-process tail budget to a small fraction of -// available memory; the tail is a cache, so undersizing only costs hit rate. -const tailBudgetDivisor = 32 - -// reserve grabs n bytes if the budget has room, returning false when full. -func (b *tailBudget) reserve(n int64) bool { - for { - used := b.used.Load() - if used+n > b.limit { - return false - } - if b.used.CompareAndSwap(used, used+n) { - return true - } - } -} - -func (b *tailBudget) release(n int64) { - if n > 0 { - b.used.Add(-n) - } -} - // tailLRU is the BranchCache LRU tail. It wraps a sharded freelru that is // jump-resized (allocate larger, copy the live entries over) as it fills, -// bounded by the shared tailBudget and a per-cache max. Reads and writes take no +// bounded by the shared cachebudget envelope and a per-cache max. Reads and writes take no // tail-level lock — they load the current freelru atomically and rely on its own // per-shard locking; the resize mutex is held only during the rare grow. A write // racing a resize may land in the freelru about to be replaced and be dropped, @@ -96,7 +60,7 @@ func newTailLRU(maxCapacity uint32) *tailLRU { } t := &tailLRU{maxCap: maxCapacity} t.reserved = int64(start) * tailEntryBytes - globalTailBudget.reserve(t.reserved) // initial slice is small; take it unconditionally + cachebudget.Global.Take(t.reserved) // initial slice is small; take it unconditionally t.curCap = start t.cur.Store(newTailShards(start)) return t @@ -140,7 +104,7 @@ func (t *tailLRU) maybeGrow() { newCap = t.maxCap } delta := int64(newCap-t.curCap) * tailEntryBytes - if !globalTailBudget.reserve(delta) { + if !cachebudget.Global.Reserve(delta) { return } next := newTailShards(newCap) @@ -167,7 +131,7 @@ func (t *tailLRU) reset() { if start > t.maxCap { start = t.maxCap } - globalTailBudget.release(t.reserved - int64(start)*tailEntryBytes) + cachebudget.Global.Release(t.reserved - int64(start)*tailEntryBytes) t.reserved = int64(start) * tailEntryBytes t.curCap = start t.cur.Store(newTailShards(start)) diff --git a/execution/execmodule/execmoduletester/exec_module_tester.go b/execution/execmodule/execmoduletester/exec_module_tester.go index 4f643c47194..2fa9ce82343 100644 --- a/execution/execmodule/execmoduletester/exec_module_tester.go +++ b/execution/execmodule/execmoduletester/exec_module_tester.go @@ -123,6 +123,7 @@ type ExecModuleTester struct { ForkValidator *execmodule.ForkValidator ExecModule *execmodule.ExecModule StateCache *execmodule.Cache + domainCache *cache.StateCache retirementStart chan bool retirementDone chan struct{} retirementWg sync.WaitGroup @@ -157,6 +158,9 @@ func (emt *ExecModuleTester) Close() { if emt.DB != nil { emt.DB.Close() } + if emt.domainCache != nil { + emt.domainCache.Close() + } if emt.tb == nil && emt.Dirs.DataDir != "" { dir.RemoveAll(emt.Dirs.DataDir) } @@ -717,6 +721,12 @@ func New(tb testing.TB, opts ...Option) *ExecModuleTester { Accumulator: mock.Notifications.Accumulator, RecentReceipts: mock.Notifications.RecentReceipts, } + // Per-instance domain cache, held on the tester so Close releases its + // envelope reservation. The production default is safe here: the caches + // jump-grow from a small start into the shared envelope on demand, so a + // fixture with a small working set stays small regardless of the configured + // budget — no hand-tuned test size, same code path as production. + mock.domainCache = cache.NewDefaultStateCache() mock.ExecModule = execmodule.NewExecModule( ctx, mock.BlockReader, @@ -728,9 +738,7 @@ func New(tb testing.TB, opts ...Option) *ExecModuleTester { hook, accum, mock.StateCache, - // Small per-instance domain cache: the harness builds one ExecModule per - // fixture, so production-size caches would allocate hundreds of MB each. - cache.NewStateCache(1*datasize.MB, 1*datasize.MB, 1*datasize.MB, 1*datasize.MB), + mock.domainCache, logger, engine, cfg.Sync, From 2ca5333e23b71ab55d4bddb8da2c46e07dc62da9 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 3 Jul 2026 19:07:13 +0000 Subject: [PATCH 11/18] execution/cache: jump-grow the CodeCache content layers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CodeCache pre-allocated its content-addressed layers to full capacity at construction — most notably the fixed 1M-entry codeSizeByCodeHash (~52MB) plus the code LRUs — so every per-fixture CodeCache in the execution-test suite committed tens of MB up front. Held for each fixture's lifetime, their concurrent sum (amplified by the race detector) was the remaining driver of the execution-tests -race OOM after the state caches were already bounded. Add a generic growLRU[V] (start small, jump-resize ×4 toward a byte-budget ceiling, funding each step from the shared cachebudget envelope, shrink on Purge, release on Close) and route hashToCode, codeHashToCode, and codeSizeByCodeHash through it. A cache over few contracts stays KB-sized; a busy one grows into its budget. Per-instance and content-addressed as before — the addr layers and the keyHash-collision gate are unchanged, so correctness holds. Under a runner-like cap (14GB, 4 vCPU, no swap, -race) the recursive execution/tests now completes where it previously OOM-killed. --- execution/cache/code_cache.go | 75 +++++++---------- execution/cache/grow_lru.go | 148 ++++++++++++++++++++++++++++++++++ 2 files changed, 175 insertions(+), 48 deletions(-) create mode 100644 execution/cache/grow_lru.go diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 06b436dedee..586065df899 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -22,11 +22,9 @@ import ( "unsafe" "github.com/c2h5oh/datasize" - "github.com/elastic/go-freelru" lru "github.com/hashicorp/golang-lru/v2" "github.com/erigontech/erigon/common" - "github.com/erigontech/erigon/common/cachebudget" "github.com/erigontech/erigon/common/log/v3" "github.com/erigontech/erigon/common/maphash" "github.com/erigontech/erigon/execution/cache/coherence" @@ -53,6 +51,10 @@ const ( // the persistent (MDBX-backed) cold tier backstops entries evicted from this // hot tier, so a loose byte bound here is acceptable. avgCodeEntryBytes = 4096 + // codeSizeEntryBytes is the resident cost of one size-layer slot (freelru + // element holding size/keyHash/txNum/epoch), used to map the size-layer entry + // ceiling to an envelope byte budget. + codeSizeEntryBytes = 64 ) // CodeCache is a multi-level concurrent cache for contract code, keyed by the @@ -122,8 +124,8 @@ type CodeCache struct { // codeID for the code at that address. An LRU so fresh-address workloads // evict oldest entries and warm up the working set. addrToHash *lru.Cache[common.Address, versionedAddressID] - hashToCode *freelru.ShardedLRU[uint64, codeEntry] // codeID(maphash(code)) → code, LRU-evicting - codeSize atomic.Int64 // resident bytes (stat; hard bound is the entry cap) + hashToCode *growLRU[codeEntry] // codeID(maphash(code)) → code, jump-grow + LRU-evicting + codeSize atomic.Int64 // resident bytes (stat; hard bound is the entry cap) // addrToCodeHash maps a 20-byte address to its 32-byte Ethereum codeHash // (keccak), separately from addrToHash (which uses the cheap maphash @@ -137,14 +139,14 @@ type CodeCache struct { // of L1 — Get-by-codeHash bypasses addr lookup entirely. Memory cost: // duplicates code bytes vs L2 (worst case 2x byte storage); accepted // for the per-key fast-path on many-addrs-one-code workloads. - codeHashToCode *freelru.ShardedLRU[uint64, codeEntry] // keccak(code) → code, LRU-evicting - codeHashCodeSize atomic.Int64 // resident bytes (stat; hard bound is the entry cap) + codeHashToCode *growLRU[codeEntry] // keccak(code) → code, jump-grow + LRU-evicting + codeHashCodeSize atomic.Int64 // resident bytes (stat; hard bound is the entry cap) // Size-only layer: ethCodeHash → int (length in bytes). Answers // EXTCODESIZE / EXTCODEHASH without loading the bytes. Tiny per-entry // footprint (32B key + 8B value) so the same memory budget gives ~1000x // the hit surface vs the bytes cache. - codeSizeByCodeHash *freelru.ShardedLRU[uint64, codeSizeEntry] + codeSizeByCodeHash *growLRU[codeSizeEntry] codeSizeEntries atomic.Int64 codeSizeCapEntries int64 @@ -177,10 +179,9 @@ type CodeCache struct { addrCapacityB datasize.ByteSize // capacity in bytes codeCapacityB datasize.ByteSize // capacity in bytes - // reservedBytes is what NewCodeCache took from the shared envelope; closed - // guards the single paired Release so a double Close can't over-return it. - reservedBytes int64 - closed atomic.Bool + // closed guards the single paired Close of the content layers so a double + // Close can't over-return their envelope reservations. + closed atomic.Bool } // isStale reports whether an entry stamped (txNum, epoch) reflects dead-fork @@ -200,7 +201,7 @@ func (c *CodeCache) isStale(txNum uint64, epoch uint32) bool { // bytes as a stat; the hard bound is the LRU's entry cap. stamp/valCost are // non-capturing so passing them allocates nothing on the put path. func putContent[T any]( - lru *freelru.ShardedLRU[uint64, T], + lru *growLRU[T], h uint64, newEntry T, stamp func(T) (uint64, uint32), @@ -244,47 +245,23 @@ func NewCodeCache(codeCapacityBytes, addrCapacityBytes datasize.ByteSize) *CodeC if err != nil { panic(err) } - // Account the code-residency budget in the shared envelope so the code cache - // draws down the same pool as the state caches. Take is unconditional (the - // code slot array is modest and the cache must never be born unusable); - // returned by Close. - cachebudget.Global.Take(int64(codeCapacityBytes)) - // Byte budget → entry-count cap for the two bytes layers; the size-only - // layer is entry-counted directly. Floor at 1 so tiny (test) budgets still - // construct a valid, evicting LRU. - codeEntries := uint32(uint64(codeCapacityBytes) / avgCodeEntryBytes) - if codeEntries < 1 { - codeEntries = 1 - } - hashToCode, err := freelru.NewSharded[uint64, codeEntry](codeEntries, u64identity) - if err != nil { - panic(err) - } - codeHashToCode, err := freelru.NewSharded[uint64, codeEntry](codeEntries, u64identity) - if err != nil { - panic(err) - } - sizeEntries := uint32(DefaultCodeSizeCacheEntries) - codeSizeByCodeHash, err := freelru.NewSharded[uint64, codeSizeEntry](sizeEntries, u64identity) - if err != nil { - panic(err) - } cc := &CodeCache{ addrToHash: addrLRU, addrToCodeHash: addrCodeHashLRU, - hashToCode: hashToCode, - codeHashToCode: codeHashToCode, - codeSizeByCodeHash: codeSizeByCodeHash, codeSizeCapEntries: DefaultCodeSizeCacheEntries, addrCapacityB: addrCapacityBytes, codeCapacityB: codeCapacityBytes, - reservedBytes: int64(codeCapacityBytes), } - // OnEvict fires on capacity-driven LRU eviction and on explicit Remove, so - // the byte/entry counters follow residency without a separate scan. - hashToCode.SetOnEvict(func(_ uint64, e codeEntry) { cc.codeSize.Add(-(8 + int64(len(e.code)))) }) - codeHashToCode.SetOnEvict(func(_ uint64, e codeEntry) { cc.codeHashCodeSize.Add(-(32 + int64(len(e.code)))) }) - codeSizeByCodeHash.SetOnEvict(func(_ uint64, _ codeSizeEntry) { cc.codeSizeEntries.Add(-1) }) + // The content-addressed layers jump-grow from a small start into the shared + // envelope, so a cache over few contracts (a test fixture) never pre-commits + // the full budget. OnEvict keeps the byte/entry counters following residency. + cc.hashToCode = newGrowLRU[codeEntry](codeCapacityBytes, avgCodeEntryBytes, + func(_ uint64, e codeEntry) { cc.codeSize.Add(-(8 + int64(len(e.code)))) }) + cc.codeHashToCode = newGrowLRU[codeEntry](codeCapacityBytes, avgCodeEntryBytes, + func(_ uint64, e codeEntry) { cc.codeHashCodeSize.Add(-(32 + int64(len(e.code)))) }) + cc.codeSizeByCodeHash = newGrowLRU[codeSizeEntry]( + datasize.ByteSize(DefaultCodeSizeCacheEntries*codeSizeEntryBytes), codeSizeEntryBytes, + func(_ uint64, _ codeSizeEntry) { cc.codeSizeEntries.Add(-1) }) // Before any unwind every entry's txNum is below the floor, so the epoch // check never strands a valid entry. cc.coh.Init() @@ -549,10 +526,12 @@ func (c *CodeCache) Clear() { c.coh.Init() } -// Close returns this cache's reservation to the shared envelope. Idempotent. +// Close returns the content layers' envelope reservations. Idempotent. func (c *CodeCache) Close() { if c.closed.CompareAndSwap(false, true) { - cachebudget.Global.Release(c.reservedBytes) + c.hashToCode.Close() + c.codeHashToCode.Close() + c.codeSizeByCodeHash.Close() } } diff --git a/execution/cache/grow_lru.go b/execution/cache/grow_lru.go new file mode 100644 index 00000000000..84b42583238 --- /dev/null +++ b/execution/cache/grow_lru.go @@ -0,0 +1,148 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package cache + +import ( + "fmt" + "sync" + "sync/atomic" + + "github.com/c2h5oh/datasize" + "github.com/elastic/go-freelru" + + "github.com/erigontech/erigon/common/cachebudget" +) + +// growLRU is a uint64-keyed sharded LRU that starts small and jump-resizes ×4 +// toward a byte-budget ceiling as it fills, funding each step from the shared +// cachebudget envelope. It exists so a cache with a small working set never +// pre-commits its full configured capacity — the same demand-growth the state +// caches use — reused across the CodeCache's content and size layers. +// +// A write racing a resize may land in the LRU about to be replaced and be +// dropped; that is a benign cache miss (the value is re-read from the DB). +type growLRU[V any] struct { + cur atomic.Pointer[freelru.ShardedLRU[uint64, V]] + onEvict func(uint64, V) + avgBytes int64 + + startCap uint32 + maxCap uint32 + + resizeMu sync.Mutex + curCap uint32 + reserved int64 + closed bool +} + +func newGrowLRU[V any](maxBytes datasize.ByteSize, avgBytes uint32, onEvict func(uint64, V)) *growLRU[V] { + if avgBytes == 0 { + avgBytes = avgBytesPerEntry + } + maxCap := uint32(uint64(maxBytes) / uint64(avgBytes)) + if maxCap < 1 { + maxCap = 1 + } + if maxCap > 1<<24 { + maxCap = 1 << 24 + } + // Start small (bounded by the ceiling); the floor is on the start size, not + // the ceiling — a tiny configured budget yields a tiny, still-evicting cap. + start := uint32(genericCacheStartCapacity) + if start > maxCap { + start = maxCap + } + g := &growLRU[V]{onEvict: onEvict, avgBytes: int64(avgBytes), startCap: start, maxCap: maxCap, curCap: start} + g.reserved = int64(start) * g.avgBytes + cachebudget.Global.Take(g.reserved) + g.cur.Store(g.newShards(start)) + return g +} + +func (g *growLRU[V]) newShards(capacity uint32) *freelru.ShardedLRU[uint64, V] { + lru, err := freelru.NewSharded[uint64, V](capacity, u64identity) + if err != nil { + panic(fmt.Sprintf("growLRU: NewSharded(%d): %s", capacity, err)) + } + if g.onEvict != nil { + lru.SetOnEvict(g.onEvict) + } + return lru +} + +func (g *growLRU[V]) Get(key uint64) (V, bool) { return g.cur.Load().Get(key) } + +func (g *growLRU[V]) Add(key uint64, value V) { + lru := g.cur.Load() + if lru.Len() >= int(g.curCap) && g.curCap < g.maxCap { + g.maybeGrow() + lru = g.cur.Load() + } + lru.Add(key, value) +} + +func (g *growLRU[V]) maybeGrow() { + g.resizeMu.Lock() + defer g.resizeMu.Unlock() + old := g.cur.Load() + if g.curCap >= g.maxCap || old.Len() < int(g.curCap) { + return + } + newCap := g.curCap * genericCacheGrowFactor + if newCap > g.maxCap { + newCap = g.maxCap + } + delta := int64(newCap-g.curCap) * g.avgBytes + if !cachebudget.Global.Reserve(delta) { + return + } + next := g.newShards(newCap) + for _, k := range old.Keys() { + if v, ok := old.Get(k); ok { + next.Add(k, v) + } + } + g.cur.Store(next) + g.curCap = newCap + g.reserved += delta +} + +func (g *growLRU[V]) Remove(key uint64) { g.cur.Load().Remove(key) } +func (g *growLRU[V]) Len() int { return g.cur.Load().Len() } + +// Purge empties the LRU and shrinks it back to the start size, returning the +// grown budget to the envelope (it regrows on demand). +func (g *growLRU[V]) Purge() { + g.resizeMu.Lock() + defer g.resizeMu.Unlock() + cachebudget.Global.Release(g.reserved - int64(g.startCap)*g.avgBytes) + g.reserved = int64(g.startCap) * g.avgBytes + g.curCap = g.startCap + g.cur.Store(g.newShards(g.startCap)) +} + +// Close returns this LRU's envelope reservation. Idempotent. +func (g *growLRU[V]) Close() { + g.resizeMu.Lock() + defer g.resizeMu.Unlock() + if g.closed { + return + } + g.closed = true + cachebudget.Global.Release(g.reserved) + g.reserved = 0 +} From fe862593df3741e132603d811eaffd9f365e2a75 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 6 Jul 2026 11:53:48 +0000 Subject: [PATCH 12/18] execution/commitment: trim BranchCache doc to comment policy --- execution/commitment/branch_cache.go | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 27a237c3100..755cb0bb43e 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -46,17 +46,10 @@ func isCommitmentStateKey(prefix []byte) bool { } // BranchCache stores commitment-trie branch data: a bounded LRU tail plus a -// single never-evicted slot for the root branch (a length-0 / no-key prefix). -// Aggregator-scope (one instance per Domain), pulled via BranchCacheProvider and -// plumbed to the trie through InitializeTrieAndUpdates. It is a passive store — -// the trie walker/encoder drive all reads and writes; the cache never fetches -// state itself. -// -// Concurrency: the LRU tail and the atomic-pointer root slot make any mix of -// concurrent Get/Put/Invalidate mechanically safe, but the cache does not -// coordinate writers — callers must ensure a single writer per prefix -// (last-Put-wins otherwise); add any such coordination at the orchestrator, not -// by locking the cache. +// never-evicted root slot, aggregator-scope and passive (the trie drives all +// reads/writes). Concurrent Get/Put/Invalidate are mechanically safe, but the +// cache does not coordinate writers — callers must ensure a single writer per +// prefix, coordinated at the orchestrator, not by locking the cache. type BranchCache struct { // Root tier — single slot for the root branch (always hottest, always // present). Atomic-pointer access so no lock is needed for the hot From 8d6a1d7c773704a401ba535bdcaefdf5555e4911 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 6 Jul 2026 12:25:27 +0000 Subject: [PATCH 13/18] execution/cache: make jump-grow curCap atomic to fix data race MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The jump-grow caches read curCap on the lock-free put fast-path (GenericCache under a per-key stripe lock, growLRU under no lock) but wrote it under resizeMu in maybeGrow, so a writer crossing the grow threshold raced concurrent puts on the same shared cache — caught by the -race eest shard via StateCache.Put during parallel execution. Make curCap an atomic.Uint32 in both GenericCache and growLRU (data is already atomic.Pointer). Adds a concurrent-put-across-grow race regression test. --- execution/cache/generic_cache.go | 20 +++---- .../cache/generic_cache_concurrency_test.go | 53 +++++++++++++++++++ execution/cache/grow_lru.go | 18 ++++--- 3 files changed, 74 insertions(+), 17 deletions(-) create mode 100644 execution/cache/generic_cache_concurrency_test.go diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index fb2c890260d..f11d3103994 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -71,10 +71,11 @@ type GenericCache[T any] struct { // envelope; a step the envelope can't fund stops the growth (freelru then // evicts within the current size). A cache with a small working set never // grows past startCap, so it costs a few KB regardless of its configured - // budget. resizeMu guards curCap/reservedBytes and the resize itself. + // budget. resizeMu serialises the resize and guards reservedBytes; curCap is + // atomic because the put fast-path reads it outside resizeMu. startCap uint32 maxCap uint32 - curCap uint32 + curCap atomic.Uint32 avgEntryBytes int64 // per-domain byte estimate; maps slot count ↔ envelope bytes resizeMu sync.Mutex reservedBytes int64 @@ -166,12 +167,12 @@ func newGenericCacheEntries[T any](capacityBytes datasize.ByteSize, capacityEntr c := &GenericCache[T]{ capacityB: capacityBytes, startCap: capacityEntries, - curCap: capacityEntries, maxCap: capacityEntries, avgEntryBytes: avgBytesPerEntry, mode: mode, sizeFunc: sizeFunc, } + c.curCap.Store(capacityEntries) // Before any unwind every entry predates the (nonexistent) floor, so all // reads are valid; the floor only drops once an unwind happens. c.coh.Init() @@ -201,14 +202,15 @@ func (c *GenericCache[T]) maybeGrow() { defer c.resizeMu.Unlock() old := c.data.Load() - if c.curCap >= c.maxCap || old.Len() < int(c.curCap) { + curCap := c.curCap.Load() + if curCap >= c.maxCap || old.Len() < int(curCap) { return } - newCap := c.curCap * genericCacheGrowFactor + newCap := curCap * genericCacheGrowFactor if newCap > c.maxCap { newCap = c.maxCap } - delta := int64(newCap-c.curCap) * c.avgEntryBytes + delta := int64(newCap-curCap) * c.avgEntryBytes if !cachebudget.Global.Reserve(delta) { return } @@ -219,7 +221,7 @@ func (c *GenericCache[T]) maybeGrow() { } } c.data.Store(next) - c.curCap = newCap + c.curCap.Store(newCap) c.reservedBytes += delta } @@ -341,7 +343,7 @@ func (c *GenericCache[T]) put(key []byte, value T, txNum uint64, overwrite bool) // ModeEvictLRU: grow toward the ceiling before inserting into a full LRU, so a // busy cache expands into its budget rather than evicting at the start size. - if c.mode != ModeNoOp && lru.Len() >= int(c.curCap) && c.curCap < c.maxCap { + if curCap := c.curCap.Load(); c.mode != ModeNoOp && lru.Len() >= int(curCap) && curCap < c.maxCap { c.maybeGrow() lru = c.data.Load() } @@ -396,7 +398,7 @@ func (c *GenericCache[T]) Clear() { cachebudget.Global.Release(c.reservedBytes - int64(c.startCap)*c.avgEntryBytes) c.reservedBytes = int64(c.startCap) * c.avgEntryBytes } - c.curCap = c.startCap + c.curCap.Store(c.startCap) c.data.Store(c.newShards(c.startCap)) } diff --git a/execution/cache/generic_cache_concurrency_test.go b/execution/cache/generic_cache_concurrency_test.go new file mode 100644 index 00000000000..6e27e997385 --- /dev/null +++ b/execution/cache/generic_cache_concurrency_test.go @@ -0,0 +1,53 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + +package cache + +import ( + "encoding/binary" + "sync" + "testing" + + "github.com/c2h5oh/datasize" +) + +// TestGenericCache_ConcurrentPutAcrossGrow guards the jump-grow data race: +// curCap is written under resizeMu in maybeGrow but read on the put fast-path +// outside it, so concurrent writers crossing the grow threshold raced on it +// (surfaced by the -race eest shard). Many goroutines insert enough distinct +// keys to trigger several grow steps while others put concurrently; run with +// -race, this must stay clean. +func TestGenericCache_ConcurrentPutAcrossGrow(t *testing.T) { + // Budget well above the start size (1024 slots) so maybeGrow fires repeatedly. + c := NewGenericCache[[]byte](64*datasize.MB, func(v []byte) int { return len(v) }, ModeEvictLRU) + + const workers = 8 + const perWorker = 20_000 + var wg sync.WaitGroup + for w := 0; w < workers; w++ { + wg.Add(1) + go func(base int) { + defer wg.Done() + key := make([]byte, 8) + for i := 0; i < perWorker; i++ { + binary.BigEndian.PutUint64(key, uint64(base*perWorker+i)) + c.Put(key, []byte{byte(i)}, uint64(i)) + c.Get(key) + } + }(w) + } + wg.Wait() +} diff --git a/execution/cache/grow_lru.go b/execution/cache/grow_lru.go index 84b42583238..4df26b3c1d3 100644 --- a/execution/cache/grow_lru.go +++ b/execution/cache/grow_lru.go @@ -44,7 +44,7 @@ type growLRU[V any] struct { maxCap uint32 resizeMu sync.Mutex - curCap uint32 + curCap atomic.Uint32 reserved int64 closed bool } @@ -66,7 +66,8 @@ func newGrowLRU[V any](maxBytes datasize.ByteSize, avgBytes uint32, onEvict func if start > maxCap { start = maxCap } - g := &growLRU[V]{onEvict: onEvict, avgBytes: int64(avgBytes), startCap: start, maxCap: maxCap, curCap: start} + g := &growLRU[V]{onEvict: onEvict, avgBytes: int64(avgBytes), startCap: start, maxCap: maxCap} + g.curCap.Store(start) g.reserved = int64(start) * g.avgBytes cachebudget.Global.Take(g.reserved) g.cur.Store(g.newShards(start)) @@ -88,7 +89,7 @@ func (g *growLRU[V]) Get(key uint64) (V, bool) { return g.cur.Load().Get(key) } func (g *growLRU[V]) Add(key uint64, value V) { lru := g.cur.Load() - if lru.Len() >= int(g.curCap) && g.curCap < g.maxCap { + if curCap := g.curCap.Load(); lru.Len() >= int(curCap) && curCap < g.maxCap { g.maybeGrow() lru = g.cur.Load() } @@ -99,14 +100,15 @@ func (g *growLRU[V]) maybeGrow() { g.resizeMu.Lock() defer g.resizeMu.Unlock() old := g.cur.Load() - if g.curCap >= g.maxCap || old.Len() < int(g.curCap) { + curCap := g.curCap.Load() + if curCap >= g.maxCap || old.Len() < int(curCap) { return } - newCap := g.curCap * genericCacheGrowFactor + newCap := curCap * genericCacheGrowFactor if newCap > g.maxCap { newCap = g.maxCap } - delta := int64(newCap-g.curCap) * g.avgBytes + delta := int64(newCap-curCap) * g.avgBytes if !cachebudget.Global.Reserve(delta) { return } @@ -117,7 +119,7 @@ func (g *growLRU[V]) maybeGrow() { } } g.cur.Store(next) - g.curCap = newCap + g.curCap.Store(newCap) g.reserved += delta } @@ -131,7 +133,7 @@ func (g *growLRU[V]) Purge() { defer g.resizeMu.Unlock() cachebudget.Global.Release(g.reserved - int64(g.startCap)*g.avgBytes) g.reserved = int64(g.startCap) * g.avgBytes - g.curCap = g.startCap + g.curCap.Store(g.startCap) g.cur.Store(g.newShards(g.startCap)) } From 6266facc7519029ea969246e352a3778d2a749c1 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 7 Jul 2026 10:54:21 +0000 Subject: [PATCH 14/18] execution/cache, execution/commitment: fix tailLRU curCap race + CodeStore mmap-retain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review blockers: - tailLRU.curCap was a plain uint32 read in Add while maybeGrow/reset wrote it under resizeMu — the same race GenericCache already fixes with an atomic, not applied to the tail copy. Make curCap an atomic.Uint32; add a concurrent tail-grow regression test (run under -race). Also reorder the grow check to `curCap < maxCap && Len() >= curCap` in all three grow-LRUs so a fully-grown cache never pays Len()'s per-shard locks on every insert. - CodeStore.GetByHash cached and returned the mmap-backed tx.GetOne result into the process-lifetime otter tier without copying, violating the kv contract (bytes must not outlive the tx) — a recycled MDBX page could silently mutate cached bytecode and diverge the state root. Copy before caching/return. Also: Evict positioned its cursor with First() before Next(), and code_store.go gets the missing LGPL header. --- execution/cache/code_store.go | 26 +++++++++++++++++---- execution/cache/generic_cache.go | 2 +- execution/cache/grow_lru.go | 2 +- execution/commitment/branch_cache_tail.go | 19 ++++++++------- execution/commitment/branch_cache_test.go | 28 +++++++++++++++++++++++ 5 files changed, 62 insertions(+), 15 deletions(-) diff --git a/execution/cache/code_store.go b/execution/cache/code_store.go index bfc0bcf9dd2..4ede832f590 100644 --- a/execution/cache/code_store.go +++ b/execution/cache/code_store.go @@ -1,6 +1,23 @@ +// Copyright 2026 The Erigon Authors +// This file is part of Erigon. +// +// Erigon is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . + package cache import ( + "bytes" "sync/atomic" "github.com/maypok86/otter/v2" @@ -67,6 +84,9 @@ func (s *CodeStore) GetByHash(tx kv.Getter, codeHash []byte) ([]byte, bool) { s.misses.Add(1) return nil, false } + // GetOne returns mmap-backed memory that must not outlive the tx; the otter + // tier is process-lifetime, so copy before caching/returning (kv contract). + code = bytes.Clone(code) s.mem.Set(key, code) s.tableHits.Add(1) return code, true @@ -121,14 +141,10 @@ func (s *CodeStore) Evict(tx kv.RwTx) error { } defer c.Close() target := int64(s.tableCapBytes / 10 * 9) - for s.tableSizeBytes.Load() > target { - k, v, err := c.Next() + for k, v, err := c.First(); k != nil && s.tableSizeBytes.Load() > target; k, v, err = c.Next() { if err != nil { return err } - if k == nil { - break - } if err := c.DeleteCurrent(); err != nil { return err } diff --git a/execution/cache/generic_cache.go b/execution/cache/generic_cache.go index f11d3103994..76161512c42 100644 --- a/execution/cache/generic_cache.go +++ b/execution/cache/generic_cache.go @@ -343,7 +343,7 @@ func (c *GenericCache[T]) put(key []byte, value T, txNum uint64, overwrite bool) // ModeEvictLRU: grow toward the ceiling before inserting into a full LRU, so a // busy cache expands into its budget rather than evicting at the start size. - if curCap := c.curCap.Load(); c.mode != ModeNoOp && lru.Len() >= int(curCap) && curCap < c.maxCap { + if curCap := c.curCap.Load(); c.mode != ModeNoOp && curCap < c.maxCap && lru.Len() >= int(curCap) { c.maybeGrow() lru = c.data.Load() } diff --git a/execution/cache/grow_lru.go b/execution/cache/grow_lru.go index 4df26b3c1d3..e4c13bcc2a1 100644 --- a/execution/cache/grow_lru.go +++ b/execution/cache/grow_lru.go @@ -89,7 +89,7 @@ func (g *growLRU[V]) Get(key uint64) (V, bool) { return g.cur.Load().Get(key) } func (g *growLRU[V]) Add(key uint64, value V) { lru := g.cur.Load() - if curCap := g.curCap.Load(); lru.Len() >= int(curCap) && curCap < g.maxCap { + if curCap := g.curCap.Load(); curCap < g.maxCap && lru.Len() >= int(curCap) { g.maybeGrow() lru = g.cur.Load() } diff --git a/execution/commitment/branch_cache_tail.go b/execution/commitment/branch_cache_tail.go index 4bb64199c26..02f5fbe4988 100644 --- a/execution/commitment/branch_cache_tail.go +++ b/execution/commitment/branch_cache_tail.go @@ -49,7 +49,7 @@ type tailLRU struct { maxCap uint32 resizeMu sync.Mutex - curCap uint32 + curCap atomic.Uint32 reserved int64 } @@ -61,7 +61,7 @@ func newTailLRU(maxCapacity uint32) *tailLRU { t := &tailLRU{maxCap: maxCapacity} t.reserved = int64(start) * tailEntryBytes cachebudget.Global.Take(t.reserved) // initial slice is small; take it unconditionally - t.curCap = start + t.curCap.Store(start) t.cur.Store(newTailShards(start)) return t } @@ -81,7 +81,9 @@ func (t *tailLRU) Get(key uint64) (*branchCacheEntry, bool) { func (t *tailLRU) Add(key uint64, entry *branchCacheEntry) { lru := t.cur.Load() - if lru.Len() >= int(t.curCap) { + // curCap < maxCap first: a fully-grown tail can never grow, so it must not + // pay lru.Len()'s per-shard locks on every insert. + if curCap := t.curCap.Load(); curCap < t.maxCap && lru.Len() >= int(curCap) { t.maybeGrow() lru = t.cur.Load() } @@ -96,14 +98,15 @@ func (t *tailLRU) maybeGrow() { defer t.resizeMu.Unlock() old := t.cur.Load() - if t.curCap >= t.maxCap || old.Len() < int(t.curCap) { + curCap := t.curCap.Load() + if curCap >= t.maxCap || old.Len() < int(curCap) { return } - newCap := t.curCap * tailGrowFactor + newCap := curCap * tailGrowFactor if newCap > t.maxCap { newCap = t.maxCap } - delta := int64(newCap-t.curCap) * tailEntryBytes + delta := int64(newCap-curCap) * tailEntryBytes if !cachebudget.Global.Reserve(delta) { return } @@ -114,7 +117,7 @@ func (t *tailLRU) maybeGrow() { } } t.cur.Store(next) - t.curCap = newCap + t.curCap.Store(newCap) t.reserved += delta } @@ -133,7 +136,7 @@ func (t *tailLRU) reset() { } cachebudget.Global.Release(t.reserved - int64(start)*tailEntryBytes) t.reserved = int64(start) * tailEntryBytes - t.curCap = start + t.curCap.Store(start) t.cur.Store(newTailShards(start)) } diff --git a/execution/commitment/branch_cache_test.go b/execution/commitment/branch_cache_test.go index 6541f365c8a..5d0e8db68f3 100644 --- a/execution/commitment/branch_cache_test.go +++ b/execution/commitment/branch_cache_test.go @@ -18,6 +18,7 @@ package commitment import ( "strings" + "sync" "testing" "github.com/stretchr/testify/require" @@ -284,3 +285,30 @@ func TestBranchCache_Unwind_FrozenSurvives(t *testing.T) { _, _, ok := c.Get(key) require.True(t, ok, "frozen txN=0 entry must survive any positive-txN unwind") } + +// TestBranchCache_ConcurrentTailGrow drives concurrent tail Puts well past the +// 512-entry start capacity so maybeGrow runs under contention. It regresses the +// data race where Add read tailLRU.curCap unsynchronized while maybeGrow/reset +// wrote it under resizeMu. Must be run under -race to be meaningful. +func TestBranchCache_ConcurrentTailGrow(t *testing.T) { + c := NewBranchCache(4096) // max >> 512 start, so the tail actually grows + + const ( + workers = 8 + perWorker = 2000 // 16k distinct deep keys >> 512 → forces maybeGrow + ) + var wg sync.WaitGroup + for w := 0; w < workers; w++ { + wg.Add(1) + go func(w int) { + defer wg.Done() + for i := 0; i < perWorker; i++ { + // odd flag (0x10) + 3 bytes → 7 nibbles → tail; unique per (w,i). + key := []byte{0x10, byte(w), byte(i), byte(i >> 8)} + c.Put(key, []byte{byte(i)}, 0, 100) + c.Get(key) + } + }(w) + } + wg.Wait() +} From f1d4cfecd68634fdba1bc7c04f0a6bc2e1de8a10 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 7 Jul 2026 11:23:17 +0000 Subject: [PATCH 15/18] db/state, execution/commitment: hoist adaptive pin controller to aggregator scope The pin controller held its recency accounting (promoted contracts, cold-block streaks) per-SharedDomains while the pins live in the aggregator-scope BranchCache. Because SDs bind continuously (every ExecV3 run + fork validation), that put the pin lifecycle under SD churn: a replaced SD's promotions were orphaned with no owner to age them out, and last-writer-wins on the miss callback let a fork SD's controller steal miss attribution. Move the controller to the same lifetime as the cache (the commitment Domain), fetched by SharedDomains via a duck-typed provider like the cache itself. Pin residency now ages purely by block-access recency across SD rotations. The tx-scoped reader/resolver-factory/dbBranches-provider are passed per OnBlockComplete call instead of stored via SetParallelMode, so the now-shared controller carries no per-tx mutable state (c.mu serializes callers); Bind becomes idempotent (one controller, one onCacheMiss). Also bounds the dbBranches provider scan by the per-contract pin budget and fixes its nil-upper-bound (all-0xff prefix) early-stop and swallowed cursor err. --- db/state/aggregator.go | 13 +++++++ db/state/domain.go | 9 +++++ db/state/execctx/domain_shared.go | 30 ++++++++++------ execution/commitment/adaptive_pin.go | 46 +++++++++++++----------- execution/commitment/branch_cache.go | 12 +++++++ execution/commitment/preload.go | 6 +++- execution/commitment/preload_parallel.go | 6 +++- 7 files changed, 88 insertions(+), 34 deletions(-) diff --git a/db/state/aggregator.go b/db/state/aggregator.go index 216bf4db0a1..46f7397f894 100644 --- a/db/state/aggregator.go +++ b/db/state/aggregator.go @@ -418,6 +418,10 @@ func (a *Aggregator) ConfigureDomains() error { if dbg.UseStateCache && !a.branchCacheDisabled { if cd := a.d[kv.CommitmentDomain]; cd != nil && cd.branchCache == nil { cd.branchCache = commitment.NewBranchCache(commitment.DefaultBranchCacheTailCapacity) + if !dbg.EnvBool("DISABLE_ADAPTIVE_PIN", false) { + cd.adaptivePinController = commitment.NewAdaptivePinController( + cd.branchCache, commitment.DefaultAdaptivePinControllerConfig(), a.logger) + } } } @@ -2483,6 +2487,15 @@ func (at *AggregatorRoTx) BranchCache() *commitment.BranchCache { return at.d[kv.CommitmentDomain].d.branchCache } +// AdaptivePinController attached to the commitment domain (implements +// commitment.AdaptivePinControllerProvider). +func (at *AggregatorRoTx) AdaptivePinController() *commitment.AdaptivePinController { + if at.d[kv.CommitmentDomain] == nil { + return nil + } + return at.d[kv.CommitmentDomain].d.adaptivePinController +} + // MetricsCollector exposes the aggregator-scope KV-read metrics collector, // fetched by SharedDomains through the duck-typed kvmetrics.MetricsCollectorProvider // (same pattern as BranchCache), so every read path folds into one process-level diff --git a/db/state/domain.go b/db/state/domain.go index f75eb1ed04e..a6b6f8568c6 100644 --- a/db/state/domain.go +++ b/db/state/domain.go @@ -92,6 +92,9 @@ type Domain struct { // Long-lived commitment-branch cache; non-nil only on the commitment domain. branchCache *commitment.BranchCache + // Adaptive pin controller, co-located with branchCache so pin residency ages + // by block-access recency across all SharedDomains rather than per-SD. + adaptivePinController *commitment.AdaptivePinController // _testBuildAccessorHook - test-only: called with the recsplit before the build loop in buildHashMapAccessor _testBuildAccessorHook func(rs *recsplit.RecSplit) @@ -147,6 +150,12 @@ func (d *Domain) BranchCache() *commitment.BranchCache { return d.branchCache } +// AdaptivePinController returns the aggregator-lifetime pin controller +// co-located with BranchCache. Non-nil only on the commitment domain. +func (d *Domain) AdaptivePinController() *commitment.AdaptivePinController { + return d.adaptivePinController +} + // kvWriteVersion is the version stamped on a new .kv file: the domain's KVWriteVersion hook if set, else DataKV.Current. func (d *Domain) kvWriteVersion() version.Version { if d.KVWriteVersion != nil { diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index f7ff4d9857e..8576215849b 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -217,12 +217,10 @@ func NewSharedDomains(ctx context.Context, tx kv.TemporalTx, logger log.Logger, } sd.sdCtx = commitmentdb.NewSharedDomainsCommitmentContext(sd, commitment.ModeDirect, tx.Debug().Dirs().Tmp, trieCfg) - if branchCache != nil && !dbg.EnvBool("DISABLE_ADAPTIVE_PIN", false) { - sd.adaptivePinController = commitment.NewAdaptivePinController( - branchCache, - commitment.DefaultAdaptivePinControllerConfig(), - logger, - ) + // The pin controller is aggregator-scoped (co-located with branchCache) so pin + // residency ages by block-access recency across all SharedDomains, not per-SD. + if p, ok := tx.AggTx().(commitment.AdaptivePinControllerProvider); ok { + sd.adaptivePinController = p.AdaptivePinController() } _, blockNum, err := sd.SeekCommitment(ctx, tx) @@ -1041,24 +1039,34 @@ func (sd *SharedDomains) Commit(ctx context.Context, tx kv.RwTx, validate ...fun } defer c.Close() evenFrom, evenTo, oddFrom, oddTo := commitment.ContractTrunkKeyRanges(commitment.ContractNibbles(contractHash)) + // Bound the scan by the per-contract pin ceiling — the preload can't + // pin more than that, so gathering further is pure waste on the + // Commit path. A nil `to` (all-0xff prefix) means scan to the range's + // natural end, not stop immediately. + budget := sd.adaptivePinController.PerContractBudgetBytes() + scanned := 0 scan := func(from, to []byte) { - for k, v, err := c.Seek(from); k != nil && err == nil; k, v, err = c.NextNoDup() { - if bytes.Compare(k, to) >= 0 { + for k, v, err := c.Seek(from); k != nil; k, v, err = c.NextNoDup() { + if err != nil { + return // best-effort residency hint: keep what was gathered + } + if to != nil && bytes.Compare(k, to) >= 0 { return } if len(v) < 8 { continue } m[string(common.Copy(k))] = common.Copy(v[8:]) + if scanned += len(k) + len(v); scanned >= budget { + return + } } } scan(evenFrom, evenTo) scan(oddFrom, oddTo) return m } - sd.adaptivePinController.SetParallelMode(factory, provider) - sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader) - sd.adaptivePinController.SetParallelMode(nil, nil) + sd.adaptivePinController.OnBlockComplete(ctx, sd.txNum, reader, factory, provider) } } if err := tx.Commit(); err != nil { diff --git a/execution/commitment/adaptive_pin.go b/execution/commitment/adaptive_pin.go index 9db4db62d5f..454dcbba12e 100644 --- a/execution/commitment/adaptive_pin.go +++ b/execution/commitment/adaptive_pin.go @@ -53,9 +53,6 @@ type AdaptivePinController struct { mu sync.Mutex states map[[32]byte]*adaptiveContractState - - parallelResolverFactory ParallelResolverFactory - dbBranchesProvider DbBranchesProvider } // ParallelResolverFactory builds a fresh BatchBranchResolver for one @@ -138,15 +135,10 @@ func (c *AdaptivePinController) Bind() { c.cache.SetMissCallback(c.onCacheMiss) } -// SetParallelMode switches promote/extend to the wave-BFS parallel preload. -// Either argument may be nil to clear; with factory==nil the controller uses -// the serial-BFS CommitmentReader path. Already-promoted contracts keep -// their existing serial/parallel state until next demote. -func (c *AdaptivePinController) SetParallelMode(factory ParallelResolverFactory, provider DbBranchesProvider) { - c.mu.Lock() - defer c.mu.Unlock() - c.parallelResolverFactory = factory - c.dbBranchesProvider = provider +// PerContractBudgetBytes is the per-contract pin ceiling; a dbBranches provider +// need never gather more than this since the preload can't pin beyond it. +func (c *AdaptivePinController) PerContractBudgetBytes() int { + return c.cfg.PerContractMaxBudgetBytes } func (c *AdaptivePinController) onCacheMiss(prefix []byte) { @@ -165,7 +157,12 @@ func (c *AdaptivePinController) onCacheMiss(prefix []byte) { // OnBlockComplete consumes the per-block miss snapshot and decides // promotions, extensions, and demotions. Synchronous — preloads run // inline so the new pin set is available for the next block's reads. -func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, txNum uint64, reader CommitmentReader) { +// +// The controller is aggregator-scoped (one owner across SharedDomains) so pin +// residency ages by block-access recency, not SD binds; the tx-scoped reader/ +// factory/provider are therefore passed per call rather than stored, and c.mu +// serializes concurrent callers. +func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, txNum uint64, reader CommitmentReader, factory ParallelResolverFactory, provider DbBranchesProvider) { misses := c.snapshotMisses() c.mu.Lock() @@ -174,8 +171,8 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, txNum uint6 // One factory call per block, shared across all contracts. nil falls back to serial. var parallelResolve BatchBranchResolver var releaseParallel func() - if c.parallelResolverFactory != nil { - r, release, err := c.parallelResolverFactory() + if factory != nil { + r, release, err := factory() if err != nil { c.warnf("[adaptive-pin] parallel resolver factory failed, falling back to serial", "err", err, "txNum", txNum) } else { @@ -200,7 +197,7 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, txNum uint6 if step > remaining { step = remaining } - if err := c.runExtensionLocked(ctx, state, step, parallelResolve, reader); err != nil { + if err := c.runExtensionLocked(ctx, state, txNum, step, parallelResolve, reader, provider); err != nil { c.warnf("[adaptive-pin] extend failed", "hash", hex.EncodeToString(hash[:]), "err", err) } else { extended++ @@ -219,7 +216,7 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, txNum uint6 if len(misses) > 0 && len(c.states) < c.cfg.MaxPromotedContracts { candidates := pickPromotionCandidates(misses, c.cfg.PromoteThresholdMisses, c.cfg.MaxPromotedContracts-len(c.states)) for _, hash := range candidates { - state, err := c.promoteLocked(ctx, hash, txNum, parallelResolve, reader) + state, err := c.promoteLocked(ctx, hash, txNum, parallelResolve, reader, provider) if err != nil { c.warnf("[adaptive-pin] initial-view failed", "hash", hex.EncodeToString(hash[:]), "err", err) continue @@ -284,15 +281,17 @@ func (c *AdaptivePinController) promoteLocked( txNum uint64, parallelResolve BatchBranchResolver, reader CommitmentReader, + provider DbBranchesProvider, ) (*adaptiveContractState, error) { if parallelResolve != nil { p, err := NewContractTrunkPreloadParallel(hash[:]) if err != nil { return nil, err } + p.pinTxNum = txNum var dbBranches map[string][]byte - if c.dbBranchesProvider != nil { - dbBranches = c.dbBranchesProvider(hash[:]) + if provider != nil { + dbBranches = provider(hash[:]) } if _, _, err := p.Run(c.cfg.InitialViewBudgetBytes, dbBranches, parallelResolve, c.cache, c.logger); err != nil { for _, prefix := range p.PinnedPrefixes() { @@ -310,6 +309,7 @@ func (c *AdaptivePinController) promoteLocked( if err != nil { return nil, err } + p.pinTxNum = txNum if _, _, err := p.Run(c.cfg.InitialViewBudgetBytes, reader, c.cache, c.logger); err != nil { for _, prefix := range p.PinnedPrefixes() { c.cache.Invalidate(prefix) @@ -329,21 +329,25 @@ func (c *AdaptivePinController) promoteLocked( func (c *AdaptivePinController) runExtensionLocked( ctx context.Context, state *adaptiveContractState, + txNum uint64, stepBudget int, parallelResolve BatchBranchResolver, reader CommitmentReader, + provider DbBranchesProvider, ) error { if state.parallel != nil { if parallelResolve == nil { return nil } var dbBranches map[string][]byte - if c.dbBranchesProvider != nil { - dbBranches = c.dbBranchesProvider(state.contractHash[:]) + if provider != nil { + dbBranches = provider(state.contractHash[:]) } + state.parallel.pinTxNum = txNum _, _, err := state.parallel.Run(stepBudget, dbBranches, parallelResolve, c.cache, c.logger) return err } + state.preload.pinTxNum = txNum _, _, err := state.preload.Run(stepBudget, reader, c.cache, c.logger) return err } diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 755cb0bb43e..0d7de38e98b 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -299,6 +299,13 @@ type BranchCacheProvider interface { BranchCache() *BranchCache } +// AdaptivePinControllerProvider exposes the aggregator-lifetime pin controller +// co-located with the BranchCache, duck-typed for the same reason (avoids a +// db/state import cycle). Returning nil means adaptive pinning is disabled. +type AdaptivePinControllerProvider interface { + AdaptivePinController() *AdaptivePinController +} + // branchCacheTailShards splits the LRU tail into independently-locked shards so // concurrent commitment mounts / warmup workers don't serialize on one mutex. const branchCacheTailShards = 256 @@ -403,6 +410,11 @@ func (c *BranchCache) storageRoute(prefix []byte, create bool) (st *trunk, acct if len(prefix) < 33 { return nil, nil, nil, false } + // Nothing pinned and not creating: skip the CompactToHex + packed-key alloc + // that every >=64-nibble read would otherwise pay before finding no pins. + if !create && c.pinned.Load() == nil { + return nil, nil, nil, false + } nib := nibbles.CompactToHex(prefix) if len(nib) < 64 { return nil, nil, nil, false diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index dfc2152c26b..5bb80c48ea9 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -39,6 +39,10 @@ type ContractTrunkPreload struct { pinned int usedBytes int maxDepthReached int + // pinTxNum stamps pinned entries with the head txNum they were read at, so a + // later unwind below that point evicts them via the BranchCache floor (a + // txN=0 pin would escape it and be served stale after a deep unwind). + pinTxNum uint64 } // NewContractTrunkPreload seeds a preload state at depth 64 (storage @@ -98,7 +102,7 @@ func (p *ContractTrunkPreload) Run( break } - cache.PinEntry(prefix, v, step, 0) + cache.PinEntry(prefix, v, step, p.pinTxNum) // HexToCompact may alias a reused buffer; copy for a stable Invalidate handle. prefixCopy := make([]byte, len(prefix)) copy(prefixCopy, prefix) diff --git a/execution/commitment/preload_parallel.go b/execution/commitment/preload_parallel.go index 080a1f1dfd7..b9fbaf0cfd8 100644 --- a/execution/commitment/preload_parallel.go +++ b/execution/commitment/preload_parallel.go @@ -71,6 +71,10 @@ type ContractTrunkPreloadParallel struct { usedBytes int maxDepthReached int dbHitsPinned int + // pinTxNum stamps pinned entries with the head txNum they were read at, so a + // later unwind below that point evicts them via the BranchCache floor (a + // txN=0 pin would escape it and be served stale after a deep unwind). + pinTxNum uint64 } // NewContractTrunkPreloadParallel seeds a preload at depth 64 (storage subtree root). @@ -119,7 +123,7 @@ func (p *ContractTrunkPreloadParallel) Run( budgetHit = true return false } - cache.PinEntry(pk.key, v, 0, 0) + cache.PinEntry(pk.key, v, 0, p.pinTxNum) kc := make([]byte, len(pk.key)) copy(kc, pk.key) p.pinnedPrefixes = append(p.pinnedPrefixes, kc) From 4e8c56a36c40077bcbfca776f2ed3ee8f07d39f2 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 7 Jul 2026 11:36:28 +0000 Subject: [PATCH 16/18] execution/cache, execution/commitment: review cleanups (sizing, pin metrics, dead code) - CodeCache avgCodeEntryBytes 4KB -> 12KB: the entry-count cap is the only bound (byte counters don't evict), so size it to the resident-code skew (hot contracts 10-24KB) to keep RAM near the budget instead of several x over. - Wire BranchCache.PublishMetrics from OnBlockComplete so the pin/preload metrics are actually emitted, and reset the publish watermarks in Clear() so the next publish can't compute a wrapped delta. - BranchCache.Close now releases the tail's cachebudget reservation (was asymmetric with CodeCache/GenericCache.Close). - Remove dead APIs (PinnedStats, TryClaimPreload + its preloadClaimed field, PromotedContracts); lower the per-construction branch-cache init log to Debug. - Declare DISABLE_ADAPTIVE_PIN in dbg experiments (parity with USE_CODE_STORE) and trim two scope-narration comments per the comment policy. --- common/dbg/experiments.go | 1 + db/state/aggregator.go | 2 +- db/state/execctx/domain_shared.go | 6 ++--- execution/cache/code_cache.go | 9 +++---- execution/commitment/adaptive_pin.go | 11 +-------- execution/commitment/branch_cache.go | 24 +++++++------------ execution/commitment/branch_cache_tail.go | 9 +++++++ .../execmoduletester/exec_module_tester.go | 6 ++--- 8 files changed, 29 insertions(+), 39 deletions(-) diff --git a/common/dbg/experiments.go b/common/dbg/experiments.go index ff8e0605ca8..dcdcd32c9d3 100644 --- a/common/dbg/experiments.go +++ b/common/dbg/experiments.go @@ -117,6 +117,7 @@ var ( UseTxDependencies = EnvBool("USE_TX_DEPENDENCIES", false) UseStateCache = EnvBool("USE_STATE_CACHE", true) UseCodeStore = EnvBool("USE_CODE_STORE", true) + DisableAdaptivePin = EnvBool("DISABLE_ADAPTIVE_PIN", false) AssertStateCache = EnvBool("ASSERT_STATE_CACHE", false) ReadAhead = EnvBool("READ_AHEAD", true) diff --git a/db/state/aggregator.go b/db/state/aggregator.go index 46f7397f894..82f769c8318 100644 --- a/db/state/aggregator.go +++ b/db/state/aggregator.go @@ -418,7 +418,7 @@ func (a *Aggregator) ConfigureDomains() error { if dbg.UseStateCache && !a.branchCacheDisabled { if cd := a.d[kv.CommitmentDomain]; cd != nil && cd.branchCache == nil { cd.branchCache = commitment.NewBranchCache(commitment.DefaultBranchCacheTailCapacity) - if !dbg.EnvBool("DISABLE_ADAPTIVE_PIN", false) { + if !dbg.DisableAdaptivePin { cd.adaptivePinController = commitment.NewAdaptivePinController( cd.branchCache, commitment.DefaultAdaptivePinControllerConfig(), a.logger) } diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 8576215849b..1ea55b15a30 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -1001,10 +1001,8 @@ func (sd *SharedDomains) Commit(ctx context.Context, tx kv.RwTx, validate ...fun if err := runValidate(); err != nil { return err } - // Adaptive controller hook: decide promotions/demotions from this batch's - // miss pressure using the in-flight (pre-Commit) tx so commitment reads see - // the just-flushed bytes. Runs before Commit because the tx is finalized - // after; the coherence floor evicts pins from a rolled-back batch. + // Adaptive pin promotions/demotions run on the in-flight (pre-Commit) tx so + // the preload sees the just-flushed bytes. if sd.adaptivePinController != nil { if ttx, ok := tx.(kv.TemporalTx); ok { reader := func(prefix []byte) ([]byte, uint64, bool, error) { diff --git a/execution/cache/code_cache.go b/execution/cache/code_cache.go index 586065df899..393c7409a04 100644 --- a/execution/cache/code_cache.go +++ b/execution/cache/code_cache.go @@ -47,10 +47,11 @@ const ( // EXTCODESIZE / EXTCODEHASH callers). DefaultCodeSizeCacheEntries int64 = 1_000_000 // avgCodeEntryBytes translates the code byte budget into the freelru - // entry-count cap. Contract bytecode varies widely (a few bytes to 24 KB); - // the persistent (MDBX-backed) cold tier backstops entries evicted from this - // hot tier, so a loose byte bound here is acceptable. - avgCodeEntryBytes = 4096 + // entry-count cap (the only bound — the byte counters don't evict). Sized to + // the resident-code skew (hot contracts run 10-24 KB) rather than the raw + // average so the cap keeps RAM near the budget instead of several × over; the + // persistent (MDBX-backed) cold tier backstops entries the tighter cap evicts. + avgCodeEntryBytes = 12 * 1024 // codeSizeEntryBytes is the resident cost of one size-layer slot (freelru // element holding size/keyHash/txNum/epoch), used to map the size-layer entry // ceiling to an envelope byte budget. diff --git a/execution/commitment/adaptive_pin.go b/execution/commitment/adaptive_pin.go index 454dcbba12e..2ec9c6c711d 100644 --- a/execution/commitment/adaptive_pin.go +++ b/execution/commitment/adaptive_pin.go @@ -236,6 +236,7 @@ func (c *AdaptivePinController) OnBlockComplete(ctx context.Context, txNum uint6 mxAdaptiveDemoted.AddUint64(uint64(demoted)) } mxAdaptiveActive.SetUint64(uint64(len(c.states))) + c.cache.PublishMetrics() if c.logger != nil && (promoted+extended+demoted > 0 || len(c.states) > 0) { c.logger.Info("[adaptive-pin]", @@ -352,16 +353,6 @@ func (c *AdaptivePinController) runExtensionLocked( return err } -func (c *AdaptivePinController) PromotedContracts() [][32]byte { - c.mu.Lock() - defer c.mu.Unlock() - out := make([][32]byte, 0, len(c.states)) - for h := range c.states { - out = append(out, h) - } - return out -} - func pickPromotionCandidates(misses map[[32]byte]uint64, threshold uint64, maxN int) [][32]byte { if maxN <= 0 { return nil diff --git a/execution/commitment/branch_cache.go b/execution/commitment/branch_cache.go index 0d7de38e98b..d57df7e3791 100644 --- a/execution/commitment/branch_cache.go +++ b/execution/commitment/branch_cache.go @@ -111,9 +111,6 @@ type BranchCache struct { // contract; nil hot path is one atomic load + nil check. onMiss atomic.Pointer[MissCallback] - // preloadClaimed gates the one-shot residency preload trigger. - preloadClaimed atomic.Bool - // last-published pinned counter snapshots — PublishMetrics emits the delta // since the previous publish so the Prometheus counters track per-Flush // activity, not snapshot absolutes. @@ -326,7 +323,7 @@ func NewBranchCache(tailCapacity int) *BranchCache { // Before any unwind every entry's txN is at/below the floor, so the epoch // check never strands a valid entry. bc.coh.Init() - log.Info("[branch-cache] init", "trunkEnabled", !bc.trunkDisabled, "tailCap", tailCapacity, "trunkDepth", maxDepth) + log.Debug("[branch-cache] init", "trunkEnabled", !bc.trunkDisabled, "tailCap", tailCapacity, "trunkDepth", maxDepth) return bc } @@ -334,6 +331,9 @@ func NewBranchCache(tailCapacity int) *BranchCache { // size their trunk depth against real concurrency. Idempotent. func (c *BranchCache) Close() { if c.closed.CompareAndSwap(false, true) { + if t := c.tail.Load(); t != nil { + t.Close() + } activeBranchCaches.Add(-1) } } @@ -635,18 +635,6 @@ func (c *BranchCache) PinnedCount() int { return int(c.pinnedEntries.Load()) } -// PinnedStats returns the pinned-tier hit/miss/entries counters. -func (c *BranchCache) PinnedStats() (hits, misses uint64, entries int) { - return c.pinnedHits.Load(), c.pinnedMisses.Load(), int(c.pinnedEntries.Load()) -} - -// TryClaimPreload returns true exactly once per cache lifetime — the residency -// preload trigger uses it so the preload runs once regardless of how many -// SharedDomains instances are constructed. -func (c *BranchCache) TryClaimPreload() bool { - return c.preloadClaimed.CompareAndSwap(false, true) -} - // Get retrieves branch data from the cache. Returns the canonical encoded // bytes (with the leading 2-byte touch-map prefix) plus the on-disk file // step the bytes came from (0 if not tracked). @@ -746,6 +734,10 @@ func (c *BranchCache) Clear() { c.trunkMisses.Store(0) c.pinnedHits.Store(0) c.pinnedMisses.Store(0) + // Reset the publish watermarks too, else the next PublishMetrics computes a + // wrapped (huge) delta against the pre-Clear counter. + c.lastPublishedPinnedHits.Store(0) + c.lastPublishedPinnedMisses.Store(0) c.tailHits.Store(0) c.tailMisses.Store(0) c.bytesServed.Store(0) diff --git a/execution/commitment/branch_cache_tail.go b/execution/commitment/branch_cache_tail.go index 02f5fbe4988..225090ece28 100644 --- a/execution/commitment/branch_cache_tail.go +++ b/execution/commitment/branch_cache_tail.go @@ -143,3 +143,12 @@ func (t *tailLRU) reset() { func (t *tailLRU) Len() int { return t.cur.Load().Len() } + +// Close returns the tail's envelope reservation. Call once (BranchCache.Close +// guards against double-release with its closed CAS). +func (t *tailLRU) Close() { + t.resizeMu.Lock() + defer t.resizeMu.Unlock() + cachebudget.Global.Release(t.reserved) + t.reserved = 0 +} diff --git a/execution/execmodule/execmoduletester/exec_module_tester.go b/execution/execmodule/execmoduletester/exec_module_tester.go index 2fa9ce82343..b5d063693f8 100644 --- a/execution/execmodule/execmoduletester/exec_module_tester.go +++ b/execution/execmodule/execmoduletester/exec_module_tester.go @@ -722,10 +722,8 @@ func New(tb testing.TB, opts ...Option) *ExecModuleTester { RecentReceipts: mock.Notifications.RecentReceipts, } // Per-instance domain cache, held on the tester so Close releases its - // envelope reservation. The production default is safe here: the caches - // jump-grow from a small start into the shared envelope on demand, so a - // fixture with a small working set stays small regardless of the configured - // budget — no hand-tuned test size, same code path as production. + // envelope reservation. Uses the production default — the caches jump-grow on + // demand, so a small-working-set fixture stays small. mock.domainCache = cache.NewDefaultStateCache() mock.ExecModule = execmodule.NewExecModule( ctx, From 615c83e0530dfd6c0032f98d12f6f86fcb50ed67 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 7 Jul 2026 11:45:19 +0000 Subject: [PATCH 17/18] execution/commitment, execution/cache: restore deleted tests, fix headers - Restore TestBranchCache_StateKeyNeverCached (state-key-never-cached invariant) and TestBranchCache_ShardedTailUnwindAcrossShards (lazy epoch+floor unwind across tail shards), updated to the current API. The latter is sized within the tail start capacity so it asserts the unwind floor deterministically rather than depending on budget-funded tail growth, and closes the cache to return its budget/active-count (avoids cross-test interference). - code_store_test.go copyright year 2024 -> 2026; complete the truncated LGPL headers on adaptive_pin.go / preload.go / preload_parallel.go / trunk_pin_metrics.go. --- execution/cache/code_store_test.go | 2 +- execution/commitment/adaptive_pin.go | 8 ++++ execution/commitment/branch_cache_test.go | 48 +++++++++++++++++++++++ execution/commitment/preload.go | 8 ++++ execution/commitment/preload_parallel.go | 8 ++++ execution/commitment/trunk_pin_metrics.go | 8 ++++ 6 files changed, 81 insertions(+), 1 deletion(-) diff --git a/execution/cache/code_store_test.go b/execution/cache/code_store_test.go index e0f675d05b5..7559464ac9e 100644 --- a/execution/cache/code_store_test.go +++ b/execution/cache/code_store_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 The Erigon Authors +// Copyright 2026 The Erigon Authors // This file is part of Erigon. // // Erigon is free software: you can redistribute it and/or modify diff --git a/execution/commitment/adaptive_pin.go b/execution/commitment/adaptive_pin.go index 2ec9c6c711d..674f3077e71 100644 --- a/execution/commitment/adaptive_pin.go +++ b/execution/commitment/adaptive_pin.go @@ -5,6 +5,14 @@ // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . package commitment diff --git a/execution/commitment/branch_cache_test.go b/execution/commitment/branch_cache_test.go index 5d0e8db68f3..b7bd4c8b2fd 100644 --- a/execution/commitment/branch_cache_test.go +++ b/execution/commitment/branch_cache_test.go @@ -286,6 +286,54 @@ func TestBranchCache_Unwind_FrozenSurvives(t *testing.T) { require.True(t, ok, "frozen txN=0 entry must survive any positive-txN unwind") } +// TestBranchCache_StateKeyNeverCached pins that the commitment checkpoint key is +// never served or stored (serving a stale checkpoint corrupts the trie root), +// and that invalidating it doesn't evict real entries. +func TestBranchCache_StateKeyNeverCached(t *testing.T) { + c := NewBranchCache(100) + + c.Put(KeyCommitmentState, []byte("checkpoint"), 1, 1) + _, _, ok := c.Get(KeyCommitmentState) + require.False(t, ok, "state key must never be served from the cache") + require.Equal(t, 0, c.tailLen(), "state key must not occupy a tail slot") + + deepKey := []byte{0x12, 0x34} + c.Put(deepKey, []byte("d"), 0, 0) + c.Invalidate(KeyCommitmentState) + got, _, ok := c.Get(deepKey) + require.True(t, ok, "invalidating the state key must not evict real entries") + require.Equal(t, []byte("d"), got) +} + +// TestBranchCache_ShardedTailUnwindAcrossShards verifies the lazy (epoch+floor) +// unwind drops exactly the entries at/above the floor across all tail shards. +func TestBranchCache_ShardedTailUnwindAcrossShards(t *testing.T) { + c := NewBranchCache(DefaultBranchCacheTailCapacity) + defer c.Close() + + // Stay within the tail's start capacity so the entries can't LRU-evict: the + // tail only jump-grows when the shared cachebudget has room, which a full + // test run may have consumed — this test asserts the unwind floor, not growth. + const n = 64 + const watermark = 32 + for i := 0; i < n; i++ { + prefix := []byte{0x01, byte(i), byte(i >> 8)} + c.Put(prefix, []byte{byte(i)}, 0, uint64(i)) + } + + c.Unwind(watermark) + + for i := 0; i < n; i++ { + prefix := []byte{0x01, byte(i), byte(i >> 8)} + _, _, ok := c.Get(prefix) + if uint64(i) >= watermark { + require.False(t, ok, "entry txN=%d must be dropped by floor=%d", i, watermark) + } else { + require.True(t, ok, "entry txN=%d must survive floor=%d", i, watermark) + } + } +} + // TestBranchCache_ConcurrentTailGrow drives concurrent tail Puts well past the // 512-entry start capacity so maybeGrow runs under contention. It regresses the // data race where Add read tailLRU.curCap unsynchronized while maybeGrow/reset diff --git a/execution/commitment/preload.go b/execution/commitment/preload.go index 5bb80c48ea9..cb1c0f50508 100644 --- a/execution/commitment/preload.go +++ b/execution/commitment/preload.go @@ -5,6 +5,14 @@ // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . package commitment diff --git a/execution/commitment/preload_parallel.go b/execution/commitment/preload_parallel.go index b9fbaf0cfd8..d8a66e6f641 100644 --- a/execution/commitment/preload_parallel.go +++ b/execution/commitment/preload_parallel.go @@ -5,6 +5,14 @@ // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . package commitment diff --git a/execution/commitment/trunk_pin_metrics.go b/execution/commitment/trunk_pin_metrics.go index b4d3000eb43..3a0b956e4fc 100644 --- a/execution/commitment/trunk_pin_metrics.go +++ b/execution/commitment/trunk_pin_metrics.go @@ -5,6 +5,14 @@ // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. +// +// Erigon is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with Erigon. If not, see . package commitment From 9a56051426c178f25ea8ca3a99087654692752d9 Mon Sep 17 00:00:00 2001 From: mh0lt Date: Tue, 7 Jul 2026 17:14:40 +0000 Subject: [PATCH 18/18] execution/cache, execution/commitment, db/state: clear post-approval residuals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - code_store: fix the Evict/sumTableBytes error swallow — a cursor error returns k=nil, which exited the k!=nil loop condition before the in-body err check; now checked after the loop. - branch_cache_test: defer Close on the ConcurrentTailGrow and StateKeyNeverCached tests so they return their tail reservation to the shared envelope. - preload_parallel: record why the parallel preload pins with step=0 (no single source step across merged files; pinTxNum already gives unwind coherence). - Remove dead code: ClearBranchCache, WarmerBranchOutcomeStats (+ its write-only counters), write-only diskLoadStorage/diskLoadAccount, ProbeStateLayers/ SiteIdentity (+ probeSd/probeTx wiring and SharedDomains.ProbeReadLayers), and the redundant `var _ = context.Background`. - state_object: drop a stray blank line (whitespace-only diff vs main). --- db/state/execctx/domain_shared.go | 37 ------------------- execution/cache/code_store.go | 24 ++++++------ execution/commitment/adaptive_pin.go | 2 - execution/commitment/branch_cache_test.go | 2 + .../commitmentdb/commitment_context.go | 25 ------------- execution/commitment/hex_patricia_hashed.go | 11 ++---- execution/commitment/preload_parallel.go | 4 ++ execution/commitment/warmuper.go | 16 -------- execution/state/state_object.go | 1 - 9 files changed, 21 insertions(+), 101 deletions(-) diff --git a/db/state/execctx/domain_shared.go b/db/state/execctx/domain_shared.go index 1ea55b15a30..9f9bbd887c1 100644 --- a/db/state/execctx/domain_shared.go +++ b/db/state/execctx/domain_shared.go @@ -757,31 +757,6 @@ func (sd *SharedDomains) PrintCacheStats() { } } -// ProbeReadLayers samples each independent state layer (this SD's mem, -// the parent SD's mem, and direct MDBX via tx.GetLatest) and returns -// the bytes from each. Read-only, intended for divergence-detection -// diagnostics — pinpoints which layer holds bytes that disagree with -// the BranchCache. Bytes are copied so the caller can hold them past -// tx lifetime. -func (sd *SharedDomains) ProbeReadLayers(domain kv.Domain, tx kv.TemporalTx, key []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) { - if v, _, ok := sd.mem.GetLatest(domain, key); ok { - memOk = true - mem = append([]byte(nil), v...) - } - if sd.parent != nil { - if v, _, ok := sd.parent.mem.GetLatest(domain, key); ok { - parentOk = true - parentMem = append([]byte(nil), v...) - } - } - if tx != nil { - if v, _, err := tx.GetLatest(domain, key); err == nil { - mdbx = append([]byte(nil), v...) - } - } - return -} - func (sd *SharedDomains) ClearRam(resetCommitment bool) { // When the commitment calculator goroutine owns the Updates buffer, // skip ClearRam on the commitment context to avoid concurrent btree access. @@ -1111,18 +1086,6 @@ func (sd *SharedDomains) Commit(ctx context.Context, tx kv.RwTx, validate ...fun return nil } -// ClearBranchCache empties the aggregator-scope commitment BranchCache. -// Use after operations that mutate commitment state outside the normal -// sd.Flush callback path — notably SetHead unwind, which truncates -// commitment domain history but does not re-write all the keys whose -// cached values are now stale. Without this call, the next FCU sees -// the pre-unwind KeyCommitmentState and trips ErrBehindCommitment. -func (sd *SharedDomains) ClearBranchCache() { - if sd.branchCache != nil { - sd.branchCache.Clear() - } -} - // DetachBranchCache makes this SharedDomains ignore the aggregator-scope // BranchCache: commitment branch reads go straight to sd.mem/overlay/MDBX and // no read populates the shared cache. Used for fork-validation SDs, which read diff --git a/execution/cache/code_store.go b/execution/cache/code_store.go index 4ede832f590..03206cc39c1 100644 --- a/execution/cache/code_store.go +++ b/execution/cache/code_store.go @@ -141,16 +141,17 @@ func (s *CodeStore) Evict(tx kv.RwTx) error { } defer c.Close() target := int64(s.tableCapBytes / 10 * 9) - for k, v, err := c.First(); k != nil && s.tableSizeBytes.Load() > target; k, v, err = c.Next() { - if err != nil { - return err - } - if err := c.DeleteCurrent(); err != nil { - return err + // Check err after the loop, not inside: a cursor error returns k=nil, which + // exits the k!=nil condition before any in-body check runs. + k, v, err := c.First() + for k != nil && s.tableSizeBytes.Load() > target { + if derr := c.DeleteCurrent(); derr != nil { + return derr } s.tableSizeBytes.Add(-int64(len(k) + len(v))) + k, v, err = c.Next() } - return nil + return err } // sumTableBytes returns the total key+value byte size of TblCodeCache, in the @@ -162,11 +163,10 @@ func sumTableBytes(tx kv.RwTx) (int64, error) { } defer c.Close() var total int64 - for k, v, err := c.First(); k != nil; k, v, err = c.Next() { - if err != nil { - return 0, err - } + k, v, err := c.First() + for k != nil { total += int64(len(k) + len(v)) + k, v, err = c.Next() } - return total, nil + return total, err } diff --git a/execution/commitment/adaptive_pin.go b/execution/commitment/adaptive_pin.go index 674f3077e71..8549420e67b 100644 --- a/execution/commitment/adaptive_pin.go +++ b/execution/commitment/adaptive_pin.go @@ -399,5 +399,3 @@ func (c *AdaptivePinController) warnf(msg string, kv ...any) { c.logger.Warn(msg, kv...) } } - -var _ = context.Background // reserved for cancellation of in-flight preloads diff --git a/execution/commitment/branch_cache_test.go b/execution/commitment/branch_cache_test.go index b7bd4c8b2fd..11cdac4a4df 100644 --- a/execution/commitment/branch_cache_test.go +++ b/execution/commitment/branch_cache_test.go @@ -291,6 +291,7 @@ func TestBranchCache_Unwind_FrozenSurvives(t *testing.T) { // and that invalidating it doesn't evict real entries. func TestBranchCache_StateKeyNeverCached(t *testing.T) { c := NewBranchCache(100) + defer c.Close() c.Put(KeyCommitmentState, []byte("checkpoint"), 1, 1) _, _, ok := c.Get(KeyCommitmentState) @@ -340,6 +341,7 @@ func TestBranchCache_ShardedTailUnwindAcrossShards(t *testing.T) { // wrote it under resizeMu. Must be run under -race to be meaningful. func TestBranchCache_ConcurrentTailGrow(t *testing.T) { c := NewBranchCache(4096) // max >> 512 start, so the tail actually grows + defer c.Close() const ( workers = 8 diff --git a/execution/commitment/commitmentdb/commitment_context.go b/execution/commitment/commitmentdb/commitment_context.go index 9412d5ee5b7..d31f16ac163 100644 --- a/execution/commitment/commitmentdb/commitment_context.go +++ b/execution/commitment/commitmentdb/commitment_context.go @@ -44,9 +44,6 @@ type sd interface { // read), tagged with source. MergeMetrics(source kvmetrics.Source, wm *kvmetrics.DomainMetrics) StepSize() uint64 - // ProbeReadLayers samples sd.mem, parent.mem and tx-direct (MDBX) for one - // key — BranchCache divergence-detection probe. Read-only. - ProbeReadLayers(domain kv.Domain, tx kv.TemporalTx, key []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) // Metrics exposes the per-SD DomainMetrics so callers can read // per-domain (cache, db, file) read counters. Used by the @@ -220,8 +217,6 @@ func (sdc *SharedDomainsCommitmentContext) trieContext(tx kv.TemporalTx, blockNu stepSize: sdc.sharedDomains.StepSize(), txNum: txNum, blockNum: blockNum, - probeSd: sdc.sharedDomains, - probeTx: tx, traceW: sdc.traceW, } if sdc.stateReader != nil { @@ -899,10 +894,6 @@ type TrieContext struct { traceW io.Writer // nil = disabled; traces branch reads/writes (see [SDC] lines) stateReader StateReader localCollector *etl.Collector // per-goroutine collector for concurrent PutBranch - - // Diagnostics-only — both nil for read-only / test contexts. - probeSd sd - probeTx kv.TemporalTx } // NewTrieContextRo creates a read-only TrieContext suitable for TrieReader lookups. @@ -927,22 +918,6 @@ func (sdc *TrieContext) Branch(pref []byte) ([]byte, kv.Step, error) { return common.Copy(enc), step, nil } -// ProbeStateLayers samples sd.mem, parent.mem and tx-direct (MDBX) for one -// key — divergence diagnostics. Returns empty / not-ok when constructed -// without a probe-capable SharedDomains (e.g. NewTrieContextRo). -func (sdc *TrieContext) ProbeStateLayers(domain kv.Domain, key []byte) (mem, parentMem, mdbx []byte, memOk, parentOk bool) { - if sdc.probeSd == nil { - return - } - return sdc.probeSd.ProbeReadLayers(domain, sdc.probeTx, key) -} - -// SiteIdentity tags cache entries with the SD lineage that produced them so -// divergence diagnostics can tell parent-SD writes from fork-SD writes. -func (sdc *TrieContext) SiteIdentity() string { - return fmt.Sprintf("sd=%p", sdc.probeSd) -} - func (sdc *TrieContext) PutBranch(prefix []byte, data []byte, prevData []byte) error { if sdc.stateReader.WithHistory() { // do not store branches if explicitly operate on history return nil diff --git a/execution/commitment/hex_patricia_hashed.go b/execution/commitment/hex_patricia_hashed.go index 16ff4114253..bf0b16b43dd 100644 --- a/execution/commitment/hex_patricia_hashed.go +++ b/execution/commitment/hex_patricia_hashed.go @@ -954,7 +954,6 @@ func (hph *HexPatriciaHashed) witnessComputeCellHashWithStorage(cell *cell, dept } else { if !cell.loaded.storage() { hph.metrics.StorageLoad(cell.storageAddr[:cell.storageAddrLen]) - diskLoadStorage.Add(1) update, err := hph.storageFromCacheOrDB(cell.storageAddr[:cell.storageAddrLen]) if err != nil { return nil, storageRootHashIsSet, nil, err @@ -1040,7 +1039,6 @@ func (hph *HexPatriciaHashed) witnessComputeCellHashWithStorage(cell *cell, dept } // storage root update or extension update could invalidate older stateHash, so we need to reload state hph.metrics.AccountLoad(cell.accountAddr[:cell.accountAddrLen]) - diskLoadAccount.Add(1) update, err := hph.accountFromCacheOrDB(cell.accountAddr[:cell.accountAddrLen]) if err != nil { return nil, storageRootHashIsSet, storageRootHash[:], err @@ -1193,7 +1191,6 @@ func (hph *HexPatriciaHashed) computeCellHash(cell *cell, depth int16, buf []byt } // storage root update or extension update could invalidate older stateHash, so we need to reload state hph.metrics.AccountLoad(cell.accountAddr[:cell.accountAddrLen]) - diskLoadAccount.Add(1) update, err := hph.accountFromCacheOrDB(cell.accountAddr[:cell.accountAddrLen]) if err != nil { return nil, err @@ -1569,11 +1566,9 @@ func (hph *HexPatriciaHashed) needFolding(hashedKey []byte) bool { // Process-cumulative trie-compute counters feeding the KVReadLevelledMetrics // "skip ratio"/"reset ratio" Debug log at the end of ComputeCommitment. var ( - hadToLoad atomic.Uint64 - skippedLoad atomic.Uint64 - hadToReset atomic.Uint64 - diskLoadStorage atomic.Uint64 - diskLoadAccount atomic.Uint64 + hadToLoad atomic.Uint64 + skippedLoad atomic.Uint64 + hadToReset atomic.Uint64 ) var ( diff --git a/execution/commitment/preload_parallel.go b/execution/commitment/preload_parallel.go index d8a66e6f641..955871fc253 100644 --- a/execution/commitment/preload_parallel.go +++ b/execution/commitment/preload_parallel.go @@ -131,6 +131,10 @@ func (p *ContractTrunkPreloadParallel) Run( budgetHit = true return false } + // step=0: a storage-trunk branch resolved across merged files has no single + // source step, and the pinTxNum stamp already gives unwind coherence — the + // floor drops a preloaded pin before the cStep<=maxStep gate is consulted, + // so leaving step unset only keeps that gate trivially true for live pins. cache.PinEntry(pk.key, v, 0, p.pinTxNum) kc := make([]byte, len(pk.key)) copy(kc, pk.key) diff --git a/execution/commitment/warmuper.go b/execution/commitment/warmuper.go index d65a5a7dadf..abec0bb2f0c 100644 --- a/execution/commitment/warmuper.go +++ b/execution/commitment/warmuper.go @@ -31,20 +31,6 @@ import ( "github.com/erigontech/erigon/execution/commitment/nibbles" ) -// Warmer branch-read outcome counters. Hit: the branch read returned -// >= 4 bytes; Empty: returned nothing or unparseable. Used to size the -// value of bypassing the xorfilter in this call path. -var ( - warmerBranchHitCount atomic.Uint64 - warmerBranchEmptyCount atomic.Uint64 -) - -// WarmerBranchOutcomeStats returns process-cumulative counts. Snapshot -// before/after for per-block deltas. -func WarmerBranchOutcomeStats() (hit, empty uint64) { - return warmerBranchHitCount.Load(), warmerBranchEmptyCount.Load() -} - // TrieContextFactory creates new PatriciaContext instances for parallel warmup. type TrieContextFactory func() (PatriciaContext, func()) @@ -177,10 +163,8 @@ func (w *Warmuper) warmupKey(trieCtx PatriciaContext, hashedKey []byte, startDep // Branch data format: 2-byte touch map + 2-byte bitmap + per-child data if len(branchData) < 4 { - warmerBranchEmptyCount.Add(1) break } - warmerBranchHitCount.Add(1) if depth >= len(hashedKey) { break diff --git a/execution/state/state_object.go b/execution/state/state_object.go index c74bda57257..737b212c13e 100644 --- a/execution/state/state_object.go +++ b/execution/state/state_object.go @@ -416,7 +416,6 @@ func (so *stateObject) CodeTyped() (accounts.Code, error) { return c, nil } } - if dbg.TraceDomainIO || (dbg.TraceTransactionIO && (so.db.trace || dbg.TraceAccount(so.address.Handle()))) { so.db.stateReader.SetTrace(true, fmt.Sprintf("%d (%d.%d)", so.db.blockNum, so.db.txIndex, so.db.version)) }