From 758ff5f0474ca0fb29abdf119394532ea9953254 Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 9 Jul 2026 22:57:28 -0400 Subject: [PATCH 1/5] fullhistory: add per-CF tuning knobs to rocksdb.CFOptions Grow CFOptions with the memtable, compaction, and bloom knobs that were DB-wide fields on Tuning, and route the bloom filter through the per-CF override. Tuning now carries only the genuinely DB-wide resources shared across every CF of one store: block cache, WAL cap, background jobs, and open-file cap. This lets a facade scope a workload's tuning to the CF that has that workload instead of leaking it onto every CF in the shared per-chunk DB. --- .../fullhistory/storage/rocksdb/rocksdb.go | 99 ++++++++++--------- .../storage/rocksdb/rocksdb_test.go | 35 ++++--- .../fullhistory/storage/rocksdb/tuning.go | 77 ++++++++------- 3 files changed, 112 insertions(+), 99 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb.go b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb.go index 54545e6a5..bb1b6e3c8 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb.go @@ -586,31 +586,55 @@ func (s *Store) constructAndOpen() error { return nil } -// applyTuning splits configuration into wrapper-pinned values -// (every CF, unconditional), per-facade Tuning fields (applied -// only when non-zero), and per-CF overrides (applied last so they -// win over the pinned defaults). BloomFilterBitsPerKey == 0 is -// the documented "no bloom filter" sentinel. +// applyTuning splits configuration into wrapper-pinned values (every CF, +// unconditional), per-CF overrides (applied after the pinned defaults so +// they win), and DB-wide Tuning fields (applied to the shared Options). +// BloomFilterBitsPerKey == 0 is the documented "no bloom filter" sentinel. func (s *Store) applyTuning(opts *grocksdb.Options, cfNames []string, cfOpts []*grocksdb.Options) { t := s.cfg.Tuning for i, o := range cfOpts { applyPinnedCFOptions(o) - applyCFTuning(o, t) applyCFOverride(o, s.cfg.PerCFOptions[cfNames[i]]) } applyDBTuning(opts, t) s.applySharedTableOptions(cfNames, cfOpts, t) } -// applyCFOverride applies the per-CF Compression override. BlockSize -// is applied inside applySharedTableOptions when the BBTO is built. -func applyCFOverride(o *grocksdb.Options, override CFOptions) { +// applyCFOverride applies the per-CF memtable/compaction/compression knobs. +// Each is applied only when non-zero, leaving the pinned default otherwise. +// BlockSize and BloomFilterBitsPerKey are applied inside +// applySharedTableOptions when the BBTO is built. +func applyCFOverride(o *grocksdb.Options, c CFOptions) { // CompressionType is an int alias; NoCompression (the pinned // default) is 0. A zero-value override is therefore a no-op and // leaves the pinned NoCompression in place. Non-zero values // (Snappy, ZSTD, ...) replace it. - if override.Compression != grocksdb.NoCompression { - o.SetCompression(override.Compression) + if c.Compression != grocksdb.NoCompression { + o.SetCompression(c.Compression) + } + if c.WriteBufferMB > 0 { + o.SetWriteBufferSize(uint64(c.WriteBufferMB) << 20) + } + if c.MaxWriteBufferNumber > 0 { + o.SetMaxWriteBufferNumber(c.MaxWriteBufferNumber) + } + if c.Level0FileNumCompactionTrigger > 0 { + o.SetLevel0FileNumCompactionTrigger(c.Level0FileNumCompactionTrigger) + } + if c.Level0SlowdownWritesTrigger > 0 { + o.SetLevel0SlowdownWritesTrigger(c.Level0SlowdownWritesTrigger) + } + if c.Level0StopWritesTrigger > 0 { + o.SetLevel0StopWritesTrigger(c.Level0StopWritesTrigger) + } + if c.DisableAutoCompactions { + o.SetDisableAutoCompactions(true) + } + if c.TargetFileSizeMB > 0 { + o.SetTargetFileSizeBase(uint64(c.TargetFileSizeMB) << 20) + } + if c.MaxBytesForLevelBaseMB > 0 { + o.SetMaxBytesForLevelBase(uint64(c.MaxBytesForLevelBaseMB) << 20) } } @@ -625,33 +649,6 @@ func applyPinnedCFOptions(o *grocksdb.Options) { o.SetCompression(grocksdb.NoCompression) } -func applyCFTuning(o *grocksdb.Options, t Tuning) { - if t.WriteBufferMB > 0 { - o.SetWriteBufferSize(uint64(t.WriteBufferMB) << 20) - } - if t.MaxWriteBufferNumber > 0 { - o.SetMaxWriteBufferNumber(t.MaxWriteBufferNumber) - } - if t.Level0FileNumCompactionTrigger > 0 { - o.SetLevel0FileNumCompactionTrigger(t.Level0FileNumCompactionTrigger) - } - if t.Level0SlowdownWritesTrigger > 0 { - o.SetLevel0SlowdownWritesTrigger(t.Level0SlowdownWritesTrigger) - } - if t.Level0StopWritesTrigger > 0 { - o.SetLevel0StopWritesTrigger(t.Level0StopWritesTrigger) - } - if t.DisableAutoCompactions { - o.SetDisableAutoCompactions(true) - } - if t.TargetFileSizeMB > 0 { - o.SetTargetFileSizeBase(uint64(t.TargetFileSizeMB) << 20) - } - if t.MaxBytesForLevelBaseMB > 0 { - o.SetMaxBytesForLevelBase(uint64(t.MaxBytesForLevelBaseMB) << 20) - } -} - func applyDBTuning(opts *grocksdb.Options, t Tuning) { if t.MaxBackgroundJobs > 0 { opts.SetMaxBackgroundJobs(t.MaxBackgroundJobs) @@ -665,15 +662,19 @@ func applyDBTuning(opts *grocksdb.Options, t Tuning) { } // applySharedTableOptions builds one BBTO per CF, referencing the -// shared block cache (when set), a fresh per-CF bloom filter (when -// set), and the per-CF BlockSize override (when set). The Store -// retains every BBTO and the cache; Close destroys them after -// opts/cfOpts. +// DB-wide block cache (when set), a fresh per-CF bloom filter (when +// that CF sets BloomFilterBitsPerKey), and the per-CF BlockSize +// override (when set). The Store retains every BBTO and the cache; +// Close destroys them after opts/cfOpts. +// +// The block cache is DB-wide (one LRU shared across every CF from +// Tuning.BlockCacheMB); the bloom filter and block size are per-CF +// (from PerCFOptions), so only a CF that opts in pays for them. // -// A BBTO is installed on a CF iff any of the cache, a bloom filter, -// or that CF's BlockSize override is configured — preserving the -// previous behavior of leaving RocksDB's default BBTO untouched when -// no table-level knob is set. +// A BBTO is installed on a CF iff the shared cache, that CF's bloom +// filter, or that CF's BlockSize override is configured — preserving +// the previous behavior of leaving RocksDB's default BBTO untouched +// when no table-level knob is set. // // The bloom filter is built per CF because SetFilterPolicy MOVES the // policy into the BBTO (it nils the source pointer), so a single @@ -685,15 +686,15 @@ func (s *Store) applySharedTableOptions(cfNames []string, cfOpts []*grocksdb.Opt } for i, o := range cfOpts { override := s.cfg.PerCFOptions[cfNames[i]] - if s.cache == nil && t.BloomFilterBitsPerKey == 0 && override.BlockSize == 0 { + if s.cache == nil && override.BloomFilterBitsPerKey == 0 && override.BlockSize == 0 { continue } bbto := grocksdb.NewDefaultBlockBasedTableOptions() if s.cache != nil { bbto.SetBlockCache(s.cache) } - if t.BloomFilterBitsPerKey > 0 { - bbto.SetFilterPolicy(grocksdb.NewBloomFilter(float64(t.BloomFilterBitsPerKey))) + if override.BloomFilterBitsPerKey > 0 { + bbto.SetFilterPolicy(grocksdb.NewBloomFilter(float64(override.BloomFilterBitsPerKey))) } if override.BlockSize > 0 { bbto.SetBlockSize(override.BlockSize) diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb_test.go b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb_test.go index 8af932b10..c80564983 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb_test.go @@ -133,13 +133,14 @@ func TestNew_TwoStoresSamePathCollide(t *testing.T) { func TestStore_ConstructAndOpenFailureFreesCacheAndBBTOs(t *testing.T) { dir := t.TempDir() - tuning := Tuning{BlockCacheMB: 4, BloomFilterBitsPerKey: 10} + tuning := Tuning{BlockCacheMB: 4} + perCF := map[string]CFOptions{defaultCFName: {BloomFilterBitsPerKey: 10}} - holder, err := New(Config{Path: dir, Logger: silentLogger(), Tuning: tuning}) + holder, err := New(Config{Path: dir, Logger: silentLogger(), Tuning: tuning, PerCFOptions: perCF}) require.NoError(t, err) t.Cleanup(func() { _ = holder.Close() }) - collider := &Store{cfg: Config{Path: dir, Logger: silentLogger(), Tuning: tuning}} + collider := &Store{cfg: Config{Path: dir, Logger: silentLogger(), Tuning: tuning, PerCFOptions: perCF}} require.Error(t, collider.constructAndOpen()) assert.Nil(t, collider.cache) @@ -665,18 +666,22 @@ func TestStore_TuningRoundTrip(t *testing.T) { Path: t.TempDir(), Logger: newTestLogger(&buf), Tuning: Tuning{ - WriteBufferMB: 8, - MaxWriteBufferNumber: 2, - Level0FileNumCompactionTrigger: 4, - Level0SlowdownWritesTrigger: 20, - Level0StopWritesTrigger: 36, - TargetFileSizeMB: 16, - MaxBytesForLevelBaseMB: 64, - MaxBackgroundJobs: 2, - MaxOpenFiles: 500, - BlockCacheMB: 4, - BloomFilterBitsPerKey: 10, - MaxTotalWalSizeMB: 16, + MaxBackgroundJobs: 2, + MaxOpenFiles: 500, + BlockCacheMB: 4, + MaxTotalWalSizeMB: 16, + }, + PerCFOptions: map[string]CFOptions{ + defaultCFName: { + WriteBufferMB: 8, + MaxWriteBufferNumber: 2, + Level0FileNumCompactionTrigger: 4, + Level0SlowdownWritesTrigger: 20, + Level0StopWritesTrigger: 36, + TargetFileSizeMB: 16, + MaxBytesForLevelBaseMB: 64, + BloomFilterBitsPerKey: 10, + }, }, }) require.NoError(t, err) diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/tuning.go b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/tuning.go index 12f5df1d8..f41a2e305 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/tuning.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/tuning.go @@ -2,12 +2,15 @@ package rocksdb import "github.com/linxGnu/grocksdb" -// CFOptions — per-CF overrides applied after the shared pinned -// defaults and global Tuning. Zero means "inherit the pinned -// default" (NoCompression for Compression, RocksDB's BBTO default -// block size for BlockSize). Use to opt specific CFs into -// compression or non-default block sizes while leaving the rest at -// the wrapper-pinned defaults. +// CFOptions — per-CF overrides applied after the shared pinned defaults. +// Zero means "inherit the pinned default" (NoCompression for Compression, +// RocksDB's BBTO default block size for BlockSize, no bloom filter, and +// grocksdb's memtable/compaction defaults for the rest). Each field a CF +// leaves zero rides on the wrapper-pinned defaults; a facade opts a specific +// CF into a knob by setting it here. The knobs are per-CF because each hot +// data type's workload differs (txhash is write-once/point-lookup; ledgers is +// never probed for missing keys; events holds compressible XDR) and one CF's +// tuning must not leak onto the others. type CFOptions struct { // Compression overrides the CF's compression type. The wrapper's // pinned default is NoCompression; CFs that hold compressible @@ -25,65 +28,69 @@ type CFOptions struct { // block holding the target key; larger blocks waste I/O per // cache miss. BlockSize int -} -// Tuning — per-store RocksDB knobs. Zero means "leave grocksdb's -// default alone" (wrapper skips the setter). BloomFilterBitsPerKey == 0 -// is the documented "install no bloom filter" sentinel. -// -// Wrapper-pinned values not exposed here (applied to every facade): -// MinWriteBufferNumberToMerge=1, CompactionStyle=Level, -// TargetFileSizeMultiplier=1, MaxBytesForLevelMultiplier=10, -// Compression=None, WAL=on, per-write Sync=on. -// Per-CF overrides for Compression and BlockSize live in -// Config.PerCFOptions. -type Tuning struct { - // WriteBufferMB sizes the active memtable per CF, in MB. + // WriteBufferMB sizes the active memtable for this CF, in MB. Zero + // leaves grocksdb's default. WriteBufferMB int // MaxWriteBufferNumber caps the active + immutable memtable count - // per CF before writes back-pressure. + // for this CF before writes back-pressure. Zero leaves the default. MaxWriteBufferNumber int // Level0FileNumCompactionTrigger — L0 file count that starts - // an L0→L1 compaction. + // an L0→L1 compaction. Zero leaves the default. Level0FileNumCompactionTrigger int // Level0SlowdownWritesTrigger — L0 file count that slows writes. + // Zero leaves the default. Level0SlowdownWritesTrigger int // Level0StopWritesTrigger — L0 file count that stalls writes - // entirely. + // entirely. Zero leaves the default. Level0StopWritesTrigger int - // DisableAutoCompactions turns automatic compaction off per CF — - // for write-once, point-lookup stores where compaction would - // rewrite the same data with no reordering benefit. + // DisableAutoCompactions turns automatic compaction off for this CF — + // for write-once, point-lookup CFs where compaction would rewrite the + // same data with no reordering benefit. DisableAutoCompactions bool // TargetFileSizeMB — size at which compaction produces new SSTs. + // Zero leaves the default. TargetFileSizeMB int // MaxBytesForLevelBaseMB — byte budget for level 1; each later - // level is this × MaxBytesForLevelMultiplier (10, pinned). + // level is this × MaxBytesForLevelMultiplier (10, pinned). Zero + // leaves the default. MaxBytesForLevelBaseMB int + // BloomFilterBitsPerKey installs this CF's bloom filter. 0 = no + // filter (the default) — right for a CF that is never probed for + // keys it may not hold. Positive values install one with that many + // bits; typical: 10 (~1% false positive), 12 (~0.4%). + BloomFilterBitsPerKey int +} + +// Tuning — DB-wide RocksDB knobs shared across every CF of one store. Zero +// means "leave grocksdb's default alone" (wrapper skips the setter). Per-CF +// knobs (memtables, compaction, bloom, block size, compression) live in +// Config.PerCFOptions, not here. +// +// Wrapper-pinned per-CF values not exposed anywhere (applied to every CF): +// MinWriteBufferNumberToMerge=1, CompactionStyle=Level, +// TargetFileSizeMultiplier=1, MaxBytesForLevelMultiplier=10, +// Compression=None, WAL=on, per-write Sync=on. +type Tuning struct { // MaxBackgroundJobs caps background threads for compactions - // and flushes combined. Orthogonal to DisableAutoCompactions: - // this rate-limits work, that turns compaction off entirely. + // and flushes combined, across the whole DB. MaxBackgroundJobs int - // MaxOpenFiles caps concurrent open SST files. + // MaxOpenFiles caps concurrent open SST files, across the whole DB. MaxOpenFiles int - // BlockCacheMB sizes the shared LRU block cache, per store. + // BlockCacheMB sizes the LRU block cache shared across every CF in + // the store. BlockCacheMB int - // BloomFilterBitsPerKey — per-CF bloom filter. 0 = no filter - // installed; positive values install one with that many bits. - // Typical: 10 (~1% false positive), 12 (~0.4%). - BloomFilterBitsPerKey int - // MaxTotalWalSizeMB caps total live WAL size. Crash-recovery // replay scales with this cap; graceful Close drains the // memtable so this only bounds ungraceful shutdowns. From 6a6a32e94b5b8c7b4f2f030a6be7fa49c902cdcf Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Thu, 9 Jul 2026 22:57:36 -0400 Subject: [PATCH 2/5] fullhistory: scope txhash tuning to the txhash CF Split txhash's config into the DB-wide Tuning it always set (block cache, WAL cap, background jobs, open files) and a new CFOptions carrying its per-CF workload knobs (bloom, write buffers, compaction triggers). The hotchunk opener merges the per-CF options from every facade so those knobs land on the txhash CF alone. Restores each store's pre-unification standalone tuning: the ledgers CF runs with no bloom filter (it is never probed for missing keys) and default memtables/compaction; the events CFs keep their ZSTD and block-size overrides; the txhash CF keeps its full calibration. The DB-wide resources stay shared, sized to what the three standalone instances used together (only txhash set them). --- .../storage/stores/hotchunk/hotchunk.go | 17 ++-- .../storage/stores/hotchunk/hotchunk_test.go | 33 +++++++ .../storage/stores/txhash/hot_store.go | 94 +++++++++++-------- 3 files changed, 99 insertions(+), 45 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk.go index 63f1a2ca6..1e9ae1346 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk.go @@ -12,6 +12,7 @@ import ( "context" "fmt" "iter" + "maps" "slices" "time" @@ -54,18 +55,22 @@ func ColumnFamilies() []string { return slices.Concat(ledger.CFNames(), eventstore.CFNames(), txhash.CFNames()) } -// config builds the shared store's rocksdb.Config: events' per-CF options (ZSTD -// on DataCF, tuned block sizes) plus the txhash workload's Tuning. Tuning's -// per-CF fields apply to every CF — a benign over-application (ledger/events CFs -// just gain a bloom + larger write buffer); the per-CF overrides keep events -// distinct. +// config builds the shared store's rocksdb.Config: the DB-wide Tuning +// (block cache, WAL cap, background jobs) from txhash — the only facade +// that historically set them — plus the per-CF options merged from every +// facade. txhash's per-CF knobs (bloom, write buffers, compaction) land on +// the txhash CF alone; events' overrides (ZSTD, block sizes) on its three +// CFs; the ledgers CF rides on RocksDB defaults with no bloom, matching each +// store's pre-unification standalone tuning. func config(path string, logger *supportlog.Entry, readOnly, mustExist bool) rocksdb.Config { + perCF := eventstore.CFOptions() + maps.Copy(perCF, txhash.CFOptions()) return rocksdb.Config{ Path: path, ColumnFamilies: ColumnFamilies(), Logger: logger, Tuning: txhash.Tuning(), - PerCFOptions: eventstore.CFOptions(), + PerCFOptions: perCF, ReadOnly: readOnly, MustExist: mustExist, } diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk_test.go b/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk_test.go index cffa7b4c6..539d1a4e1 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/linxGnu/grocksdb" "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -51,6 +52,38 @@ func assertWriteItems(t *testing.T, rep LedgerReport, txhash int) { assert.Equal(t, 1, rep.Phases[PhaseEvents].Items, "events items") } +// TestConfig_PerCFOptionRouting asserts the txhash Tuning knobs reach ONLY the +// txhash CF: the ledger CF gets no bloom (it is never probed for missing keys) +// and the events CFs keep their own compression/block-size overrides. Asserted +// at the config-construction seam because grocksdb options can't be read back +// off a live DB. +func TestConfig_PerCFOptionRouting(t *testing.T) { + perCF := config(t.TempDir(), silentLogger(), false, false).PerCFOptions + + txCF := txhash.CFNames()[0] + assert.Equal(t, 12, perCF[txCF].BloomFilterBitsPerKey, "txhash CF keeps its bloom") + assert.Equal(t, 64, perCF[txCF].WriteBufferMB, "txhash CF keeps its write buffer") + assert.True(t, perCF[txCF].DisableAutoCompactions, "txhash CF keeps compaction off") + + ledgerCF := ledger.CFNames()[0] + assert.Zero(t, perCF[ledgerCF].BloomFilterBitsPerKey, "ledger CF gets no bloom") + assert.Zero(t, perCF[ledgerCF].WriteBufferMB, "ledger CF rides on RocksDB defaults") + assert.False(t, perCF[ledgerCF].DisableAutoCompactions, "ledger CF keeps auto-compaction") + + assert.Equal(t, grocksdb.ZSTDCompression, perCF[eventstore.DataCF].Compression, + "events data CF keeps its ZSTD override") + assert.NotZero(t, perCF[eventstore.DataCF].BlockSize, "events data CF keeps its block-size override") + assert.Zero(t, perCF[eventstore.DataCF].BloomFilterBitsPerKey, "events data CF gets no bloom") +} + +func TestConfig_DBWideTuningStaysShared(t *testing.T) { + tuning := config(t.TempDir(), silentLogger(), false, false).Tuning + assert.Equal(t, 512, tuning.BlockCacheMB) + assert.Equal(t, 1024, tuning.MaxTotalWalSizeMB) + assert.Equal(t, 8, tuning.MaxBackgroundJobs) + assert.Equal(t, 10_000, tuning.MaxOpenFiles) +} + func TestOpen_ValidatesInputs(t *testing.T) { _, err := Open("", chunk.ID(0), silentLogger()) require.ErrorIs(t, err, stores.ErrInvalidConfig) diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go index 5d1312862..ffa849280 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go @@ -44,42 +44,14 @@ func NewWithStore(store *rocksdb.Store) *HotStore { // the hotchunk shared-DB opener can register it alongside the other CFs. func CFNames() []string { return []string{txhashCF} } -// Tuning returns this facade's RocksDB tuning, applied to the shared per-chunk -// DB by the hotchunk opener. The hot txhash workload is write-once / -// point-lookup; the cross-knob interactions below are non-obvious enough that -// they get an explicit per-stanza rationale. The other facades ride on RocksDB -// defaults by contrast — only this workload earned the calibration. +// Tuning returns the DB-wide RocksDB knobs the shared per-chunk DB adopts — +// resources that were per-instance before the hot stores collapsed into one +// multi-CF DB and are now shared across every CF. The txhash instance was the +// only one that set them; the ledger and events instances rode on RocksDB +// defaults, so keeping txhash's values is the faithful consolidation of the +// three instances' combined footprint. func Tuning() rocksdb.Tuning { return rocksdb.Tuning{ - // 64 MB memtable so one flush produces one ~64 MB SST under - // uniform writes. - WriteBufferMB: 64, - MaxWriteBufferNumber: 2, - - // L0 triggers pinned high + DisableAutoCompactions=true: - // compaction would re-write the same data with no reordering - // benefit (txhash is write-once, random-key, point-lookup). - // The L0 999s match DisableAutoCompactions so even if a future - // flush somehow exceeded the trigger, the engine still - // wouldn't try to compact. NOTE: DisableAutoCompactions and - // MaxBackgroundJobs are orthogonal — the former turns - // compaction off entirely, the latter only caps the thread - // budget for background work. - Level0FileNumCompactionTrigger: 999, - Level0SlowdownWritesTrigger: 999, - Level0StopWritesTrigger: 999, - DisableAutoCompactions: true, - - // 64 MB target file matches WriteBufferMB so one memtable - // flush produces one ~64 MB SST — fewer bloom checks per - // query at no-compaction scale. - // MaxBytesForLevelBaseMB is set explicitly even though it's - // irrelevant under DisableAutoCompactions (compaction never - // promotes past L0); explicit > implicit so a future reader - // doesn't have to derive that it's a no-op. - TargetFileSizeMB: 64, - MaxBytesForLevelBaseMB: 256, - // Background-job budget for the periodic memtable flushes. MaxBackgroundJobs: 8, MaxOpenFiles: 10_000, @@ -87,11 +59,7 @@ func Tuning() rocksdb.Tuning { // 512 MB block cache — bloom-filter blocks are the hot // working set; the cache needs to hold recently-touched // bloom blocks at scale. - // 12 bits/key bloom (~0.4% false-positive) is tighter than - // the standard 10 bits/key because every false positive at - // no-compaction SST count costs a disk seek across many SSTs. - BlockCacheMB: 512, - BloomFilterBitsPerKey: 12, + BlockCacheMB: 512, // 1 GB WAL cap. Graceful Close auto-Flushes (see // rocksdb.Store.Close), so this cap only bounds ungraceful-shutdown @@ -100,6 +68,54 @@ func Tuning() rocksdb.Tuning { } } +// CFOptions returns the per-CF knobs the hotchunk opener installs on the +// txhash CF alone (merged into the shared DB's PerCFOptions). The hot txhash +// workload is write-once / point-lookup; the cross-knob interactions below are +// non-obvious enough that they get an explicit per-stanza rationale. The other +// CFs ride on RocksDB defaults by contrast — only this workload earned the +// calibration, and scoping it here keeps it off the ledger and events CFs. +func CFOptions() map[string]rocksdb.CFOptions { + return map[string]rocksdb.CFOptions{ + txhashCF: { + // 64 MB memtable so one flush produces one ~64 MB SST under + // uniform writes. + WriteBufferMB: 64, + MaxWriteBufferNumber: 2, + + // L0 triggers pinned high + DisableAutoCompactions=true: + // compaction would re-write the same data with no reordering + // benefit (txhash is write-once, random-key, point-lookup). + // The L0 999s match DisableAutoCompactions so even if a future + // flush somehow exceeded the trigger, the engine still + // wouldn't try to compact. NOTE: DisableAutoCompactions and + // Tuning.MaxBackgroundJobs are orthogonal — the former turns + // compaction off entirely, the latter only caps the thread + // budget for background work. + Level0FileNumCompactionTrigger: 999, + Level0SlowdownWritesTrigger: 999, + Level0StopWritesTrigger: 999, + DisableAutoCompactions: true, + + // 64 MB target file matches WriteBufferMB so one memtable + // flush produces one ~64 MB SST — fewer bloom checks per + // query at no-compaction scale. + // MaxBytesForLevelBaseMB is set explicitly even though it's + // irrelevant under DisableAutoCompactions (compaction never + // promotes past L0); explicit > implicit so a future reader + // doesn't have to derive that it's a no-op. + TargetFileSizeMB: 64, + MaxBytesForLevelBaseMB: 256, + + // 12 bits/key bloom (~0.4% false-positive) is tighter than + // the standard 10 bits/key because every false positive at + // no-compaction SST count costs a disk seek across many SSTs. + // Scoped to this CF: it is the only one probed for keys it may + // not hold, so the ledger and events CFs install no bloom. + BloomFilterBitsPerKey: 12, + }, + } +} + // AddEntriesToBatch queues each (txhash → ledgerSeq) Put into b on the txhash // CF — the building block hotchunk uses to fold the tx-hash writes into the one // shared per-ledger WriteBatch (decision (a)). Does not commit (caller owns the From af55246ec4b5614a7cf5129c946c73330dc4cbcf Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Mon, 13 Jul 2026 16:28:07 -0400 Subject: [PATCH 3/5] fullhistory: tighten the per-CF tuning comments --- .../fullhistory/storage/rocksdb/rocksdb.go | 4 --- .../fullhistory/storage/rocksdb/tuning.go | 33 +++++++------------ .../storage/stores/hotchunk/hotchunk.go | 11 +++---- .../storage/stores/txhash/hot_store.go | 22 +++++-------- 4 files changed, 23 insertions(+), 47 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb.go b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb.go index bb1b6e3c8..73caf58bd 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb.go @@ -667,10 +667,6 @@ func applyDBTuning(opts *grocksdb.Options, t Tuning) { // override (when set). The Store retains every BBTO and the cache; // Close destroys them after opts/cfOpts. // -// The block cache is DB-wide (one LRU shared across every CF from -// Tuning.BlockCacheMB); the bloom filter and block size are per-CF -// (from PerCFOptions), so only a CF that opts in pays for them. -// // A BBTO is installed on a CF iff the shared cache, that CF's bloom // filter, or that CF's BlockSize override is configured — preserving // the previous behavior of leaving RocksDB's default BBTO untouched diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/tuning.go b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/tuning.go index f41a2e305..3b80d0c19 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/tuning.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/tuning.go @@ -3,14 +3,10 @@ package rocksdb import "github.com/linxGnu/grocksdb" // CFOptions — per-CF overrides applied after the shared pinned defaults. -// Zero means "inherit the pinned default" (NoCompression for Compression, -// RocksDB's BBTO default block size for BlockSize, no bloom filter, and -// grocksdb's memtable/compaction defaults for the rest). Each field a CF -// leaves zero rides on the wrapper-pinned defaults; a facade opts a specific -// CF into a knob by setting it here. The knobs are per-CF because each hot -// data type's workload differs (txhash is write-once/point-lookup; ledgers is -// never probed for missing keys; events holds compressible XDR) and one CF's -// tuning must not leak onto the others. +// Zero means "inherit the default" for every field (NoCompression, RocksDB's +// BBTO block size, no bloom, grocksdb's memtable/compaction defaults). The +// knobs are per-CF because each hot workload differs — one CF's tuning must +// not leak onto the others. type CFOptions struct { // Compression overrides the CF's compression type. The wrapper's // pinned default is NoCompression; CFs that hold compressible @@ -29,24 +25,21 @@ type CFOptions struct { // cache miss. BlockSize int - // WriteBufferMB sizes the active memtable for this CF, in MB. Zero - // leaves grocksdb's default. + // WriteBufferMB sizes the active memtable for this CF, in MB. WriteBufferMB int // MaxWriteBufferNumber caps the active + immutable memtable count - // for this CF before writes back-pressure. Zero leaves the default. + // for this CF before writes back-pressure. MaxWriteBufferNumber int // Level0FileNumCompactionTrigger — L0 file count that starts - // an L0→L1 compaction. Zero leaves the default. + // an L0→L1 compaction. Level0FileNumCompactionTrigger int // Level0SlowdownWritesTrigger — L0 file count that slows writes. - // Zero leaves the default. Level0SlowdownWritesTrigger int - // Level0StopWritesTrigger — L0 file count that stalls writes - // entirely. Zero leaves the default. + // Level0StopWritesTrigger — L0 file count that stalls writes entirely. Level0StopWritesTrigger int // DisableAutoCompactions turns automatic compaction off for this CF — @@ -55,18 +48,14 @@ type CFOptions struct { DisableAutoCompactions bool // TargetFileSizeMB — size at which compaction produces new SSTs. - // Zero leaves the default. TargetFileSizeMB int // MaxBytesForLevelBaseMB — byte budget for level 1; each later - // level is this × MaxBytesForLevelMultiplier (10, pinned). Zero - // leaves the default. + // level is this × MaxBytesForLevelMultiplier (10, pinned). MaxBytesForLevelBaseMB int - // BloomFilterBitsPerKey installs this CF's bloom filter. 0 = no - // filter (the default) — right for a CF that is never probed for - // keys it may not hold. Positive values install one with that many - // bits; typical: 10 (~1% false positive), 12 (~0.4%). + // BloomFilterBitsPerKey installs this CF's bloom filter; 0 = none — + // right for a CF never probed for keys it may not hold. BloomFilterBitsPerKey int } diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk.go index 1e9ae1346..6c3d4189a 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk.go @@ -55,13 +55,10 @@ func ColumnFamilies() []string { return slices.Concat(ledger.CFNames(), eventstore.CFNames(), txhash.CFNames()) } -// config builds the shared store's rocksdb.Config: the DB-wide Tuning -// (block cache, WAL cap, background jobs) from txhash — the only facade -// that historically set them — plus the per-CF options merged from every -// facade. txhash's per-CF knobs (bloom, write buffers, compaction) land on -// the txhash CF alone; events' overrides (ZSTD, block sizes) on its three -// CFs; the ledgers CF rides on RocksDB defaults with no bloom, matching each -// store's pre-unification standalone tuning. +// config builds the shared store's rocksdb.Config: DB-wide Tuning from +// txhash (the only facade that historically set it) plus the per-CF options +// merged from every facade — each CF keeps its pre-unification standalone +// tuning; the ledgers CF rides on RocksDB defaults. func config(path string, logger *supportlog.Entry, readOnly, mustExist bool) rocksdb.Config { perCF := eventstore.CFOptions() maps.Copy(perCF, txhash.CFOptions()) diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go index ffa849280..960de76b3 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go @@ -44,12 +44,10 @@ func NewWithStore(store *rocksdb.Store) *HotStore { // the hotchunk shared-DB opener can register it alongside the other CFs. func CFNames() []string { return []string{txhashCF} } -// Tuning returns the DB-wide RocksDB knobs the shared per-chunk DB adopts — -// resources that were per-instance before the hot stores collapsed into one -// multi-CF DB and are now shared across every CF. The txhash instance was the -// only one that set them; the ledger and events instances rode on RocksDB -// defaults, so keeping txhash's values is the faithful consolidation of the -// three instances' combined footprint. +// Tuning returns the DB-wide knobs the shared per-chunk DB adopts. The +// standalone txhash instance was the only one that set them (ledger and +// events rode on defaults), so its values are the pre-unification +// instances' combined footprint. func Tuning() rocksdb.Tuning { return rocksdb.Tuning{ // Background-job budget for the periodic memtable flushes. @@ -68,12 +66,9 @@ func Tuning() rocksdb.Tuning { } } -// CFOptions returns the per-CF knobs the hotchunk opener installs on the -// txhash CF alone (merged into the shared DB's PerCFOptions). The hot txhash -// workload is write-once / point-lookup; the cross-knob interactions below are -// non-obvious enough that they get an explicit per-stanza rationale. The other -// CFs ride on RocksDB defaults by contrast — only this workload earned the -// calibration, and scoping it here keeps it off the ledger and events CFs. +// CFOptions returns the calibration the hotchunk opener installs on the +// txhash CF alone. The workload is write-once / point-lookup; only it +// earned this tuning — the other CFs ride on RocksDB defaults. func CFOptions() map[string]rocksdb.CFOptions { return map[string]rocksdb.CFOptions{ txhashCF: { @@ -109,8 +104,7 @@ func CFOptions() map[string]rocksdb.CFOptions { // 12 bits/key bloom (~0.4% false-positive) is tighter than // the standard 10 bits/key because every false positive at // no-compaction SST count costs a disk seek across many SSTs. - // Scoped to this CF: it is the only one probed for keys it may - // not hold, so the ledger and events CFs install no bloom. + // Only this CF is probed for keys it may not hold. BloomFilterBitsPerKey: 12, }, } From 4e0c912e1437fe9c931033f097b5ddf213c2c02b Mon Sep 17 00:00:00 2001 From: "openai-code-agent[bot]" <242516109+Codex@users.noreply.github.com> Date: Mon, 13 Jul 2026 20:37:26 +0000 Subject: [PATCH 4/5] fullhistory: remove redundant max-bytes-per-level CF knob Co-authored-by: chowbao <20801401+chowbao@users.noreply.github.com> --- .../internal/fullhistory/storage/rocksdb/rocksdb.go | 3 --- .../internal/fullhistory/storage/rocksdb/rocksdb_test.go | 1 - .../internal/fullhistory/storage/rocksdb/tuning.go | 4 ---- .../fullhistory/storage/stores/txhash/hot_store.go | 7 +------ 4 files changed, 1 insertion(+), 14 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb.go b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb.go index 73caf58bd..a214109ec 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb.go @@ -633,9 +633,6 @@ func applyCFOverride(o *grocksdb.Options, c CFOptions) { if c.TargetFileSizeMB > 0 { o.SetTargetFileSizeBase(uint64(c.TargetFileSizeMB) << 20) } - if c.MaxBytesForLevelBaseMB > 0 { - o.SetMaxBytesForLevelBase(uint64(c.MaxBytesForLevelBaseMB) << 20) - } } // applyPinnedCFOptions sets the per-CF values that hold for every diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb_test.go b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb_test.go index c80564983..c5357d044 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/rocksdb_test.go @@ -679,7 +679,6 @@ func TestStore_TuningRoundTrip(t *testing.T) { Level0SlowdownWritesTrigger: 20, Level0StopWritesTrigger: 36, TargetFileSizeMB: 16, - MaxBytesForLevelBaseMB: 64, BloomFilterBitsPerKey: 10, }, }, diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/tuning.go b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/tuning.go index 3b80d0c19..111bae826 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/tuning.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/rocksdb/tuning.go @@ -50,10 +50,6 @@ type CFOptions struct { // TargetFileSizeMB — size at which compaction produces new SSTs. TargetFileSizeMB int - // MaxBytesForLevelBaseMB — byte budget for level 1; each later - // level is this × MaxBytesForLevelMultiplier (10, pinned). - MaxBytesForLevelBaseMB int - // BloomFilterBitsPerKey installs this CF's bloom filter; 0 = none — // right for a CF never probed for keys it may not hold. BloomFilterBitsPerKey int diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go index 960de76b3..db0c1cfda 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go @@ -94,12 +94,7 @@ func CFOptions() map[string]rocksdb.CFOptions { // 64 MB target file matches WriteBufferMB so one memtable // flush produces one ~64 MB SST — fewer bloom checks per // query at no-compaction scale. - // MaxBytesForLevelBaseMB is set explicitly even though it's - // irrelevant under DisableAutoCompactions (compaction never - // promotes past L0); explicit > implicit so a future reader - // doesn't have to derive that it's a no-op. - TargetFileSizeMB: 64, - MaxBytesForLevelBaseMB: 256, + TargetFileSizeMB: 64, // 12 bits/key bloom (~0.4% false-positive) is tighter than // the standard 10 bits/key because every false positive at From 5b018a08105a13a033f3b857b3d705276f68c13b Mon Sep 17 00:00:00 2001 From: Simon Chow Date: Mon, 13 Jul 2026 16:58:53 -0400 Subject: [PATCH 5/5] fullhistory: move the DB-wide tuning into hotchunk, the DB owner --- .../storage/stores/hotchunk/hotchunk.go | 33 ++++++++++++++++--- .../storage/stores/hotchunk/hotchunk_test.go | 2 +- .../storage/stores/txhash/hot_store.go | 22 ------------- .../storage/stores/txhash/hot_store_test.go | 2 +- 4 files changed, 30 insertions(+), 29 deletions(-) diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk.go b/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk.go index 6c3d4189a..cbbd394c1 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk.go @@ -55,10 +55,33 @@ func ColumnFamilies() []string { return slices.Concat(ledger.CFNames(), eventstore.CFNames(), txhash.CFNames()) } -// config builds the shared store's rocksdb.Config: DB-wide Tuning from -// txhash (the only facade that historically set it) plus the per-CF options -// merged from every facade — each CF keeps its pre-unification standalone -// tuning; the ledgers CF rides on RocksDB defaults. +// dbTuning is the DB-wide half of the shared store's configuration, owned +// here because hotchunk owns the DB (each facade only configures its own +// CFs). The values originate from the standalone txhash store's calibration +// — the only pre-unification instance that set them. +func dbTuning() rocksdb.Tuning { + return rocksdb.Tuning{ + // Background-job budget for memtable flushes and the + // ledger/events compactions. + MaxBackgroundJobs: 8, + MaxOpenFiles: 10_000, + + // 512 MB block cache — txhash bloom-filter blocks are the hot + // working set; the cache needs to hold recently-touched bloom + // blocks at scale. + BlockCacheMB: 512, + + // 1 GB WAL cap. Graceful Close auto-Flushes (see + // rocksdb.Store.Close), so this cap only bounds + // ungraceful-shutdown recovery (kernel panic, power loss, OOM + // kill). + MaxTotalWalSizeMB: 1024, + } +} + +// config builds the shared store's rocksdb.Config: the DB-wide dbTuning plus +// the per-CF options merged from every facade — each CF keeps its +// pre-unification standalone tuning; the ledgers CF rides on RocksDB defaults. func config(path string, logger *supportlog.Entry, readOnly, mustExist bool) rocksdb.Config { perCF := eventstore.CFOptions() maps.Copy(perCF, txhash.CFOptions()) @@ -66,7 +89,7 @@ func config(path string, logger *supportlog.Entry, readOnly, mustExist bool) roc Path: path, ColumnFamilies: ColumnFamilies(), Logger: logger, - Tuning: txhash.Tuning(), + Tuning: dbTuning(), PerCFOptions: perCF, ReadOnly: readOnly, MustExist: mustExist, diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk_test.go b/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk_test.go index 539d1a4e1..79b2692b3 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/stores/hotchunk/hotchunk_test.go @@ -52,7 +52,7 @@ func assertWriteItems(t *testing.T, rep LedgerReport, txhash int) { assert.Equal(t, 1, rep.Phases[PhaseEvents].Items, "events items") } -// TestConfig_PerCFOptionRouting asserts the txhash Tuning knobs reach ONLY the +// TestConfig_PerCFOptionRouting asserts the txhash calibration reaches ONLY the // txhash CF: the ledger CF gets no bloom (it is never probed for missing keys) // and the events CFs keep their own compression/block-size overrides. Asserted // at the config-construction seam because grocksdb options can't be read back diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go b/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go index db0c1cfda..c1a3c8762 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store.go @@ -44,28 +44,6 @@ func NewWithStore(store *rocksdb.Store) *HotStore { // the hotchunk shared-DB opener can register it alongside the other CFs. func CFNames() []string { return []string{txhashCF} } -// Tuning returns the DB-wide knobs the shared per-chunk DB adopts. The -// standalone txhash instance was the only one that set them (ledger and -// events rode on defaults), so its values are the pre-unification -// instances' combined footprint. -func Tuning() rocksdb.Tuning { - return rocksdb.Tuning{ - // Background-job budget for the periodic memtable flushes. - MaxBackgroundJobs: 8, - MaxOpenFiles: 10_000, - - // 512 MB block cache — bloom-filter blocks are the hot - // working set; the cache needs to hold recently-touched - // bloom blocks at scale. - BlockCacheMB: 512, - - // 1 GB WAL cap. Graceful Close auto-Flushes (see - // rocksdb.Store.Close), so this cap only bounds ungraceful-shutdown - // recovery (kernel panic, power loss, OOM kill). - MaxTotalWalSizeMB: 1024, - } -} - // CFOptions returns the calibration the hotchunk opener installs on the // txhash CF alone. The workload is write-once / point-lookup; only it // earned this tuning — the other CFs ride on RocksDB defaults. diff --git a/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store_test.go b/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store_test.go index f2318a205..3f410694c 100644 --- a/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store_test.go +++ b/cmd/stellar-rpc/internal/fullhistory/storage/stores/txhash/hot_store_test.go @@ -49,7 +49,7 @@ func openTestHotStoreAt(t *testing.T, path string) (*HotStore, *rocksdb.Store) { Path: path, ColumnFamilies: CFNames(), Logger: silentLogger(), - Tuning: Tuning(), + PerCFOptions: CFOptions(), }) require.NoError(t, err) t.Cleanup(func() { _ = store.Close() })