Benchmark scaffolding + documentation

.gitignore

@@ -1 +1,7 @@
 pgremapper
+
+# benchmark outputs
+bench-results.txt
+profiles/
+*.prof
+*.test

BENCHMARKING.md

@@ -0,0 +1,167 @@
+# Benchmarking
+
+This repository includes benchmark scaffolding in `benchmark_test.go` so you can profile parsing and remap decision paths without a live Ceph cluster.
+
+For general background on Go profiling, see the [runtime/pprof package documentation](https://pkg.go.dev/runtime/pprof).
+
+## Scripts
+
+[`scripts/bench-all.sh`](scripts/bench-all.sh) runs benchmarks with **CPU, heap, mutex, and block** profiles.
+
+### Recommended: per-benchmark profiles
+
+Runs each sub-benchmark in its own `go test` invocation. Text metrics go to `bench-results.txt`; profiles land in `profiles/` as separate files per benchmark (for example `profiles/BenchmarkCalcPgMappingsToUndoBackfill_large.cpu.prof`).
+
+```
+./scripts/bench-all.sh
+```
+
+Defaults: `-benchtime=15s`, `-count=3`, `-benchmem`, all four profile types.
+
+Focus on production-scale fixtures only:
+
+```
+BENCH_FILTER='/(medium|large)$' ./scripts/bench-all.sh
+```
+
+Tune duration and repetitions:
+
+```
+BENCHTIME=30s COUNT=5 ./scripts/bench-all.sh
+```
+
+### Combined profiles (single run)
+
+One `go test` pass; all benchmarks share merged profile files under `profiles/{cpu,mem,mutex,block}.prof`. Faster to kick off, harder to attribute hotspots to a single code path.
+
+```
+./scripts/bench-all.sh --combined
+```
+
+### Smoke check
+
+One iteration per benchmark, no profiles:
+
+```
+./scripts/bench-all.sh --smoke
+```
+
+### Race detector
+
+One pass over the benchmark suite with Go's race detector enabled. No profiles; intended to catch data races in code paths exercised by the benchmarks. This is significantly slower than `--smoke` — use `BENCH_FILTER` to narrow scope when needed.
+
+```
+./scripts/bench-all.sh --race
+```
+
+```
+BENCH_FILTER='/(medium|large)$' ./scripts/bench-all.sh --race
+```
+
+### Environment variables
+
+| Variable | Default | Meaning |
+|----------|---------|---------|
+| `BENCHTIME` | `15s` | Minimum time per sub-benchmark |
+| `COUNT` | `3` | Timed repetitions (for `ns/op`, `B/op`, `allocs/op` spread) |
+| `BENCH_FILTER` | `.` | Passed to `go test -bench` |
+| `OUT_DIR` | `profiles` | Profile output directory |
+| `RESULTS` | `bench-results.txt` | Text benchmark output |
+
+`-benchtime` applies **per sub-benchmark**, not once for the whole suite. A full run over every `small`/`medium`/`large` case can take hours; use `BENCH_FILTER` to narrow scope when iterating.
+
+## Fixture sizes
+
+Benchmark fixture profiles used in `benchmark_test.go`:
+
+* `small`: `pgCount=4096`, `osdCount=128`, and for upmap-heavy cases `upmapCount=300`.
+* `medium`: `pgCount=16384`, `osdCount=512`, and for upmap-heavy cases `upmapCount=1200`.
+* `large`: `pgCount=65536`, `osdCount=2048`, and for upmap-heavy cases `upmapCount=4800`.
+
+Notes:
+
+* Most benchmark families include `small`, `medium`, and `large` sub-benchmarks.
+* Benchmark setup is included in timed sections, so wall-clock runtime more closely tracks the benchtime target.
+
+## Manual commands
+
+Run all benchmarks (no profiles):
+
+```
+go test -run '^$' -bench=. -benchmem ./...
+```
+
+Run a specific benchmark family:
+
+```
+go test -run '^$' -bench='^BenchmarkCalc' -benchtime=15s -benchmem ./...
+```
+
+`-benchmem` adds `B/op` and `allocs/op` columns alongside `ns/op`.
+
+## Comparing results across commits
+
+Save a baseline before changes:
+
+```
+./scripts/bench-all.sh
+cp bench-results.txt baseline-bench-results.txt
+```
+
+After changes, rerun and compare with [benchstat](https://pkg.go.dev/golang.org/x/perf/cmd/benchstat):
+
+```
+go install golang.org/x/perf/cmd/benchstat@latest
+benchstat baseline-bench-results.txt bench-results.txt
+```
+
+Run each pass at least three times (`COUNT=3` by default) and compare medians. Keep machine load low while benchmarking.
+
+## Inspect profiles with pprof
+
+Per-benchmark CPU profile:
+
+```
+go tool pprof -top -cum profiles/BenchmarkMustGetCurrentMappingState_large.cpu.prof
+```
+
+Heap allocations (GC pressure sources):
+
+```
+go tool pprof -top -alloc_space profiles/BenchmarkMustGetCurrentMappingState_large.mem.prof
+```
+
+Mutex contention:
+
+```
+go tool pprof -top profiles/BenchmarkCalcPgMappingsToUndoBackfill_large.mutex.prof
+```
+
+Block profile:
+
+```
+go tool pprof -top profiles/BenchmarkCalcPgMappingsToUndoBackfill_large.block.prof
+go tool pprof -top -sample_index=contentions profiles/BenchmarkCalcPgMappingsToUndoBackfill_large.block.prof
+```
+
+Combined-mode profiles:
+
+```
+go tool pprof -top -cum profiles/cpu.prof
+go tool pprof -top -alloc_space profiles/mem.prof
+```
+
+Interactive web UI:
+
+```
+go tool pprof -http=:8080 profiles/BenchmarkMustGetCurrentMappingState_large.cpu.prof
+```
+
+Then open `http://localhost:8080` in a browser.
+
+### Useful pprof commands inside interactive mode
+
+* `top` - hottest functions by self time
+* `top -cum` - hottest functions by cumulative time
+* `list <function>` - annotated source for a function
+* `web` - render the call graph (requires Graphviz installed)

README.md

@@ -46,11 +46,16 @@ go install github.com/digitalocean/pgremapper@latest
 
 Otherwise, clone this repository and use a golang Docker container to build:
 ```
-docker run --rm -v $(pwd):/pgremapper -w /pgremapper golang:1.21.4 go build -o pgremapper .
+docker run --rm -v $(pwd):/pgremapper -w /pgremapper golang:1.26.3 go build -o pgremapper .
 ```
 
 You can also download one of the pre-built binaries from the [releases page](https://github.com/digitalocean/pgremapper/releases).
 
+## Benchmarks
+
+Synthetic cluster benchmarks live in `benchmark_test.go` (no live Ceph cluster required).
+Use [`scripts/bench-all.sh`](scripts/bench-all.sh) for a full profiling run, or see [BENCHMARKING.md](BENCHMARKING.md) for details.
+
 ## Usage
 
 `pgremapper` makes no changes by default and has some global options:

backfillstate_test.go

@@ -22,7 +22,7 @@ import (
 
 func TestBackfillState(t *testing.T) {
 	setupTest(t)
-	defer teardownTest(t)
+	t.Cleanup(func() { teardownTest(t) })
 	pgDumpOut := `
 [
  { "pgid": "1.01", "up": [ 77, 1, 2 ], "acting": [ 77, 1, 2 ] },
@@ -106,7 +106,7 @@ func TestBackfillState(t *testing.T) {
 
 func TestPgCountsByOsd(t *testing.T) {
 	setupTest(t)
-	defer teardownTest(t)
+	t.Cleanup(func() { teardownTest(t) })
 	pgDumpOut := `
 [
  { "pgid": "1.01", "up": [ 77, 1, 2 ], "acting": [ 77, 1, 2 ] },
@@ -124,3 +124,134 @@ func TestPgCountsByOsd(t *testing.T) {
 	require.Equal(t, 1, c[3])
 	require.Equal(t, 1, c[4])
 }
+
+func TestHasRoomForRemapDoesNotMutateState(t *testing.T) {
+	setupTest(t)
+	t.Cleanup(func() { teardownTest(t) })
+
+	pgDumpOut := `
+[
+ { "pgid": "1.01", "up": [ 77, 1, 2 ], "acting": [ 77, 1, 2 ] },
+ { "pgid": "1.02", "up": [ 77, 3, 4 ], "acting": [ 77, 3, 5 ] },
+ { "pgid": "1.03", "up": [ 77, 5, 6 ], "acting": [ 3, 5, 7 ] }
+]
+`
+	runOsdDump = func() (string, error) { return "{}", nil }
+	runPgDumpPgsBrief = func() (string, error) { return pgDumpOut, nil }
+
+	bs := mustGetCurrentBackfillState()
+
+	type osdCounters struct {
+		local, remote, from, max int
+	}
+	snapshotOSDs := func() map[int]osdCounters {
+		out := make(map[int]osdCounters, len(bs.osds))
+		for osd, s := range bs.osds {
+			out[osd] = osdCounters{
+				local:  s.localReservations,
+				remote: s.remoteReservations,
+				from:   s.backfillsFrom,
+				max:    s.maxBackfillReservations,
+			}
+		}
+		return out
+	}
+	snapshotUp := func() map[string][]int {
+		out := make(map[string][]int, len(bs.pgbs))
+		for id, pgb := range bs.pgbs {
+			dup := make([]int, len(pgb.Up))
+			copy(dup, pgb.Up)
+			out[id] = dup
+		}
+		return out
+	}
+
+	beforeOSDs := snapshotOSDs()
+	beforeUp := snapshotUp()
+
+	// Valid remap probe path; hasRoomForRemap applies and reverts internally.
+	_ = bs.hasRoomForRemap("1.02", 4, 5)
+
+	require.Equal(t, beforeOSDs, snapshotOSDs())
+	require.Equal(t, beforeUp, snapshotUp())
+}
+
+func TestHasRoomForRemapRespectsSourceAndTargetLimits(t *testing.T) {
+	setupTest(t)
+	t.Cleanup(func() { teardownTest(t) })
+
+	pgDumpOut := `
+[
+ { "pgid": "1.01", "up": [ 77, 1, 2 ], "acting": [ 77, 1, 2 ] },
+ { "pgid": "1.02", "up": [ 77, 3, 4 ], "acting": [ 77, 3, 5 ] },
+ { "pgid": "1.03", "up": [ 77, 5, 6 ], "acting": [ 3, 5, 7 ] }
+]
+`
+	runOsdDump = func() (string, error) { return "{}", nil }
+	runPgDumpPgsBrief = func() (string, error) { return pgDumpOut, nil }
+
+	t.Run("source backfill limit", func(t *testing.T) {
+		bs := mustGetCurrentBackfillState()
+		// from=4 currently has no source backfills; cap at 0 blocks it.
+		bs.maxBackfillsFrom = 0
+		require.False(t, bs.hasRoomForRemap("1.02", 4, 5))
+	})
+
+	t.Run("target reservation limit", func(t *testing.T) {
+		bs := mustGetCurrentBackfillState()
+		// Make source limit permissive so target-side check is exercised.
+		bs.maxBackfillsFrom = 1000
+
+		// 1.01 is currently clean. Remapping 1->6 introduces backfill with target=6.
+		// Cap target OSD 6 at zero reservations so this must be rejected.
+		bs.osd(6).maxBackfillReservations = 0
+		require.False(t, bs.hasRoomForRemap("1.01", 1, 6))
+	})
+}
+
+func TestComputeBackfillSrcsTgts(t *testing.T) {
+	t.Run("mismatches produce aligned src and tgt sets", func(t *testing.T) {
+		pgb := &pgBriefItem{
+			PgID:   "1.2",
+			Up:     []int{1, 2, 33, 4, 55, 6},
+			Acting: []int{1, 22, 3, 4, 5, 66},
+		}
+
+		srcs, tgts := computeBackfillSrcsTgts(pgb)
+
+		require.Equal(t, []int{22, 3, 5, 66}, srcs)
+		require.Equal(t, []int{2, 33, 55, 6}, tgts)
+	})
+
+	t.Run("no mismatches produce empty sets", func(t *testing.T) {
+		pgb := &pgBriefItem{
+			PgID:   "1.3",
+			Up:     []int{1, 2, 3},
+			Acting: []int{1, 2, 3},
+		}
+
+		srcs, tgts := computeBackfillSrcsTgts(pgb)
+
+		require.Empty(t, srcs)
+		require.Empty(t, tgts)
+	})
+
+	t.Run("supports entries larger than reusable buffer", func(t *testing.T) {
+		up := make([]int, 10)
+		acting := make([]int, 10)
+		expectSrc := make([]int, 10)
+		expectTgt := make([]int, 10)
+		for i := range 10 {
+			acting[i] = i
+			up[i] = 100 + i
+			expectSrc[i] = i
+			expectTgt[i] = 100 + i
+		}
+		pgb := &pgBriefItem{PgID: "1.4", Up: up, Acting: acting}
+
+		srcs, tgts := computeBackfillSrcsTgts(pgb)
+
+		require.Equal(t, expectSrc, srcs)
+		require.Equal(t, expectTgt, tgts)
+	})
+}