diff --git a/AGENTS.md b/AGENTS.md index 25077130..85ffd0db 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,14 +27,19 @@ gorums/ ├── cmd/protoc-gen-gorums/ # Compiler plugin for code generation │ ├── dev/ # Static code + generated code examples │ └── gengorums/ # Compiler logic + templates +├── benchkit/ # Separate module: measurement and benchmarking +│ ├── proto/ # .proto sources for the benchkit module ├── examples/ # Separate module: example implementations ├── internal/ # Internal packages ├── doc/ # Documentation └── *.go # Core library files ``` -The repository holds two modules: `github.com/relab/gorums` at the root and -`github.com/relab/gorums/examples`, joined by `go.work`. +The repository holds three modules: `github.com/relab/gorums` at the root, +`github.com/relab/gorums/benchkit`, and `github.com/relab/gorums/examples`. +They are joined by `go.work`. +The dependency edge runs one way: benchkit imports gorums, never the reverse. +Keep the root `go.mod` free of benchmarking and orchestration dependencies. ## Development Rules diff --git a/Makefile b/Makefile index 7e33edb9..a27e8af3 100644 --- a/Makefile +++ b/Makefile @@ -7,12 +7,18 @@ static_file := $(gen_path)/template_static.go static_files := $(shell find $(dev_path) -name "*.go" -not -name "zorums*" -not -name "*_test.go") proto_path := $(dev_path):third_party:. -workspace_packages := ./... ./examples/... +# The benchkit module keeps its .proto files under benchkit/proto so that the +# import paths protoc records stay "benchkit/*.proto" and "benchmark/*.proto". +bk_path := benchkit/proto +bk_proto_path := $(bk_path):third_party:. +bk_module := github.com/relab/gorums/benchkit +workspace_packages := ./... ./examples/... ./benchkit/... plugin_deps := gorums.pb.go $(static_file) runtime_deps := internal/stream/stream.pb.go internal/stream/stream_grpc.pb.go +benchkit_deps := benchkit/benchkit.pb.go benchkit/control.pb.go benchkit/control_gorums.pb.go -.PHONY: all dev tools bootstrapgorums installgorums test compiletests genproto benchtest bench lint deadcode modernize goplscheck +.PHONY: all dev tools bootstrapgorums installgorums benchkit test compiletests genproto benchtest bench lint deadcode modernize goplscheck all: dev compiletests @@ -24,6 +30,24 @@ dev: installgorums $(runtime_deps) --go_opt=default_api_level=API_OPAQUE \ $(zorums_proto) +benchkit: installgorums $(benchkit_deps) + +# The benchkit module's generated code is written back into the module root +# rather than next to its .proto file, so these cannot use the pattern rules. +benchkit/benchkit.pb.go: $(bk_path)/benchkit/benchkit.proto + @protoc -I=$(bk_proto_path) \ + --go_out=benchkit --go_opt=module=$(bk_module) \ + --go_opt=default_api_level=API_OPAQUE $< + +benchkit/control.pb.go: $(bk_path)/benchkit/control.proto + @protoc -I=$(bk_proto_path) \ + --go_out=benchkit --go_opt=module=$(bk_module) \ + --go_opt=default_api_level=API_OPAQUE $< + +benchkit/control_gorums.pb.go: $(bk_path)/benchkit/control.proto + @protoc -I=$(bk_proto_path) \ + --gorums_out=benchkit --gorums_opt=module=$(bk_module) $< + $(static_file): $(static_files) @cp $(static_file) $(static_file).bak @protoc-gen-gorums --bundle=$(static_file) @@ -100,20 +124,20 @@ stressgen: tools rm ./internal/testprotos/testprotos.test lint: deadcode - @golangci-lint-v2 run ./... ./examples/... + @golangci-lint-v2 run ./... ./examples/... ./benchkit/... # deadcode reports functions unreachable from any main or test across all -# workspace modules (root and examples), so cross-module usage is +# workspace modules (root, examples, benchkit), so cross-module usage is # accounted for. It is advisory: exported library API with no in-repo caller # (e.g. optional dial/server options) and example-only helpers are expected to # appear. Review new entries for genuinely dead internal code. deadcode: - @go run golang.org/x/tools/cmd/deadcode@latest -test ./... ./examples/... + @go run golang.org/x/tools/cmd/deadcode@latest -test ./... ./examples/... ./benchkit/... modernize: - @go fix ./... ./examples/... + @go fix ./... ./examples/... ./benchkit/... @go run golang.org/x/tools/go/analysis/passes/modernize/cmd/modernize@latest \ - -fix ./... ./examples/... + -fix ./... ./examples/... ./benchkit/... # Report all gopls diagnostics, including hint-level style and modernization # suggestions. Generated Go files are excluded because their generators own them. @@ -134,11 +158,12 @@ goplscheck: exit 1; \ fi -# Regenerate all Gorums and protobuf generated files across the repo (dev, internal/tests, examples). +# Regenerate all Gorums and protobuf generated files across the repo (dev, benchkit, internal/tests, examples). # This will force regeneration even though the proto files have not changed. genproto: installgorums dev - @echo "Regenerating all proto files (dev, internal/tests, examples)" + @echo "Regenerating all proto files (dev, benchkit, internal/tests, examples)" @$(MAKE) -B -s dev + @$(MAKE) -B -s $(benchkit_deps) @$(MAKE) -B -s --no-print-directory -C ./internal/tests all @$(MAKE) -B -s --no-print-directory -C ./examples all diff --git a/benchkit/aggregate.go b/benchkit/aggregate.go new file mode 100644 index 00000000..9e1dfb75 --- /dev/null +++ b/benchkit/aggregate.go @@ -0,0 +1,67 @@ +package benchkit + +import ( + "maps" + "slices" + + "github.com/relab/gorums" +) + +// AppendServerStats attaches per-server memory statistics from Stop RPC replies +// to an existing result. It is used by client-measured benchmarks (QuorumCall, +// AsyncQuorumCall) where the client's latency samples are already in result but +// the server-side allocations must be collected separately. +func AppendServerStats(result *Result, replies map[uint32]*Result) { + for _, id := range slices.Sorted(maps.Keys(replies)) { + r := replies[id] + result.SetServerStats(append(result.GetServerStats(), MemoryStat_builder{ + Allocs: r.GetAllocsPerOp() * r.GetTotalOps(), + Memory: r.GetMemPerOp() * r.GetTotalOps(), + }.Build())) + } +} + +// AggregateServerResults combines per-server Stop replies into a single +// cluster-wide Result. TotalOps and Throughput are summed across servers so +// the reported value reflects the cluster's aggregate work; TotalTime is the +// maximum across servers so it reflects the measurement window's wall-clock +// time rather than an N-fold sum. Latency samples from every reply are +// concatenated so that LatencyMean, LatencyMeanAndStdDev and Percentiles +// recompute from the full cluster-wide distribution. In StatsMode_HDR, where +// replies carry a histogram instead of raw samples, the per-server histograms +// are merged onto one canonical histogram (see [mergeHistograms]) instead. +// +// Per-server memory and alloc counters are attached as ServerStats in a stable +// node-ID order so the output columns do not reshuffle between runs. +func AggregateServerResults(replies map[uint32]*Result) (*Result, error) { + if len(replies) == 0 { + return nil, gorums.ErrIncomplete + } + + resp := &Result{} + var allSamples []int64 + var hists []*LatencyHistogram + for _, id := range slices.Sorted(maps.Keys(replies)) { + reply := replies[id] + // The benchmark name lives in RunConfig and is stamped by Run on the + // aggregated result; per-server replies carry no config. + resp.SetTotalOps(resp.GetTotalOps() + reply.GetTotalOps()) + resp.SetTotalTime(max(resp.GetTotalTime(), reply.GetTotalTime())) + resp.SetThroughput(resp.GetThroughput() + reply.GetThroughput()) + allSamples = append(allSamples, reply.GetLatencies()...) + if h := reply.GetHistogram(); h != nil { + hists = append(hists, h) + } + resp.SetServerStats(append(resp.GetServerStats(), MemoryStat_builder{ + Allocs: reply.GetAllocsPerOp() * reply.GetTotalOps(), + Memory: reply.GetMemPerOp() * reply.GetTotalOps(), + }.Build())) + } + if len(allSamples) > 0 { + resp.SetLatencies(allSamples) + } + if len(hists) > 0 { + resp.SetHistogram(mergeHistograms(hists...)) + } + return resp, nil +} diff --git a/benchkit/aggregate_test.go b/benchkit/aggregate_test.go new file mode 100644 index 00000000..50ed0e68 --- /dev/null +++ b/benchkit/aggregate_test.go @@ -0,0 +1,186 @@ +package benchkit + +import ( + "errors" + "math" + "testing" + "time" + + "github.com/relab/gorums" +) + +func TestAggregateServerResults(t *testing.T) { + const eps = 1e-9 + + // reply is a compact per-node stub used to build the input map. + // samples stands in for per-op latency measurements in nanoseconds; the + // aggregator concatenates them across servers so the cluster-wide mean + // and stddev can be recomputed from the full distribution. + type reply struct { + tTime int64 + tput float64 + samples []int64 + allocsPO uint64 + memPO uint64 + } + newReplies := func(in map[uint32]reply) map[uint32]*Result { + out := make(map[uint32]*Result, len(in)) + for id, r := range in { + out[id] = Result_builder{ + TotalOps: uint64(len(r.samples)), + TotalTime: r.tTime, + Throughput: r.tput, + Latencies: r.samples, + AllocsPerOp: r.allocsPO, + MemPerOp: r.memPO, + }.Build() + } + return out + } + + tests := []struct { + name string + replies map[uint32]reply + wantErr error + // For non-error cases, the expected aggregate values. + wantTotalOps uint64 + wantTotalTime int64 + wantThroughput float64 + wantSamplesConcat []int64 // feeds expected mean/stddev via Result methods + wantServerMem []uint64 // per-server memory in sorted node-ID order + wantServerAlloc []uint64 // per-server allocs in sorted node-ID order + }{ + { + name: "EmptyRepliesReturnsIncomplete", + replies: nil, + wantErr: gorums.ErrIncomplete, + }, + { + name: "SingleReplyPassesThrough", + replies: map[uint32]reply{ + 1: {tTime: 1_000_000, tput: 100, samples: []int64{10, 20, 30, 40}, allocsPO: 3, memPO: 64}, + }, + wantTotalOps: 4, + wantTotalTime: 1_000_000, + wantThroughput: 100, + wantSamplesConcat: []int64{10, 20, 30, 40}, + wantServerMem: []uint64{64 * 4}, + wantServerAlloc: []uint64{3 * 4}, + }, + { + name: "ThreeRepliesConcatenateSamples", + replies: map[uint32]reply{ + 1: {tTime: 1, tput: 10, samples: []int64{10, 12}, allocsPO: 2, memPO: 8}, + 2: {tTime: 2, tput: 20, samples: []int64{20, 22, 24}, allocsPO: 3, memPO: 16}, + 3: {tTime: 3, tput: 30, samples: []int64{30, 32, 34, 36}, allocsPO: 5, memPO: 32}, + }, + wantTotalOps: 9, + wantTotalTime: 3, // max across servers, not the sum + wantThroughput: 60, + wantSamplesConcat: []int64{10, 12, 20, 22, 24, 30, 32, 34, 36}, + wantServerMem: []uint64{8 * 2, 16 * 3, 32 * 4}, // sorted by node ID: 1, 2, 3 + wantServerAlloc: []uint64{2 * 2, 3 * 3, 5 * 4}, + }, + { + name: "UnsortedIDsProduceSortedServerStats", + replies: map[uint32]reply{ + // Insertion order does not matter; sorted ID order determines ServerStats[i]. + 7: {tTime: 1, tput: 1, samples: []int64{5, 5, 5}, allocsPO: 1, memPO: 1}, + 3: {tTime: 1, tput: 1, samples: []int64{5, 5, 5}, allocsPO: 2, memPO: 2}, + 5: {tTime: 1, tput: 1, samples: []int64{5, 5, 5}, allocsPO: 3, memPO: 3}, + }, + wantTotalOps: 9, + wantTotalTime: 1, // max across servers, not the sum + wantThroughput: 3, + wantSamplesConcat: []int64{5, 5, 5, 5, 5, 5, 5, 5, 5}, + wantServerMem: []uint64{2 * 3, 3 * 3, 1 * 3}, // node IDs sorted: 3, 5, 7 -> mem 2,3,1 + wantServerAlloc: []uint64{2 * 3, 3 * 3, 1 * 3}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := AggregateServerResults(newReplies(tt.replies)) + if tt.wantErr != nil { + if !errors.Is(err, tt.wantErr) { + t.Fatalf("err = %v, want %v", err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if got.GetTotalOps() != tt.wantTotalOps { + t.Errorf("TotalOps = %d, want %d", got.GetTotalOps(), tt.wantTotalOps) + } + if got.GetTotalTime() != tt.wantTotalTime { + t.Errorf("TotalTime = %d, want %d", got.GetTotalTime(), tt.wantTotalTime) + } + if math.Abs(got.GetThroughput()-tt.wantThroughput) > eps { + t.Errorf("Throughput = %v, want %v", got.GetThroughput(), tt.wantThroughput) + } + + // Reference values come from invoking the same methods on a + // Result built from the expected concatenation. time.Duration + // is int64 so equality is exact once the truncation boundary + // is shared between got and want. + want := Result_builder{Latencies: tt.wantSamplesConcat}.Build() + gotMean, gotSD := got.LatencyMeanAndStdDev() + wantMean, wantSD := want.LatencyMeanAndStdDev() + if gotMean != wantMean { + t.Errorf("LatencyMean = %v, want %v", gotMean, wantMean) + } + if gotSD != wantSD { + t.Errorf("LatencyStdDev = %v, want %v", gotSD, wantSD) + } + + if n := len(got.GetServerStats()); n != len(tt.wantServerMem) { + t.Fatalf("len(ServerStats) = %d, want %d", n, len(tt.wantServerMem)) + } + for i, s := range got.GetServerStats() { + if s.GetMemory() != tt.wantServerMem[i] { + t.Errorf("ServerStats[%d].Memory = %d, want %d", i, s.GetMemory(), tt.wantServerMem[i]) + } + if s.GetAllocs() != tt.wantServerAlloc[i] { + t.Errorf("ServerStats[%d].Allocs = %d, want %d", i, s.GetAllocs(), tt.wantServerAlloc[i]) + } + } + }) + } +} + +// TestAggregateServerResultsHistogram verifies that in StatsMode_HDR, where +// per-server replies carry a histogram instead of raw samples, the aggregate +// merges the per-server histograms (Latencies nil, Histogram set) with counts +// summed across servers, and that TotalOps and Throughput still sum as usual. +func TestAggregateServerResultsHistogram(t *testing.T) { + replies := map[uint32]*Result{ + 1: Result_builder{TotalOps: 3, TotalTime: 2, Throughput: 30, Histogram: hist(1_000, 1_000, 1_000)}.Build(), + 2: Result_builder{TotalOps: 2, TotalTime: 1, Throughput: 20, Histogram: hist(1_000, 2_000)}.Build(), + } + got, err := AggregateServerResults(replies) + if err != nil { + t.Fatalf("AggregateServerResults: %v", err) + } + if got.GetLatencies() != nil { + t.Errorf("Latencies = %v, want nil in HDR mode", got.GetLatencies()) + } + if got.GetTotalOps() != 5 { + t.Errorf("TotalOps = %d, want 5", got.GetTotalOps()) + } + if got.GetThroughput() != 50 { + t.Errorf("Throughput = %v, want 50", got.GetThroughput()) + } + h := got.GetHistogram() + if h == nil { + t.Fatal("Histogram = nil, want merged histogram") + } + if n := totalCount(h); n != 5 { + t.Errorf("merged histogram count = %d, want 5", n) + } + // Merged distribution {1,1,1,1,2}µs has median 1µs, reproduced within + // HDR precision. + if p := got.Percentiles(0.5); p == nil || math.Abs(float64(p[0]-time.Microsecond)) > 50 { + t.Errorf("Percentiles(0.5) = %v, want ≈1µs", p) + } +} diff --git a/benchkit/benchkit.pb.go b/benchkit/benchkit.pb.go new file mode 100644 index 00000000..87ed52bc --- /dev/null +++ b/benchkit/benchkit.pb.go @@ -0,0 +1,2503 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v7.35.1 +// source: benchkit/benchkit.proto + +package benchkit + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// MeasurementMode records who timed each operation, which decides whether the +// event-stream index map can cut the latency slice at read time. Client-measured +// runs record one in-order sample per op, so the cut is exact; server-measured +// samples arrive out of band and clock-corrected, so they cannot be cut by +// client op counts (see doc/benchkit.html, section 11). +type MeasurementMode int32 + +const ( + MeasurementMode_CLIENT_MEASURED MeasurementMode = 0 // latency from Stats.AddLatency, one per op (default) + MeasurementMode_SERVER_MEASURED MeasurementMode = 1 // latency from server replies, clock-corrected +) + +// Enum value maps for MeasurementMode. +var ( + MeasurementMode_name = map[int32]string{ + 0: "CLIENT_MEASURED", + 1: "SERVER_MEASURED", + } + MeasurementMode_value = map[string]int32{ + "CLIENT_MEASURED": 0, + "SERVER_MEASURED": 1, + } +) + +func (x MeasurementMode) Enum() *MeasurementMode { + p := new(MeasurementMode) + *p = x + return p +} + +func (x MeasurementMode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (MeasurementMode) Descriptor() protoreflect.EnumDescriptor { + return file_benchkit_benchkit_proto_enumTypes[0].Descriptor() +} + +func (MeasurementMode) Type() protoreflect.EnumType { + return &file_benchkit_benchkit_proto_enumTypes[0] +} + +func (x MeasurementMode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// StatsMode records the aggregate latency backing store, so consumers know +// whether raw per-op samples (exact) or a bounded-memory histogram (hdr) is +// available without inferring it from the presence of the latencies field. +type StatsMode int32 + +const ( + StatsMode_EXACT StatsMode = 0 // every sample retained; exact percentiles (default) + StatsMode_HDR StatsMode = 2 // log-linear histogram; approximate percentiles, bounded memory +) + +// Enum value maps for StatsMode. +var ( + StatsMode_name = map[int32]string{ + 0: "EXACT", + 2: "HDR", + } + StatsMode_value = map[string]int32{ + "EXACT": 0, + "HDR": 2, + } +) + +func (x StatsMode) Enum() *StatsMode { + p := new(StatsMode) + *p = x + return p +} + +func (x StatsMode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (StatsMode) Descriptor() protoreflect.EnumDescriptor { + return file_benchkit_benchkit_proto_enumTypes[1].Descriptor() +} + +func (StatsMode) Type() protoreflect.EnumType { + return &file_benchkit_benchkit_proto_enumTypes[1] +} + +func (x StatsMode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +type PhaseMarker_Phase int32 + +const ( + PhaseMarker_START PhaseMarker_Phase = 0 // t=0; rate carries the initial target ops/s (default) + PhaseMarker_RATE_STEP PhaseMarker_Phase = 1 // rate ramp step; rate carries the new target ops/s + PhaseMarker_STOP PhaseMarker_Phase = 2 // run finished +) + +// Enum value maps for PhaseMarker_Phase. +var ( + PhaseMarker_Phase_name = map[int32]string{ + 0: "START", + 1: "RATE_STEP", + 2: "STOP", + } + PhaseMarker_Phase_value = map[string]int32{ + "START": 0, + "RATE_STEP": 1, + "STOP": 2, + } +) + +func (x PhaseMarker_Phase) Enum() *PhaseMarker_Phase { + p := new(PhaseMarker_Phase) + *p = x + return p +} + +func (x PhaseMarker_Phase) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (PhaseMarker_Phase) Descriptor() protoreflect.EnumDescriptor { + return file_benchkit_benchkit_proto_enumTypes[2].Descriptor() +} + +func (PhaseMarker_Phase) Type() protoreflect.EnumType { + return &file_benchkit_benchkit_proto_enumTypes[2] +} + +func (x PhaseMarker_Phase) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// RunConfig holds the configuration metadata for one benchmark run. It is kept +// separate from the measured results so producers and consumers can reason +// about how a run was configured independently of what it measured. Durations +// are nanoseconds (compatible with time.Duration). +type RunConfig struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Name string `protobuf:"bytes,1,opt,name=name"` + xxx_hidden_NumNodes int32 `protobuf:"varint,2,opt,name=num_nodes,json=numNodes"` + xxx_hidden_Mode string `protobuf:"bytes,3,opt,name=mode"` + xxx_hidden_Duration int64 `protobuf:"varint,4,opt,name=duration"` + xxx_hidden_Workers int32 `protobuf:"varint,5,opt,name=workers"` + xxx_hidden_Payload int32 `protobuf:"varint,6,opt,name=payload"` + xxx_hidden_Rate int64 `protobuf:"varint,7,opt,name=rate"` + xxx_hidden_Interval int64 `protobuf:"varint,8,opt,name=interval"` + xxx_hidden_MeasurementMode MeasurementMode `protobuf:"varint,9,opt,name=measurement_mode,json=measurementMode,enum=benchkit.MeasurementMode"` + xxx_hidden_StatsMode StatsMode `protobuf:"varint,10,opt,name=stats_mode,json=statsMode,enum=benchkit.StatsMode"` + xxx_hidden_StreamMode string `protobuf:"bytes,11,opt,name=stream_mode,json=streamMode"` + xxx_hidden_QuorumSize int32 `protobuf:"varint,12,opt,name=quorum_size,json=quorumSize"` + xxx_hidden_MaxAsync int32 `protobuf:"varint,13,opt,name=max_async,json=maxAsync"` + xxx_hidden_RateStep int64 `protobuf:"varint,14,opt,name=rate_step,json=rateStep"` + xxx_hidden_RateStepMax int64 `protobuf:"varint,15,opt,name=rate_step_max,json=rateStepMax"` + xxx_hidden_CallTimeout int64 `protobuf:"varint,16,opt,name=call_timeout,json=callTimeout"` + xxx_hidden_SendBuffer int32 `protobuf:"varint,17,opt,name=send_buffer,json=sendBuffer"` + xxx_hidden_RecvBuffer int32 `protobuf:"varint,18,opt,name=recv_buffer,json=recvBuffer"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RunConfig) Reset() { + *x = RunConfig{} + mi := &file_benchkit_benchkit_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RunConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RunConfig) ProtoMessage() {} + +func (x *RunConfig) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_benchkit_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *RunConfig) GetName() string { + if x != nil { + return x.xxx_hidden_Name + } + return "" +} + +func (x *RunConfig) GetNumNodes() int32 { + if x != nil { + return x.xxx_hidden_NumNodes + } + return 0 +} + +func (x *RunConfig) GetMode() string { + if x != nil { + return x.xxx_hidden_Mode + } + return "" +} + +func (x *RunConfig) GetDuration() int64 { + if x != nil { + return x.xxx_hidden_Duration + } + return 0 +} + +func (x *RunConfig) GetWorkers() int32 { + if x != nil { + return x.xxx_hidden_Workers + } + return 0 +} + +func (x *RunConfig) GetPayload() int32 { + if x != nil { + return x.xxx_hidden_Payload + } + return 0 +} + +func (x *RunConfig) GetRate() int64 { + if x != nil { + return x.xxx_hidden_Rate + } + return 0 +} + +func (x *RunConfig) GetInterval() int64 { + if x != nil { + return x.xxx_hidden_Interval + } + return 0 +} + +func (x *RunConfig) GetMeasurementMode() MeasurementMode { + if x != nil { + return x.xxx_hidden_MeasurementMode + } + return MeasurementMode_CLIENT_MEASURED +} + +func (x *RunConfig) GetStatsMode() StatsMode { + if x != nil { + return x.xxx_hidden_StatsMode + } + return StatsMode_EXACT +} + +func (x *RunConfig) GetStreamMode() string { + if x != nil { + return x.xxx_hidden_StreamMode + } + return "" +} + +func (x *RunConfig) GetQuorumSize() int32 { + if x != nil { + return x.xxx_hidden_QuorumSize + } + return 0 +} + +func (x *RunConfig) GetMaxAsync() int32 { + if x != nil { + return x.xxx_hidden_MaxAsync + } + return 0 +} + +func (x *RunConfig) GetRateStep() int64 { + if x != nil { + return x.xxx_hidden_RateStep + } + return 0 +} + +func (x *RunConfig) GetRateStepMax() int64 { + if x != nil { + return x.xxx_hidden_RateStepMax + } + return 0 +} + +func (x *RunConfig) GetCallTimeout() int64 { + if x != nil { + return x.xxx_hidden_CallTimeout + } + return 0 +} + +func (x *RunConfig) GetSendBuffer() int32 { + if x != nil { + return x.xxx_hidden_SendBuffer + } + return 0 +} + +func (x *RunConfig) GetRecvBuffer() int32 { + if x != nil { + return x.xxx_hidden_RecvBuffer + } + return 0 +} + +func (x *RunConfig) SetName(v string) { + x.xxx_hidden_Name = v +} + +func (x *RunConfig) SetNumNodes(v int32) { + x.xxx_hidden_NumNodes = v +} + +func (x *RunConfig) SetMode(v string) { + x.xxx_hidden_Mode = v +} + +func (x *RunConfig) SetDuration(v int64) { + x.xxx_hidden_Duration = v +} + +func (x *RunConfig) SetWorkers(v int32) { + x.xxx_hidden_Workers = v +} + +func (x *RunConfig) SetPayload(v int32) { + x.xxx_hidden_Payload = v +} + +func (x *RunConfig) SetRate(v int64) { + x.xxx_hidden_Rate = v +} + +func (x *RunConfig) SetInterval(v int64) { + x.xxx_hidden_Interval = v +} + +func (x *RunConfig) SetMeasurementMode(v MeasurementMode) { + x.xxx_hidden_MeasurementMode = v +} + +func (x *RunConfig) SetStatsMode(v StatsMode) { + x.xxx_hidden_StatsMode = v +} + +func (x *RunConfig) SetStreamMode(v string) { + x.xxx_hidden_StreamMode = v +} + +func (x *RunConfig) SetQuorumSize(v int32) { + x.xxx_hidden_QuorumSize = v +} + +func (x *RunConfig) SetMaxAsync(v int32) { + x.xxx_hidden_MaxAsync = v +} + +func (x *RunConfig) SetRateStep(v int64) { + x.xxx_hidden_RateStep = v +} + +func (x *RunConfig) SetRateStepMax(v int64) { + x.xxx_hidden_RateStepMax = v +} + +func (x *RunConfig) SetCallTimeout(v int64) { + x.xxx_hidden_CallTimeout = v +} + +func (x *RunConfig) SetSendBuffer(v int32) { + x.xxx_hidden_SendBuffer = v +} + +func (x *RunConfig) SetRecvBuffer(v int32) { + x.xxx_hidden_RecvBuffer = v +} + +type RunConfig_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Name string + NumNodes int32 + Mode string + Duration int64 + Workers int32 + Payload int32 + Rate int64 + Interval int64 + MeasurementMode MeasurementMode + StatsMode StatsMode + StreamMode string + QuorumSize int32 + MaxAsync int32 + RateStep int64 + RateStepMax int64 + CallTimeout int64 + SendBuffer int32 + RecvBuffer int32 +} + +func (b0 RunConfig_builder) Build() *RunConfig { + m0 := &RunConfig{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Name = b.Name + x.xxx_hidden_NumNodes = b.NumNodes + x.xxx_hidden_Mode = b.Mode + x.xxx_hidden_Duration = b.Duration + x.xxx_hidden_Workers = b.Workers + x.xxx_hidden_Payload = b.Payload + x.xxx_hidden_Rate = b.Rate + x.xxx_hidden_Interval = b.Interval + x.xxx_hidden_MeasurementMode = b.MeasurementMode + x.xxx_hidden_StatsMode = b.StatsMode + x.xxx_hidden_StreamMode = b.StreamMode + x.xxx_hidden_QuorumSize = b.QuorumSize + x.xxx_hidden_MaxAsync = b.MaxAsync + x.xxx_hidden_RateStep = b.RateStep + x.xxx_hidden_RateStepMax = b.RateStepMax + x.xxx_hidden_CallTimeout = b.CallTimeout + x.xxx_hidden_SendBuffer = b.SendBuffer + x.xxx_hidden_RecvBuffer = b.RecvBuffer + return m0 +} + +// ThroughputInterval is the ops-completed count and elapsed time for one +// ticker interval (both quantities come from Stats.TickInterval). +type ThroughputInterval struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Ops uint64 `protobuf:"varint,1,opt,name=ops"` + xxx_hidden_Duration int64 `protobuf:"varint,2,opt,name=duration"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ThroughputInterval) Reset() { + *x = ThroughputInterval{} + mi := &file_benchkit_benchkit_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ThroughputInterval) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ThroughputInterval) ProtoMessage() {} + +func (x *ThroughputInterval) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_benchkit_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *ThroughputInterval) GetOps() uint64 { + if x != nil { + return x.xxx_hidden_Ops + } + return 0 +} + +func (x *ThroughputInterval) GetDuration() int64 { + if x != nil { + return x.xxx_hidden_Duration + } + return 0 +} + +func (x *ThroughputInterval) SetOps(v uint64) { + x.xxx_hidden_Ops = v +} + +func (x *ThroughputInterval) SetDuration(v int64) { + x.xxx_hidden_Duration = v +} + +type ThroughputInterval_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Ops uint64 + Duration int64 +} + +func (b0 ThroughputInterval_builder) Build() *ThroughputInterval { + m0 := &ThroughputInterval{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Ops = b.Ops + x.xxx_hidden_Duration = b.Duration + return m0 +} + +// LatencyInterval is the Welford accumulator state for one ticker interval. +// Emitted each tick to summarize latency over time; the values are derived +// online and retain no raw samples. +type LatencyInterval struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Mean float64 `protobuf:"fixed64,1,opt,name=mean"` + xxx_hidden_Stddev float64 `protobuf:"fixed64,2,opt,name=stddev"` + xxx_hidden_Count uint64 `protobuf:"varint,3,opt,name=count"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LatencyInterval) Reset() { + *x = LatencyInterval{} + mi := &file_benchkit_benchkit_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LatencyInterval) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LatencyInterval) ProtoMessage() {} + +func (x *LatencyInterval) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_benchkit_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LatencyInterval) GetMean() float64 { + if x != nil { + return x.xxx_hidden_Mean + } + return 0 +} + +func (x *LatencyInterval) GetStddev() float64 { + if x != nil { + return x.xxx_hidden_Stddev + } + return 0 +} + +func (x *LatencyInterval) GetCount() uint64 { + if x != nil { + return x.xxx_hidden_Count + } + return 0 +} + +func (x *LatencyInterval) SetMean(v float64) { + x.xxx_hidden_Mean = v +} + +func (x *LatencyInterval) SetStddev(v float64) { + x.xxx_hidden_Stddev = v +} + +func (x *LatencyInterval) SetCount(v uint64) { + x.xxx_hidden_Count = v +} + +type LatencyInterval_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Mean float64 + Stddev float64 + Count uint64 +} + +func (b0 LatencyInterval_builder) Build() *LatencyInterval { + m0 := &LatencyInterval{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Mean = b.Mean + x.xxx_hidden_Stddev = b.Stddev + x.xxx_hidden_Count = b.Count + return m0 +} + +// PhaseMarker announces a lifecycle transition in a run. There is no warmup +// phase: START fires at t=0, STOP at the end, and RATE_STEP on each rate-ramp +// transition. Consumers use phase offsets to trim and annotate. +type PhaseMarker struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Phase PhaseMarker_Phase `protobuf:"varint,1,opt,name=phase,enum=benchkit.PhaseMarker_Phase"` + xxx_hidden_Rate int64 `protobuf:"varint,2,opt,name=rate"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PhaseMarker) Reset() { + *x = PhaseMarker{} + mi := &file_benchkit_benchkit_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PhaseMarker) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PhaseMarker) ProtoMessage() {} + +func (x *PhaseMarker) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_benchkit_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *PhaseMarker) GetPhase() PhaseMarker_Phase { + if x != nil { + return x.xxx_hidden_Phase + } + return PhaseMarker_START +} + +func (x *PhaseMarker) GetRate() int64 { + if x != nil { + return x.xxx_hidden_Rate + } + return 0 +} + +func (x *PhaseMarker) SetPhase(v PhaseMarker_Phase) { + x.xxx_hidden_Phase = v +} + +func (x *PhaseMarker) SetRate(v int64) { + x.xxx_hidden_Rate = v +} + +type PhaseMarker_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Phase PhaseMarker_Phase + Rate int64 +} + +func (b0 PhaseMarker_builder) Build() *PhaseMarker { + m0 := &PhaseMarker{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Phase = b.Phase + x.xxx_hidden_Rate = b.Rate + return m0 +} + +// Event is one time-stamped entry in the per-node event stream. offset is +// nanoseconds since the START phase marker (monotonic). Field 15 is reserved +// for a google.protobuf.Any escape hatch carrying protocol-specific events +// without a schema change. +type Event struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Offset int64 `protobuf:"varint,1,opt,name=offset"` + xxx_hidden_Payload isEvent_Payload `protobuf_oneof:"payload"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Event) Reset() { + *x = Event{} + mi := &file_benchkit_benchkit_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Event) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Event) ProtoMessage() {} + +func (x *Event) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_benchkit_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *Event) GetOffset() int64 { + if x != nil { + return x.xxx_hidden_Offset + } + return 0 +} + +func (x *Event) GetThroughput() *ThroughputInterval { + if x != nil { + if x, ok := x.xxx_hidden_Payload.(*event_Throughput); ok { + return x.Throughput + } + } + return nil +} + +func (x *Event) GetLatency() *LatencyInterval { + if x != nil { + if x, ok := x.xxx_hidden_Payload.(*event_Latency); ok { + return x.Latency + } + } + return nil +} + +func (x *Event) GetPhase() *PhaseMarker { + if x != nil { + if x, ok := x.xxx_hidden_Payload.(*event_Phase); ok { + return x.Phase + } + } + return nil +} + +func (x *Event) SetOffset(v int64) { + x.xxx_hidden_Offset = v +} + +func (x *Event) SetThroughput(v *ThroughputInterval) { + if v == nil { + x.xxx_hidden_Payload = nil + return + } + x.xxx_hidden_Payload = &event_Throughput{v} +} + +func (x *Event) SetLatency(v *LatencyInterval) { + if v == nil { + x.xxx_hidden_Payload = nil + return + } + x.xxx_hidden_Payload = &event_Latency{v} +} + +func (x *Event) SetPhase(v *PhaseMarker) { + if v == nil { + x.xxx_hidden_Payload = nil + return + } + x.xxx_hidden_Payload = &event_Phase{v} +} + +func (x *Event) HasPayload() bool { + if x == nil { + return false + } + return x.xxx_hidden_Payload != nil +} + +func (x *Event) HasThroughput() bool { + if x == nil { + return false + } + _, ok := x.xxx_hidden_Payload.(*event_Throughput) + return ok +} + +func (x *Event) HasLatency() bool { + if x == nil { + return false + } + _, ok := x.xxx_hidden_Payload.(*event_Latency) + return ok +} + +func (x *Event) HasPhase() bool { + if x == nil { + return false + } + _, ok := x.xxx_hidden_Payload.(*event_Phase) + return ok +} + +func (x *Event) ClearPayload() { + x.xxx_hidden_Payload = nil +} + +func (x *Event) ClearThroughput() { + if _, ok := x.xxx_hidden_Payload.(*event_Throughput); ok { + x.xxx_hidden_Payload = nil + } +} + +func (x *Event) ClearLatency() { + if _, ok := x.xxx_hidden_Payload.(*event_Latency); ok { + x.xxx_hidden_Payload = nil + } +} + +func (x *Event) ClearPhase() { + if _, ok := x.xxx_hidden_Payload.(*event_Phase); ok { + x.xxx_hidden_Payload = nil + } +} + +const Event_Payload_not_set_case case_Event_Payload = 0 +const Event_Throughput_case case_Event_Payload = 2 +const Event_Latency_case case_Event_Payload = 3 +const Event_Phase_case case_Event_Payload = 4 + +func (x *Event) WhichPayload() case_Event_Payload { + if x == nil { + return Event_Payload_not_set_case + } + switch x.xxx_hidden_Payload.(type) { + case *event_Throughput: + return Event_Throughput_case + case *event_Latency: + return Event_Latency_case + case *event_Phase: + return Event_Phase_case + default: + return Event_Payload_not_set_case + } +} + +type Event_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Offset int64 + // Fields of oneof xxx_hidden_Payload: + Throughput *ThroughputInterval + Latency *LatencyInterval + Phase *PhaseMarker + // -- end of xxx_hidden_Payload +} + +func (b0 Event_builder) Build() *Event { + m0 := &Event{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Offset = b.Offset + if b.Throughput != nil { + x.xxx_hidden_Payload = &event_Throughput{b.Throughput} + } + if b.Latency != nil { + x.xxx_hidden_Payload = &event_Latency{b.Latency} + } + if b.Phase != nil { + x.xxx_hidden_Payload = &event_Phase{b.Phase} + } + return m0 +} + +type case_Event_Payload protoreflect.FieldNumber + +func (x case_Event_Payload) String() string { + md := file_benchkit_benchkit_proto_msgTypes[4].Descriptor() + if x == 0 { + return "not set" + } + return protoimpl.X.MessageFieldStringOf(md, protoreflect.FieldNumber(x)) +} + +type isEvent_Payload interface { + isEvent_Payload() +} + +type event_Throughput struct { + Throughput *ThroughputInterval `protobuf:"bytes,2,opt,name=throughput,oneof"` +} + +type event_Latency struct { + Latency *LatencyInterval `protobuf:"bytes,3,opt,name=latency,oneof"` +} + +type event_Phase struct { + Phase *PhaseMarker `protobuf:"bytes,4,opt,name=phase,oneof"` // 15 reserved for google.protobuf.Any extension +} + +func (*event_Throughput) isEvent_Payload() {} + +func (*event_Latency) isEvent_Payload() {} + +func (*event_Phase) isEvent_Payload() {} + +// LatencyHistogram is the bounded-memory latency distribution recorded in +// StatsMode_HDR, where no raw samples are retained. Entry i records count[i] +// samples indistinguishable from value[i] (nanoseconds) at the histogram's +// resolution; values are ascending. Consumers treat the pairs as a weighted +// sample set — quantiles, mean, and stddev are computed over them without +// knowing the producer's bucket layout. +type LatencyHistogram struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Value []int64 `protobuf:"varint,1,rep,packed,name=value"` + xxx_hidden_Count []uint64 `protobuf:"varint,2,rep,packed,name=count"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LatencyHistogram) Reset() { + *x = LatencyHistogram{} + mi := &file_benchkit_benchkit_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LatencyHistogram) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LatencyHistogram) ProtoMessage() {} + +func (x *LatencyHistogram) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_benchkit_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LatencyHistogram) GetValue() []int64 { + if x != nil { + return x.xxx_hidden_Value + } + return nil +} + +func (x *LatencyHistogram) GetCount() []uint64 { + if x != nil { + return x.xxx_hidden_Count + } + return nil +} + +func (x *LatencyHistogram) SetValue(v []int64) { + x.xxx_hidden_Value = v +} + +func (x *LatencyHistogram) SetCount(v []uint64) { + x.xxx_hidden_Count = v +} + +type LatencyHistogram_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Value []int64 + Count []uint64 +} + +func (b0 LatencyHistogram_builder) Build() *LatencyHistogram { + m0 := &LatencyHistogram{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Value = b.Value + x.xxx_hidden_Count = b.Count + return m0 +} + +// MemoryStat contains memory statistics for a single server. +type MemoryStat struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Allocs uint64 `protobuf:"varint,1,opt,name=allocs"` + xxx_hidden_Memory uint64 `protobuf:"varint,2,opt,name=memory"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *MemoryStat) Reset() { + *x = MemoryStat{} + mi := &file_benchkit_benchkit_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *MemoryStat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MemoryStat) ProtoMessage() {} + +func (x *MemoryStat) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_benchkit_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *MemoryStat) GetAllocs() uint64 { + if x != nil { + return x.xxx_hidden_Allocs + } + return 0 +} + +func (x *MemoryStat) GetMemory() uint64 { + if x != nil { + return x.xxx_hidden_Memory + } + return 0 +} + +func (x *MemoryStat) SetAllocs(v uint64) { + x.xxx_hidden_Allocs = v +} + +func (x *MemoryStat) SetMemory(v uint64) { + x.xxx_hidden_Memory = v +} + +type MemoryStat_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Allocs uint64 + Memory uint64 +} + +func (b0 MemoryStat_builder) Build() *MemoryStat { + m0 := &MemoryStat{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Allocs = b.Allocs + x.xxx_hidden_Memory = b.Memory + return m0 +} + +// Result is one benchmark's complete output for one node: the run +// configuration, the aggregate measured results, and the time-series event +// stream. The startup transient is not removed here; consumers trim it at read +// time using the event offsets (see doc/benchkit.html, section 11). +type Result struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Config *RunConfig `protobuf:"bytes,1,opt,name=config"` + xxx_hidden_TotalOps uint64 `protobuf:"varint,2,opt,name=total_ops,json=totalOps"` + xxx_hidden_TotalTime int64 `protobuf:"varint,3,opt,name=total_time,json=totalTime"` + xxx_hidden_Throughput float64 `protobuf:"fixed64,4,opt,name=throughput"` + xxx_hidden_AllocsPerOp uint64 `protobuf:"varint,5,opt,name=allocs_per_op,json=allocsPerOp"` + xxx_hidden_MemPerOp uint64 `protobuf:"varint,6,opt,name=mem_per_op,json=memPerOp"` + xxx_hidden_ServerStats *[]*MemoryStat `protobuf:"bytes,7,rep,name=server_stats,json=serverStats"` + xxx_hidden_Latencies []int64 `protobuf:"varint,8,rep,packed,name=latencies"` + xxx_hidden_Events *[]*Event `protobuf:"bytes,9,rep,name=events"` + xxx_hidden_Histogram *LatencyHistogram `protobuf:"bytes,10,opt,name=histogram"` + xxx_hidden_FailedOps uint64 `protobuf:"varint,12,opt,name=failed_ops,json=failedOps"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Result) Reset() { + *x = Result{} + mi := &file_benchkit_benchkit_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Result) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Result) ProtoMessage() {} + +func (x *Result) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_benchkit_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *Result) GetConfig() *RunConfig { + if x != nil { + return x.xxx_hidden_Config + } + return nil +} + +func (x *Result) GetTotalOps() uint64 { + if x != nil { + return x.xxx_hidden_TotalOps + } + return 0 +} + +func (x *Result) GetTotalTime() int64 { + if x != nil { + return x.xxx_hidden_TotalTime + } + return 0 +} + +func (x *Result) GetThroughput() float64 { + if x != nil { + return x.xxx_hidden_Throughput + } + return 0 +} + +func (x *Result) GetAllocsPerOp() uint64 { + if x != nil { + return x.xxx_hidden_AllocsPerOp + } + return 0 +} + +func (x *Result) GetMemPerOp() uint64 { + if x != nil { + return x.xxx_hidden_MemPerOp + } + return 0 +} + +func (x *Result) GetServerStats() []*MemoryStat { + if x != nil { + if x.xxx_hidden_ServerStats != nil { + return *x.xxx_hidden_ServerStats + } + } + return nil +} + +func (x *Result) GetLatencies() []int64 { + if x != nil { + return x.xxx_hidden_Latencies + } + return nil +} + +func (x *Result) GetEvents() []*Event { + if x != nil { + if x.xxx_hidden_Events != nil { + return *x.xxx_hidden_Events + } + } + return nil +} + +func (x *Result) GetHistogram() *LatencyHistogram { + if x != nil { + return x.xxx_hidden_Histogram + } + return nil +} + +func (x *Result) GetFailedOps() uint64 { + if x != nil { + return x.xxx_hidden_FailedOps + } + return 0 +} + +func (x *Result) SetConfig(v *RunConfig) { + x.xxx_hidden_Config = v +} + +func (x *Result) SetTotalOps(v uint64) { + x.xxx_hidden_TotalOps = v +} + +func (x *Result) SetTotalTime(v int64) { + x.xxx_hidden_TotalTime = v +} + +func (x *Result) SetThroughput(v float64) { + x.xxx_hidden_Throughput = v +} + +func (x *Result) SetAllocsPerOp(v uint64) { + x.xxx_hidden_AllocsPerOp = v +} + +func (x *Result) SetMemPerOp(v uint64) { + x.xxx_hidden_MemPerOp = v +} + +func (x *Result) SetServerStats(v []*MemoryStat) { + x.xxx_hidden_ServerStats = &v +} + +func (x *Result) SetLatencies(v []int64) { + x.xxx_hidden_Latencies = v +} + +func (x *Result) SetEvents(v []*Event) { + x.xxx_hidden_Events = &v +} + +func (x *Result) SetHistogram(v *LatencyHistogram) { + x.xxx_hidden_Histogram = v +} + +func (x *Result) SetFailedOps(v uint64) { + x.xxx_hidden_FailedOps = v +} + +func (x *Result) HasConfig() bool { + if x == nil { + return false + } + return x.xxx_hidden_Config != nil +} + +func (x *Result) HasHistogram() bool { + if x == nil { + return false + } + return x.xxx_hidden_Histogram != nil +} + +func (x *Result) ClearConfig() { + x.xxx_hidden_Config = nil +} + +func (x *Result) ClearHistogram() { + x.xxx_hidden_Histogram = nil +} + +type Result_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Config *RunConfig + // Aggregate measured results over the whole run. + TotalOps uint64 + TotalTime int64 + Throughput float64 + AllocsPerOp uint64 + MemPerOp uint64 + ServerStats []*MemoryStat + // Raw per-op latency samples, nanoseconds; nil in hdr mode. Signed: + // clock-offset correction (server-measured benchmarks) can yield negatives. + Latencies []int64 + // Time-series event stream covering the whole run; empty when interval = 0. + Events []*Event + // Latency distribution for StatsMode_HDR runs; nil otherwise. + Histogram *LatencyHistogram + // Operations that returned an error and were counted but not aborted on + // (client-measured runs). Under saturation a workload may see failed quorum + // calls; the run completes and records them here instead of exiting. Total + // attempts is total_ops + failed_ops; a high ratio flags an unhealthy run. + FailedOps uint64 +} + +func (b0 Result_builder) Build() *Result { + m0 := &Result{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Config = b.Config + x.xxx_hidden_TotalOps = b.TotalOps + x.xxx_hidden_TotalTime = b.TotalTime + x.xxx_hidden_Throughput = b.Throughput + x.xxx_hidden_AllocsPerOp = b.AllocsPerOp + x.xxx_hidden_MemPerOp = b.MemPerOp + x.xxx_hidden_ServerStats = &b.ServerStats + x.xxx_hidden_Latencies = b.Latencies + x.xxx_hidden_Events = &b.Events + x.xxx_hidden_Histogram = b.Histogram + x.xxx_hidden_FailedOps = b.FailedOps + return m0 +} + +// Report is the per-node container: one labeled set of benchmark results, +// written once per node and suitable for later comparison via -compare. The +// name is deliberately distinct from Result to avoid a one-character typo +// silently compiling against the wrong type. +type Report struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Label string `protobuf:"bytes,1,opt,name=label"` + xxx_hidden_Results *[]*Result `protobuf:"bytes,2,rep,name=results"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Report) Reset() { + *x = Report{} + mi := &file_benchkit_benchkit_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Report) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Report) ProtoMessage() {} + +func (x *Report) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_benchkit_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *Report) GetLabel() string { + if x != nil { + return x.xxx_hidden_Label + } + return "" +} + +func (x *Report) GetResults() []*Result { + if x != nil { + if x.xxx_hidden_Results != nil { + return *x.xxx_hidden_Results + } + } + return nil +} + +func (x *Report) SetLabel(v string) { + x.xxx_hidden_Label = v +} + +func (x *Report) SetResults(v []*Result) { + x.xxx_hidden_Results = &v +} + +type Report_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Label string + Results []*Result +} + +func (b0 Report_builder) Build() *Report { + m0 := &Report{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Label = b.Label + x.xxx_hidden_Results = &b.Results + return m0 +} + +// LatencySummary is a latency distribution reduced to summary statistics, +// microseconds. It is a message rather than inline fields on its parent so an +// absent distribution (a run or node that recorded no latency samples) is a +// nil message instead of a spurious all-zero summary, which would be +// indistinguishable from a real measurement. +type LatencySummary struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_MeanUs float64 `protobuf:"fixed64,1,opt,name=mean_us,json=meanUs"` + xxx_hidden_P50Us float64 `protobuf:"fixed64,2,opt,name=p50_us,json=p50Us"` + xxx_hidden_P95Us float64 `protobuf:"fixed64,3,opt,name=p95_us,json=p95Us"` + xxx_hidden_P99Us float64 `protobuf:"fixed64,4,opt,name=p99_us,json=p99Us"` + xxx_hidden_Samples uint64 `protobuf:"varint,5,opt,name=samples"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LatencySummary) Reset() { + *x = LatencySummary{} + mi := &file_benchkit_benchkit_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LatencySummary) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LatencySummary) ProtoMessage() {} + +func (x *LatencySummary) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_benchkit_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LatencySummary) GetMeanUs() float64 { + if x != nil { + return x.xxx_hidden_MeanUs + } + return 0 +} + +func (x *LatencySummary) GetP50Us() float64 { + if x != nil { + return x.xxx_hidden_P50Us + } + return 0 +} + +func (x *LatencySummary) GetP95Us() float64 { + if x != nil { + return x.xxx_hidden_P95Us + } + return 0 +} + +func (x *LatencySummary) GetP99Us() float64 { + if x != nil { + return x.xxx_hidden_P99Us + } + return 0 +} + +func (x *LatencySummary) GetSamples() uint64 { + if x != nil { + return x.xxx_hidden_Samples + } + return 0 +} + +func (x *LatencySummary) SetMeanUs(v float64) { + x.xxx_hidden_MeanUs = v +} + +func (x *LatencySummary) SetP50Us(v float64) { + x.xxx_hidden_P50Us = v +} + +func (x *LatencySummary) SetP95Us(v float64) { + x.xxx_hidden_P95Us = v +} + +func (x *LatencySummary) SetP99Us(v float64) { + x.xxx_hidden_P99Us = v +} + +func (x *LatencySummary) SetSamples(v uint64) { + x.xxx_hidden_Samples = v +} + +type LatencySummary_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + MeanUs float64 + P50Us float64 + P95Us float64 + P99Us float64 + Samples uint64 +} + +func (b0 LatencySummary_builder) Build() *LatencySummary { + m0 := &LatencySummary{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_MeanUs = b.MeanUs + x.xxx_hidden_P50Us = b.P50Us + x.xxx_hidden_P95Us = b.P95Us + x.xxx_hidden_P99Us = b.P99Us + x.xxx_hidden_Samples = b.Samples + return m0 +} + +// PlotNode is one node's reduced contribution to one benchmark of one run. +type PlotNode struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Node string `protobuf:"bytes,1,opt,name=node"` + xxx_hidden_Throughput float64 `protobuf:"fixed64,2,opt,name=throughput"` + xxx_hidden_Summary *LatencySummary `protobuf:"bytes,3,opt,name=summary"` + xxx_hidden_CdfUs []float64 `protobuf:"fixed64,4,rep,packed,name=cdf_us,json=cdfUs"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PlotNode) Reset() { + *x = PlotNode{} + mi := &file_benchkit_benchkit_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PlotNode) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PlotNode) ProtoMessage() {} + +func (x *PlotNode) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_benchkit_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *PlotNode) GetNode() string { + if x != nil { + return x.xxx_hidden_Node + } + return "" +} + +func (x *PlotNode) GetThroughput() float64 { + if x != nil { + return x.xxx_hidden_Throughput + } + return 0 +} + +func (x *PlotNode) GetSummary() *LatencySummary { + if x != nil { + return x.xxx_hidden_Summary + } + return nil +} + +func (x *PlotNode) GetCdfUs() []float64 { + if x != nil { + return x.xxx_hidden_CdfUs + } + return nil +} + +func (x *PlotNode) SetNode(v string) { + x.xxx_hidden_Node = v +} + +func (x *PlotNode) SetThroughput(v float64) { + x.xxx_hidden_Throughput = v +} + +func (x *PlotNode) SetSummary(v *LatencySummary) { + x.xxx_hidden_Summary = v +} + +func (x *PlotNode) SetCdfUs(v []float64) { + x.xxx_hidden_CdfUs = v +} + +func (x *PlotNode) HasSummary() bool { + if x == nil { + return false + } + return x.xxx_hidden_Summary != nil +} + +func (x *PlotNode) ClearSummary() { + x.xxx_hidden_Summary = nil +} + +type PlotNode_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Node string + Throughput float64 + Summary *LatencySummary + // Latency CDF, microseconds, ascending. The cumulative probability of point + // i is i/(n-1) for n points, so the probability grid is implied by position + // and is not stored. + CdfUs []float64 +} + +func (b0 PlotNode_builder) Build() *PlotNode { + m0 := &PlotNode{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Node = b.Node + x.xxx_hidden_Throughput = b.Throughput + x.xxx_hidden_Summary = b.Summary + x.xxx_hidden_CdfUs = b.CdfUs + return m0 +} + +// PlotBenchmark is one benchmark's reduced results for one run: the run-wide +// aggregate across nodes, plus each node's own reduction. +type PlotBenchmark struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Config *RunConfig `protobuf:"bytes,1,opt,name=config"` + xxx_hidden_Throughput float64 `protobuf:"fixed64,2,opt,name=throughput"` + xxx_hidden_TotalOps uint64 `protobuf:"varint,3,opt,name=total_ops,json=totalOps"` + xxx_hidden_FailedOps uint64 `protobuf:"varint,4,opt,name=failed_ops,json=failedOps"` + xxx_hidden_AllocsPerOp float64 `protobuf:"fixed64,5,opt,name=allocs_per_op,json=allocsPerOp"` + xxx_hidden_MemPerOp float64 `protobuf:"fixed64,6,opt,name=mem_per_op,json=memPerOp"` + xxx_hidden_NodesSeen int32 `protobuf:"varint,7,opt,name=nodes_seen,json=nodesSeen"` + xxx_hidden_Summary *LatencySummary `protobuf:"bytes,8,opt,name=summary"` + xxx_hidden_Nodes *[]*PlotNode `protobuf:"bytes,9,rep,name=nodes"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PlotBenchmark) Reset() { + *x = PlotBenchmark{} + mi := &file_benchkit_benchkit_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PlotBenchmark) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PlotBenchmark) ProtoMessage() {} + +func (x *PlotBenchmark) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_benchkit_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *PlotBenchmark) GetConfig() *RunConfig { + if x != nil { + return x.xxx_hidden_Config + } + return nil +} + +func (x *PlotBenchmark) GetThroughput() float64 { + if x != nil { + return x.xxx_hidden_Throughput + } + return 0 +} + +func (x *PlotBenchmark) GetTotalOps() uint64 { + if x != nil { + return x.xxx_hidden_TotalOps + } + return 0 +} + +func (x *PlotBenchmark) GetFailedOps() uint64 { + if x != nil { + return x.xxx_hidden_FailedOps + } + return 0 +} + +func (x *PlotBenchmark) GetAllocsPerOp() float64 { + if x != nil { + return x.xxx_hidden_AllocsPerOp + } + return 0 +} + +func (x *PlotBenchmark) GetMemPerOp() float64 { + if x != nil { + return x.xxx_hidden_MemPerOp + } + return 0 +} + +func (x *PlotBenchmark) GetNodesSeen() int32 { + if x != nil { + return x.xxx_hidden_NodesSeen + } + return 0 +} + +func (x *PlotBenchmark) GetSummary() *LatencySummary { + if x != nil { + return x.xxx_hidden_Summary + } + return nil +} + +func (x *PlotBenchmark) GetNodes() []*PlotNode { + if x != nil { + if x.xxx_hidden_Nodes != nil { + return *x.xxx_hidden_Nodes + } + } + return nil +} + +func (x *PlotBenchmark) SetConfig(v *RunConfig) { + x.xxx_hidden_Config = v +} + +func (x *PlotBenchmark) SetThroughput(v float64) { + x.xxx_hidden_Throughput = v +} + +func (x *PlotBenchmark) SetTotalOps(v uint64) { + x.xxx_hidden_TotalOps = v +} + +func (x *PlotBenchmark) SetFailedOps(v uint64) { + x.xxx_hidden_FailedOps = v +} + +func (x *PlotBenchmark) SetAllocsPerOp(v float64) { + x.xxx_hidden_AllocsPerOp = v +} + +func (x *PlotBenchmark) SetMemPerOp(v float64) { + x.xxx_hidden_MemPerOp = v +} + +func (x *PlotBenchmark) SetNodesSeen(v int32) { + x.xxx_hidden_NodesSeen = v +} + +func (x *PlotBenchmark) SetSummary(v *LatencySummary) { + x.xxx_hidden_Summary = v +} + +func (x *PlotBenchmark) SetNodes(v []*PlotNode) { + x.xxx_hidden_Nodes = &v +} + +func (x *PlotBenchmark) HasConfig() bool { + if x == nil { + return false + } + return x.xxx_hidden_Config != nil +} + +func (x *PlotBenchmark) HasSummary() bool { + if x == nil { + return false + } + return x.xxx_hidden_Summary != nil +} + +func (x *PlotBenchmark) ClearConfig() { + x.xxx_hidden_Config = nil +} + +func (x *PlotBenchmark) ClearSummary() { + x.xxx_hidden_Summary = nil +} + +type PlotBenchmark_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Config *RunConfig + Throughput float64 + TotalOps uint64 + FailedOps uint64 + // Per-op cost averaged over the reporting nodes. These are doubles, unlike + // their uint64 counterparts in Result, because they are means rather than a + // single node's count. + AllocsPerOp float64 + MemPerOp float64 + NodesSeen int32 + // summary covers the merged sample set across all nodes, so it is not + // derivable from the per-node summaries; nil when no node recorded samples. + Summary *LatencySummary + Nodes []*PlotNode +} + +func (b0 PlotBenchmark_builder) Build() *PlotBenchmark { + m0 := &PlotBenchmark{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Config = b.Config + x.xxx_hidden_Throughput = b.Throughput + x.xxx_hidden_TotalOps = b.TotalOps + x.xxx_hidden_FailedOps = b.FailedOps + x.xxx_hidden_AllocsPerOp = b.AllocsPerOp + x.xxx_hidden_MemPerOp = b.MemPerOp + x.xxx_hidden_NodesSeen = b.NodesSeen + x.xxx_hidden_Summary = b.Summary + x.xxx_hidden_Nodes = &b.Nodes + return m0 +} + +// PlotRun is one run's identity and its per-benchmark reductions. The identity +// fields mirror the run's manifest, so the reduction is self-describing. +type PlotRun struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Base string `protobuf:"bytes,1,opt,name=base"` + xxx_hidden_Label string `protobuf:"bytes,2,opt,name=label"` + xxx_hidden_Status string `protobuf:"bytes,3,opt,name=status"` + xxx_hidden_Rep int32 `protobuf:"varint,4,opt,name=rep"` + xxx_hidden_Benchmarks *[]*PlotBenchmark `protobuf:"bytes,5,rep,name=benchmarks"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PlotRun) Reset() { + *x = PlotRun{} + mi := &file_benchkit_benchkit_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PlotRun) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PlotRun) ProtoMessage() {} + +func (x *PlotRun) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_benchkit_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *PlotRun) GetBase() string { + if x != nil { + return x.xxx_hidden_Base + } + return "" +} + +func (x *PlotRun) GetLabel() string { + if x != nil { + return x.xxx_hidden_Label + } + return "" +} + +func (x *PlotRun) GetStatus() string { + if x != nil { + return x.xxx_hidden_Status + } + return "" +} + +func (x *PlotRun) GetRep() int32 { + if x != nil { + return x.xxx_hidden_Rep + } + return 0 +} + +func (x *PlotRun) GetBenchmarks() []*PlotBenchmark { + if x != nil { + if x.xxx_hidden_Benchmarks != nil { + return *x.xxx_hidden_Benchmarks + } + } + return nil +} + +func (x *PlotRun) SetBase(v string) { + x.xxx_hidden_Base = v +} + +func (x *PlotRun) SetLabel(v string) { + x.xxx_hidden_Label = v +} + +func (x *PlotRun) SetStatus(v string) { + x.xxx_hidden_Status = v +} + +func (x *PlotRun) SetRep(v int32) { + x.xxx_hidden_Rep = v +} + +func (x *PlotRun) SetBenchmarks(v []*PlotBenchmark) { + x.xxx_hidden_Benchmarks = &v +} + +type PlotRun_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Base string + Label string + Status string + Rep int32 + Benchmarks []*PlotBenchmark +} + +func (b0 PlotRun_builder) Build() *PlotRun { + m0 := &PlotRun{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Base = b.Base + x.xxx_hidden_Label = b.Label + x.xxx_hidden_Status = b.Status + x.xxx_hidden_Rep = b.Rep + x.xxx_hidden_Benchmarks = &b.Benchmarks + return m0 +} + +// PlotData is a whole sweep reduced for plotting: one entry per run that a +// consumer chose to retain. Having a single repeated field means two encoded +// PlotData messages concatenate into a valid message holding both sweeps' runs. +type PlotData struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Runs *[]*PlotRun `protobuf:"bytes,1,rep,name=runs"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PlotData) Reset() { + *x = PlotData{} + mi := &file_benchkit_benchkit_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PlotData) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PlotData) ProtoMessage() {} + +func (x *PlotData) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_benchkit_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *PlotData) GetRuns() []*PlotRun { + if x != nil { + if x.xxx_hidden_Runs != nil { + return *x.xxx_hidden_Runs + } + } + return nil +} + +func (x *PlotData) SetRuns(v []*PlotRun) { + x.xxx_hidden_Runs = &v +} + +type PlotData_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Runs []*PlotRun +} + +func (b0 PlotData_builder) Build() *PlotData { + m0 := &PlotData{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Runs = &b.Runs + return m0 +} + +// PlotNodeEvents is one node's event stream for one benchmark of one run. +type PlotNodeEvents struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Node string `protobuf:"bytes,1,opt,name=node"` + xxx_hidden_Events *[]*Event `protobuf:"bytes,2,rep,name=events"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PlotNodeEvents) Reset() { + *x = PlotNodeEvents{} + mi := &file_benchkit_benchkit_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PlotNodeEvents) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PlotNodeEvents) ProtoMessage() {} + +func (x *PlotNodeEvents) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_benchkit_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *PlotNodeEvents) GetNode() string { + if x != nil { + return x.xxx_hidden_Node + } + return "" +} + +func (x *PlotNodeEvents) GetEvents() []*Event { + if x != nil { + if x.xxx_hidden_Events != nil { + return *x.xxx_hidden_Events + } + } + return nil +} + +func (x *PlotNodeEvents) SetNode(v string) { + x.xxx_hidden_Node = v +} + +func (x *PlotNodeEvents) SetEvents(v []*Event) { + x.xxx_hidden_Events = &v +} + +type PlotNodeEvents_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Node string + Events []*Event +} + +func (b0 PlotNodeEvents_builder) Build() *PlotNodeEvents { + m0 := &PlotNodeEvents{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Node = b.Node + x.xxx_hidden_Events = &b.Events + return m0 +} + +// PlotBenchmarkEvents holds every node's event stream for one benchmark of one run. +type PlotBenchmarkEvents struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Benchmark string `protobuf:"bytes,1,opt,name=benchmark"` + xxx_hidden_Nodes *[]*PlotNodeEvents `protobuf:"bytes,2,rep,name=nodes"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PlotBenchmarkEvents) Reset() { + *x = PlotBenchmarkEvents{} + mi := &file_benchkit_benchkit_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PlotBenchmarkEvents) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PlotBenchmarkEvents) ProtoMessage() {} + +func (x *PlotBenchmarkEvents) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_benchkit_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *PlotBenchmarkEvents) GetBenchmark() string { + if x != nil { + return x.xxx_hidden_Benchmark + } + return "" +} + +func (x *PlotBenchmarkEvents) GetNodes() []*PlotNodeEvents { + if x != nil { + if x.xxx_hidden_Nodes != nil { + return *x.xxx_hidden_Nodes + } + } + return nil +} + +func (x *PlotBenchmarkEvents) SetBenchmark(v string) { + x.xxx_hidden_Benchmark = v +} + +func (x *PlotBenchmarkEvents) SetNodes(v []*PlotNodeEvents) { + x.xxx_hidden_Nodes = &v +} + +type PlotBenchmarkEvents_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Benchmark string + Nodes []*PlotNodeEvents +} + +func (b0 PlotBenchmarkEvents_builder) Build() *PlotBenchmarkEvents { + m0 := &PlotBenchmarkEvents{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Benchmark = b.Benchmark + x.xxx_hidden_Nodes = &b.Nodes + return m0 +} + +// PlotRunEvents holds one run's per-benchmark event streams. The run's identity +// beyond its base name (status, trim, dimensions) lives in its manifest, which +// travels alongside, so it is not duplicated here. +type PlotRunEvents struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Base string `protobuf:"bytes,1,opt,name=base"` + xxx_hidden_Benchmarks *[]*PlotBenchmarkEvents `protobuf:"bytes,2,rep,name=benchmarks"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PlotRunEvents) Reset() { + *x = PlotRunEvents{} + mi := &file_benchkit_benchkit_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PlotRunEvents) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PlotRunEvents) ProtoMessage() {} + +func (x *PlotRunEvents) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_benchkit_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *PlotRunEvents) GetBase() string { + if x != nil { + return x.xxx_hidden_Base + } + return "" +} + +func (x *PlotRunEvents) GetBenchmarks() []*PlotBenchmarkEvents { + if x != nil { + if x.xxx_hidden_Benchmarks != nil { + return *x.xxx_hidden_Benchmarks + } + } + return nil +} + +func (x *PlotRunEvents) SetBase(v string) { + x.xxx_hidden_Base = v +} + +func (x *PlotRunEvents) SetBenchmarks(v []*PlotBenchmarkEvents) { + x.xxx_hidden_Benchmarks = &v +} + +type PlotRunEvents_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Base string + Benchmarks []*PlotBenchmarkEvents +} + +func (b0 PlotRunEvents_builder) Build() *PlotRunEvents { + m0 := &PlotRunEvents{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Base = b.Base + x.xxx_hidden_Benchmarks = &b.Benchmarks + return m0 +} + +// PlotEvents is a whole sweep's event streams. As with PlotData, the single +// repeated field means two encoded messages concatenate into a valid one. +type PlotEvents struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Runs *[]*PlotRunEvents `protobuf:"bytes,1,rep,name=runs"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PlotEvents) Reset() { + *x = PlotEvents{} + mi := &file_benchkit_benchkit_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PlotEvents) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PlotEvents) ProtoMessage() {} + +func (x *PlotEvents) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_benchkit_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *PlotEvents) GetRuns() []*PlotRunEvents { + if x != nil { + if x.xxx_hidden_Runs != nil { + return *x.xxx_hidden_Runs + } + } + return nil +} + +func (x *PlotEvents) SetRuns(v []*PlotRunEvents) { + x.xxx_hidden_Runs = &v +} + +type PlotEvents_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Runs []*PlotRunEvents +} + +func (b0 PlotEvents_builder) Build() *PlotEvents { + m0 := &PlotEvents{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Runs = &b.Runs + return m0 +} + +var File_benchkit_benchkit_proto protoreflect.FileDescriptor + +const file_benchkit_benchkit_proto_rawDesc = "" + + "\n" + + "\x17benchkit/benchkit.proto\x12\bbenchkit\"\xcf\x04\n" + + "\tRunConfig\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1b\n" + + "\tnum_nodes\x18\x02 \x01(\x05R\bnumNodes\x12\x12\n" + + "\x04mode\x18\x03 \x01(\tR\x04mode\x12\x1a\n" + + "\bduration\x18\x04 \x01(\x03R\bduration\x12\x18\n" + + "\aworkers\x18\x05 \x01(\x05R\aworkers\x12\x18\n" + + "\apayload\x18\x06 \x01(\x05R\apayload\x12\x12\n" + + "\x04rate\x18\a \x01(\x03R\x04rate\x12\x1a\n" + + "\binterval\x18\b \x01(\x03R\binterval\x12D\n" + + "\x10measurement_mode\x18\t \x01(\x0e2\x19.benchkit.MeasurementModeR\x0fmeasurementMode\x122\n" + + "\n" + + "stats_mode\x18\n" + + " \x01(\x0e2\x13.benchkit.StatsModeR\tstatsMode\x12\x1f\n" + + "\vstream_mode\x18\v \x01(\tR\n" + + "streamMode\x12\x1f\n" + + "\vquorum_size\x18\f \x01(\x05R\n" + + "quorumSize\x12\x1b\n" + + "\tmax_async\x18\r \x01(\x05R\bmaxAsync\x12\x1b\n" + + "\trate_step\x18\x0e \x01(\x03R\brateStep\x12\"\n" + + "\rrate_step_max\x18\x0f \x01(\x03R\vrateStepMax\x12!\n" + + "\fcall_timeout\x18\x10 \x01(\x03R\vcallTimeout\x12\x1f\n" + + "\vsend_buffer\x18\x11 \x01(\x05R\n" + + "sendBuffer\x12\x1f\n" + + "\vrecv_buffer\x18\x12 \x01(\x05R\n" + + "recvBuffer\"B\n" + + "\x12ThroughputInterval\x12\x10\n" + + "\x03ops\x18\x01 \x01(\x04R\x03ops\x12\x1a\n" + + "\bduration\x18\x02 \x01(\x03R\bduration\"S\n" + + "\x0fLatencyInterval\x12\x12\n" + + "\x04mean\x18\x01 \x01(\x01R\x04mean\x12\x16\n" + + "\x06stddev\x18\x02 \x01(\x01R\x06stddev\x12\x14\n" + + "\x05count\x18\x03 \x01(\x04R\x05count\"\x81\x01\n" + + "\vPhaseMarker\x121\n" + + "\x05phase\x18\x01 \x01(\x0e2\x1b.benchkit.PhaseMarker.PhaseR\x05phase\x12\x12\n" + + "\x04rate\x18\x02 \x01(\x03R\x04rate\"+\n" + + "\x05Phase\x12\t\n" + + "\x05START\x10\x00\x12\r\n" + + "\tRATE_STEP\x10\x01\x12\b\n" + + "\x04STOP\x10\x02\"\xd0\x01\n" + + "\x05Event\x12\x16\n" + + "\x06offset\x18\x01 \x01(\x03R\x06offset\x12>\n" + + "\n" + + "throughput\x18\x02 \x01(\v2\x1c.benchkit.ThroughputIntervalH\x00R\n" + + "throughput\x125\n" + + "\alatency\x18\x03 \x01(\v2\x19.benchkit.LatencyIntervalH\x00R\alatency\x12-\n" + + "\x05phase\x18\x04 \x01(\v2\x15.benchkit.PhaseMarkerH\x00R\x05phaseB\t\n" + + "\apayload\">\n" + + "\x10LatencyHistogram\x12\x14\n" + + "\x05value\x18\x01 \x03(\x03R\x05value\x12\x14\n" + + "\x05count\x18\x02 \x03(\x04R\x05count\"<\n" + + "\n" + + "MemoryStat\x12\x16\n" + + "\x06allocs\x18\x01 \x01(\x04R\x06allocs\x12\x16\n" + + "\x06memory\x18\x02 \x01(\x04R\x06memory\"\xc0\x03\n" + + "\x06Result\x12+\n" + + "\x06config\x18\x01 \x01(\v2\x13.benchkit.RunConfigR\x06config\x12\x1b\n" + + "\ttotal_ops\x18\x02 \x01(\x04R\btotalOps\x12\x1d\n" + + "\n" + + "total_time\x18\x03 \x01(\x03R\ttotalTime\x12\x1e\n" + + "\n" + + "throughput\x18\x04 \x01(\x01R\n" + + "throughput\x12\"\n" + + "\rallocs_per_op\x18\x05 \x01(\x04R\vallocsPerOp\x12\x1c\n" + + "\n" + + "mem_per_op\x18\x06 \x01(\x04R\bmemPerOp\x127\n" + + "\fserver_stats\x18\a \x03(\v2\x14.benchkit.MemoryStatR\vserverStats\x12\x1c\n" + + "\tlatencies\x18\b \x03(\x03R\tlatencies\x12'\n" + + "\x06events\x18\t \x03(\v2\x0f.benchkit.EventR\x06events\x128\n" + + "\thistogram\x18\n" + + " \x01(\v2\x1a.benchkit.LatencyHistogramR\thistogram\x12\x1d\n" + + "\n" + + "failed_ops\x18\f \x01(\x04R\tfailedOpsJ\x04\b\v\x10\fR\fstream_stats\"J\n" + + "\x06Report\x12\x14\n" + + "\x05label\x18\x01 \x01(\tR\x05label\x12*\n" + + "\aresults\x18\x02 \x03(\v2\x10.benchkit.ResultR\aresults\"\x88\x01\n" + + "\x0eLatencySummary\x12\x17\n" + + "\amean_us\x18\x01 \x01(\x01R\x06meanUs\x12\x15\n" + + "\x06p50_us\x18\x02 \x01(\x01R\x05p50Us\x12\x15\n" + + "\x06p95_us\x18\x03 \x01(\x01R\x05p95Us\x12\x15\n" + + "\x06p99_us\x18\x04 \x01(\x01R\x05p99Us\x12\x18\n" + + "\asamples\x18\x05 \x01(\x04R\asamples\"\x89\x01\n" + + "\bPlotNode\x12\x12\n" + + "\x04node\x18\x01 \x01(\tR\x04node\x12\x1e\n" + + "\n" + + "throughput\x18\x02 \x01(\x01R\n" + + "throughput\x122\n" + + "\asummary\x18\x03 \x01(\v2\x18.benchkit.LatencySummaryR\asummary\x12\x15\n" + + "\x06cdf_us\x18\x04 \x03(\x01R\x05cdfUs\"\xd7\x02\n" + + "\rPlotBenchmark\x12+\n" + + "\x06config\x18\x01 \x01(\v2\x13.benchkit.RunConfigR\x06config\x12\x1e\n" + + "\n" + + "throughput\x18\x02 \x01(\x01R\n" + + "throughput\x12\x1b\n" + + "\ttotal_ops\x18\x03 \x01(\x04R\btotalOps\x12\x1d\n" + + "\n" + + "failed_ops\x18\x04 \x01(\x04R\tfailedOps\x12\"\n" + + "\rallocs_per_op\x18\x05 \x01(\x01R\vallocsPerOp\x12\x1c\n" + + "\n" + + "mem_per_op\x18\x06 \x01(\x01R\bmemPerOp\x12\x1d\n" + + "\n" + + "nodes_seen\x18\a \x01(\x05R\tnodesSeen\x122\n" + + "\asummary\x18\b \x01(\v2\x18.benchkit.LatencySummaryR\asummary\x12(\n" + + "\x05nodes\x18\t \x03(\v2\x12.benchkit.PlotNodeR\x05nodes\"\x96\x01\n" + + "\aPlotRun\x12\x12\n" + + "\x04base\x18\x01 \x01(\tR\x04base\x12\x14\n" + + "\x05label\x18\x02 \x01(\tR\x05label\x12\x16\n" + + "\x06status\x18\x03 \x01(\tR\x06status\x12\x10\n" + + "\x03rep\x18\x04 \x01(\x05R\x03rep\x127\n" + + "\n" + + "benchmarks\x18\x05 \x03(\v2\x17.benchkit.PlotBenchmarkR\n" + + "benchmarks\"1\n" + + "\bPlotData\x12%\n" + + "\x04runs\x18\x01 \x03(\v2\x11.benchkit.PlotRunR\x04runs\"M\n" + + "\x0ePlotNodeEvents\x12\x12\n" + + "\x04node\x18\x01 \x01(\tR\x04node\x12'\n" + + "\x06events\x18\x02 \x03(\v2\x0f.benchkit.EventR\x06events\"c\n" + + "\x13PlotBenchmarkEvents\x12\x1c\n" + + "\tbenchmark\x18\x01 \x01(\tR\tbenchmark\x12.\n" + + "\x05nodes\x18\x02 \x03(\v2\x18.benchkit.PlotNodeEventsR\x05nodes\"b\n" + + "\rPlotRunEvents\x12\x12\n" + + "\x04base\x18\x01 \x01(\tR\x04base\x12=\n" + + "\n" + + "benchmarks\x18\x02 \x03(\v2\x1d.benchkit.PlotBenchmarkEventsR\n" + + "benchmarks\"9\n" + + "\n" + + "PlotEvents\x12+\n" + + "\x04runs\x18\x01 \x03(\v2\x17.benchkit.PlotRunEventsR\x04runs*;\n" + + "\x0fMeasurementMode\x12\x13\n" + + "\x0fCLIENT_MEASURED\x10\x00\x12\x13\n" + + "\x0fSERVER_MEASURED\x10\x01*%\n" + + "\tStatsMode\x12\t\n" + + "\x05EXACT\x10\x00\x12\a\n" + + "\x03HDR\x10\x02\"\x04\b\x01\x10\x01B'Z github.com/relab/gorums/benchkit\x92\x03\x02\b\x02b\beditionsp\xe9\a" + +var file_benchkit_benchkit_proto_enumTypes = make([]protoimpl.EnumInfo, 3) +var file_benchkit_benchkit_proto_msgTypes = make([]protoimpl.MessageInfo, 18) +var file_benchkit_benchkit_proto_goTypes = []any{ + (MeasurementMode)(0), // 0: benchkit.MeasurementMode + (StatsMode)(0), // 1: benchkit.StatsMode + (PhaseMarker_Phase)(0), // 2: benchkit.PhaseMarker.Phase + (*RunConfig)(nil), // 3: benchkit.RunConfig + (*ThroughputInterval)(nil), // 4: benchkit.ThroughputInterval + (*LatencyInterval)(nil), // 5: benchkit.LatencyInterval + (*PhaseMarker)(nil), // 6: benchkit.PhaseMarker + (*Event)(nil), // 7: benchkit.Event + (*LatencyHistogram)(nil), // 8: benchkit.LatencyHistogram + (*MemoryStat)(nil), // 9: benchkit.MemoryStat + (*Result)(nil), // 10: benchkit.Result + (*Report)(nil), // 11: benchkit.Report + (*LatencySummary)(nil), // 12: benchkit.LatencySummary + (*PlotNode)(nil), // 13: benchkit.PlotNode + (*PlotBenchmark)(nil), // 14: benchkit.PlotBenchmark + (*PlotRun)(nil), // 15: benchkit.PlotRun + (*PlotData)(nil), // 16: benchkit.PlotData + (*PlotNodeEvents)(nil), // 17: benchkit.PlotNodeEvents + (*PlotBenchmarkEvents)(nil), // 18: benchkit.PlotBenchmarkEvents + (*PlotRunEvents)(nil), // 19: benchkit.PlotRunEvents + (*PlotEvents)(nil), // 20: benchkit.PlotEvents +} +var file_benchkit_benchkit_proto_depIdxs = []int32{ + 0, // 0: benchkit.RunConfig.measurement_mode:type_name -> benchkit.MeasurementMode + 1, // 1: benchkit.RunConfig.stats_mode:type_name -> benchkit.StatsMode + 2, // 2: benchkit.PhaseMarker.phase:type_name -> benchkit.PhaseMarker.Phase + 4, // 3: benchkit.Event.throughput:type_name -> benchkit.ThroughputInterval + 5, // 4: benchkit.Event.latency:type_name -> benchkit.LatencyInterval + 6, // 5: benchkit.Event.phase:type_name -> benchkit.PhaseMarker + 3, // 6: benchkit.Result.config:type_name -> benchkit.RunConfig + 9, // 7: benchkit.Result.server_stats:type_name -> benchkit.MemoryStat + 7, // 8: benchkit.Result.events:type_name -> benchkit.Event + 8, // 9: benchkit.Result.histogram:type_name -> benchkit.LatencyHistogram + 10, // 10: benchkit.Report.results:type_name -> benchkit.Result + 12, // 11: benchkit.PlotNode.summary:type_name -> benchkit.LatencySummary + 3, // 12: benchkit.PlotBenchmark.config:type_name -> benchkit.RunConfig + 12, // 13: benchkit.PlotBenchmark.summary:type_name -> benchkit.LatencySummary + 13, // 14: benchkit.PlotBenchmark.nodes:type_name -> benchkit.PlotNode + 14, // 15: benchkit.PlotRun.benchmarks:type_name -> benchkit.PlotBenchmark + 15, // 16: benchkit.PlotData.runs:type_name -> benchkit.PlotRun + 7, // 17: benchkit.PlotNodeEvents.events:type_name -> benchkit.Event + 17, // 18: benchkit.PlotBenchmarkEvents.nodes:type_name -> benchkit.PlotNodeEvents + 18, // 19: benchkit.PlotRunEvents.benchmarks:type_name -> benchkit.PlotBenchmarkEvents + 19, // 20: benchkit.PlotEvents.runs:type_name -> benchkit.PlotRunEvents + 21, // [21:21] is the sub-list for method output_type + 21, // [21:21] is the sub-list for method input_type + 21, // [21:21] is the sub-list for extension type_name + 21, // [21:21] is the sub-list for extension extendee + 0, // [0:21] is the sub-list for field type_name +} + +func init() { file_benchkit_benchkit_proto_init() } +func file_benchkit_benchkit_proto_init() { + if File_benchkit_benchkit_proto != nil { + return + } + file_benchkit_benchkit_proto_msgTypes[4].OneofWrappers = []any{ + (*event_Throughput)(nil), + (*event_Latency)(nil), + (*event_Phase)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_benchkit_benchkit_proto_rawDesc), len(file_benchkit_benchkit_proto_rawDesc)), + NumEnums: 3, + NumMessages: 18, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_benchkit_benchkit_proto_goTypes, + DependencyIndexes: file_benchkit_benchkit_proto_depIdxs, + EnumInfos: file_benchkit_benchkit_proto_enumTypes, + MessageInfos: file_benchkit_benchkit_proto_msgTypes, + }.Build() + File_benchkit_benchkit_proto = out.File + file_benchkit_benchkit_proto_goTypes = nil + file_benchkit_benchkit_proto_depIdxs = nil +} diff --git a/benchkit/clocksync.go b/benchkit/clocksync.go new file mode 100644 index 00000000..5199eed9 --- /dev/null +++ b/benchkit/clocksync.go @@ -0,0 +1,127 @@ +package benchkit + +import ( + "context" + "fmt" + "maps" + "slices" + "time" + + "github.com/relab/gorums" +) + +// clockSyncRounds is the number of NTP-style ClockSync exchanges performed per +// EstimateOffsets call. Per peer, the estimate from the round with the smallest +// round-trip delay is kept, following NTP's min-filter heuristic to suppress +// queuing noise on the probe path. More rounds raise the chance of catching a +// low-jitter sample and so tighten the offset estimate, at a setup cost of +// round-trips x rounds per call (paid twice, before and after the window); on a +// LAN that is negligible, so this is set well above NTP's typical burst of 4-8. +// It cannot correct systematic path asymmetry, which no number of rounds removes. +const clockSyncRounds = 50 + +// clockOffset computes the NTP clock offset and round-trip delay for a single +// exchange: t1 is the local time just before the request, serverTime is the +// server's time stamped in the reply, and t4 is the local time the reply +// arrived. Assuming a symmetric path, the offset (peer clock minus this node's +// clock) is serverTime - (t1+t4)/2 and the delay is t4 - t1. +func clockOffset(t1, serverTime, t4 int64) (offset, delay int64) { + return serverTime - (t1+t4)/2, t4 - t1 +} + +// EstimateOffsets runs clockSyncRounds NTP-style ClockSync exchanges from cfg's +// client to every peer and returns, per peer node ID, the estimated clock offset +// in nanoseconds (peer clock minus this node's clock). +// +// For each reply, with t1 the local time just before the call, t4 the local +// time the reply arrived, and st the server's time stamped in the reply, the +// offset is theta = st - (t1+t4)/2 and the round-trip delay is delay = t4 - t1 +// (assuming a symmetric path). The offset from the round with the smallest delay +// is kept for each peer. The returned offset of a node relative to itself +// (loopback) is approximately zero. +func EstimateOffsets(ctx context.Context, cfg gorums.Config) (map[uint32]int64, error) { + cfgCtx := cfg.Context(ctx) + n := cfg.Size() + bestDelay := make(map[uint32]int64, n) + offsets := make(map[uint32]int64, n) + for range clockSyncRounds { + t1 := time.Now().UnixNano() + for r := range ClockSync(cfgCtx, &ClockSyncRequest{}).Results() { + t4 := time.Now().UnixNano() + if r.Err != nil { + continue + } + theta, delay := clockOffset(t1, r.Value.GetServerTime(), t4) + if best, ok := bestDelay[r.NodeID]; !ok || delay < best { + bestDelay[r.NodeID] = delay + offsets[r.NodeID] = theta + } + } + } + if len(offsets) < n { + return nil, fmt.Errorf("clock sync incomplete: got offsets for %d of %d peers", len(offsets), n) + } + return offsets, nil +} + +// CorrectLatencies subtracts the given clock offset (peer clock minus this +// node's clock, in nanoseconds) from every latency sample in r, removing the +// cross-machine clock skew baked into a server-measured one-way latency. Used +// coordinator-side where a server's samples all come from a single sender. +// +// In StatsMode_HDR, where r carries a histogram instead of raw samples, the +// subtraction is applied to the histogram bucket values and the result is +// re-quantized onto the canonical HDR layout: the offset is a per-server +// additive constant, so it shifts the distribution without changing its shape. +func CorrectLatencies(r *Result, offset int64) { + if offset == 0 { + return + } + if lat := r.GetLatencies(); len(lat) > 0 { + for i := range lat { + lat[i] -= offset + } + r.SetLatencies(lat) + return + } + if h := r.GetHistogram(); h != nil { + r.SetHistogram(offsetHistogram(h, -offset)) + } +} + +// LogOffsets prints the estimated per-peer clock offsets and the drift between +// the before and after samples. The values are diagnostics: a large offset +// indicates significant clock skew between machines, and a large drift suggests +// the clocks moved relative to each other during the run. The self/loopback +// peer should report an offset near zero. It is emitted unconditionally (via +// benchkit.Printf, not the -verbose Logf) because the offsets document how a +// server-measured latency was corrected: recording them in every run's collected +// log lets a corrected result — including one whose smallest samples land below +// zero from residual estimation error — be audited after the fact. +func LogOffsets(label string, before, after map[uint32]int64) { + for _, id := range slices.Sorted(maps.Keys(before)) { + b, a := before[id], after[id] + Printf("[offsets %s] peer %d: before=%v after=%v drift=%v\n", + label, id, time.Duration(b), time.Duration(a), time.Duration(a-b)) + } +} + +// AverageOffsets returns the per-key mean of two offset maps. A key present in +// only one map is carried through unchanged, so a transient gap in one sample +// does not drop a peer's correction entirely. +func AverageOffsets(a, b map[uint32]int64) map[uint32]int64 { + out := make(map[uint32]int64, len(a)) + for id, va := range a { + if vb, ok := b[id]; ok { + out[id] = (va + vb) / 2 + } else { + out[id] = va + } + } + for id, vb := range b { + if _, ok := a[id]; !ok { + out[id] = vb + } + } + return out +} diff --git a/benchkit/clocksync_test.go b/benchkit/clocksync_test.go new file mode 100644 index 00000000..00652afb --- /dev/null +++ b/benchkit/clocksync_test.go @@ -0,0 +1,162 @@ +package benchkit + +import ( + "io" + "maps" + "math" + "os" + "slices" + "strings" + "testing" + "time" +) + +func TestClockOffset(t *testing.T) { + tests := []struct { + name string + t1, serverTime, t4 int64 + wantOffset, wantDelay int64 + }{ + {name: "PeerAhead", t1: 1000, serverTime: 1600, t4: 1200, wantOffset: 500, wantDelay: 200}, + {name: "PeerBehind", t1: 1000, serverTime: 800, t4: 1200, wantOffset: -300, wantDelay: 200}, + {name: "SameClock", t1: 1000, serverTime: 1100, t4: 1200, wantOffset: 0, wantDelay: 200}, + {name: "ZeroDelay", t1: 5000, serverTime: 5000, t4: 5000, wantOffset: 0, wantDelay: 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + offset, delay := clockOffset(tt.t1, tt.serverTime, tt.t4) + if offset != tt.wantOffset || delay != tt.wantDelay { + t.Errorf("clockOffset(%d, %d, %d) = (%d, %d), want (%d, %d)", + tt.t1, tt.serverTime, tt.t4, offset, delay, tt.wantOffset, tt.wantDelay) + } + }) + } +} + +func TestAverageOffsets(t *testing.T) { + tests := []struct { + name string + a, b map[uint32]int64 + want map[uint32]int64 + }{ + { + name: "SharedKeys", + a: map[uint32]int64{1: 100, 2: -40}, + b: map[uint32]int64{1: 200, 2: 0}, + want: map[uint32]int64{1: 150, 2: -20}, + }, + { + name: "KeyOnlyInA", + a: map[uint32]int64{1: 100, 3: 50}, + b: map[uint32]int64{1: 100}, + want: map[uint32]int64{1: 100, 3: 50}, + }, + { + name: "KeyOnlyInB", + a: map[uint32]int64{1: 100}, + b: map[uint32]int64{1: 100, 4: 80}, + want: map[uint32]int64{1: 100, 4: 80}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := AverageOffsets(tt.a, tt.b) + if !maps.Equal(got, tt.want) { + t.Errorf("AverageOffsets = %v, want %v", got, tt.want) + } + }) + } +} + +func TestCorrectLatencies(t *testing.T) { + tests := []struct { + name string + in []int64 + offset int64 + want []int64 + }{ + {name: "SubtractPositive", in: []int64{300, 400, 450}, offset: 100, want: []int64{200, 300, 350}}, + {name: "SubtractNegative", in: []int64{200, 100}, offset: -50, want: []int64{250, 150}}, + {name: "ZeroOffsetUnchanged", in: []int64{10, 20}, offset: 0, want: []int64{10, 20}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := Result_builder{Latencies: slices.Clone(tt.in)}.Build() + CorrectLatencies(r, tt.offset) + if got := r.GetLatencies(); !slices.Equal(got, tt.want) { + t.Errorf("CorrectLatencies(%v, %d) = %v, want %v", tt.in, tt.offset, got, tt.want) + } + }) + } +} + +// TestCorrectLatenciesHistogram verifies that in StatsMode_HDR, where a result +// carries a histogram instead of raw samples, CorrectLatencies subtracts the +// clock offset from the histogram bucket values (re-quantized onto the canonical +// layout) while preserving the sample count, and leaves a zero-offset result +// untouched. +func TestCorrectLatenciesHistogram(t *testing.T) { + src := hist(20_000, 20_000, 20_000) // 3 samples at 20µs + + r := Result_builder{Histogram: src}.Build() + CorrectLatencies(r, 5_000) // subtract 5µs + if got := totalCount(r.GetHistogram()); got != 3 { + t.Fatalf("count after correction = %d, want 3", got) + } + if got := p50(r.GetHistogram()); math.Abs(float64(got-15*time.Microsecond)) > 50 { + t.Errorf("p50 after -5µs correction = %v, want ≈15µs", got) + } + + // A zero offset must leave the histogram untouched (same pointer, no + // re-quantization). + unchanged := Result_builder{Histogram: src}.Build() + CorrectLatencies(unchanged, 0) + if unchanged.GetHistogram() != src { + t.Error("zero-offset correction replaced the histogram, want unchanged") + } +} + +// captureStderr redirects os.Stderr for the duration of f and returns everything +// written to it, restoring the original stderr before returning. +func captureStderr(t *testing.T, f func()) string { + t.Helper() + orig := os.Stderr + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("pipe: %v", err) + } + os.Stderr = w + done := make(chan string, 1) + go func() { + var b strings.Builder + _, _ = io.Copy(&b, r) + done <- b.String() + }() + f() + _ = w.Close() + os.Stderr = orig + return <-done +} + +// TestLogOffsetsAlwaysEmits verifies the clock-offset summary is written to +// stderr even when verbose logging is disabled, so a sweep's collected per-run +// log always records how a server-measured latency was corrected. It guards the +// switch from the -verbose Logf to the unconditional benchkit.Printf. +func TestLogOffsetsAlwaysEmits(t *testing.T) { + SetVerbose(false) + before := map[uint32]int64{5: 1_000_000, 2: -2_000_000} + after := map[uint32]int64{5: 1_500_000, 2: -2_000_000} + out := captureStderr(t, func() { LogOffsets("servers", before, after) }) + + for _, want := range []string{ + // Peers are printed in sorted node-ID order. + "[offsets servers] peer 2:", + "[offsets servers] peer 5:", + // Drift is after minus before; peer 5 moved by 500µs. + "drift=500µs", + } { + if !strings.Contains(out, want) { + t.Errorf("offset log missing %q; got:\n%s", want, out) + } + } +} diff --git a/benchkit/control.pb.go b/benchkit/control.pb.go new file mode 100644 index 00000000..0276b774 --- /dev/null +++ b/benchkit/control.pb.go @@ -0,0 +1,456 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v7.35.1 +// source: benchkit/control.proto + +package benchkit + +import ( + _ "github.com/relab/gorums" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// StartRequest starts a benchmarking campaign and selects the aggregate +// latency backing store the server builds for the run, so a server-measured +// benchmark honors the client's -stats-mode. +type StartRequest struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_StatsMode StatsMode `protobuf:"varint,1,opt,name=stats_mode,json=statsMode,enum=benchkit.StatsMode"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StartRequest) Reset() { + *x = StartRequest{} + mi := &file_benchkit_control_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StartRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StartRequest) ProtoMessage() {} + +func (x *StartRequest) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_control_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *StartRequest) GetStatsMode() StatsMode { + if x != nil { + return x.xxx_hidden_StatsMode + } + return StatsMode_EXACT +} + +func (x *StartRequest) SetStatsMode(v StatsMode) { + x.xxx_hidden_StatsMode = v +} + +type StartRequest_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + StatsMode StatsMode +} + +func (b0 StartRequest_builder) Build() *StartRequest { + m0 := &StartRequest{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_StatsMode = b.StatsMode + return m0 +} + +// StartResponse is an empty message to acknowledge the start of a benchmarking campaign. +type StartResponse struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StartResponse) Reset() { + *x = StartResponse{} + mi := &file_benchkit_control_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StartResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StartResponse) ProtoMessage() {} + +func (x *StartResponse) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_control_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type StartResponse_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 StartResponse_builder) Build() *StartResponse { + m0 := &StartResponse{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// StopRequest is an empty message for stopping a benchmarking campaign. +type StopRequest struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StopRequest) Reset() { + *x = StopRequest{} + mi := &file_benchkit_control_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StopRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StopRequest) ProtoMessage() {} + +func (x *StopRequest) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_control_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type StopRequest_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 StopRequest_builder) Build() *StopRequest { + m0 := &StopRequest{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// ClockSyncRequest is an empty message for requesting the server's wall clock. +type ClockSyncRequest struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ClockSyncRequest) Reset() { + *x = ClockSyncRequest{} + mi := &file_benchkit_control_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ClockSyncRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClockSyncRequest) ProtoMessage() {} + +func (x *ClockSyncRequest) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_control_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type ClockSyncRequest_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 ClockSyncRequest_builder) Build() *ClockSyncRequest { + m0 := &ClockSyncRequest{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// ClockSyncResponse carries the server's wall-clock reading, used for NTP-style +// clock-offset estimation between peers when correcting one-way latencies. +type ClockSyncResponse struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ServerTime int64 `protobuf:"varint,1,opt,name=server_time,json=serverTime"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ClockSyncResponse) Reset() { + *x = ClockSyncResponse{} + mi := &file_benchkit_control_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ClockSyncResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ClockSyncResponse) ProtoMessage() {} + +func (x *ClockSyncResponse) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_control_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *ClockSyncResponse) GetServerTime() int64 { + if x != nil { + return x.xxx_hidden_ServerTime + } + return 0 +} + +func (x *ClockSyncResponse) SetServerTime(v int64) { + x.xxx_hidden_ServerTime = v +} + +type ClockSyncResponse_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + ServerTime int64 +} + +func (b0 ClockSyncResponse_builder) Build() *ClockSyncResponse { + m0 := &ClockSyncResponse{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_ServerTime = b.ServerTime + return m0 +} + +// DoneRequest is multicast to advise peers that the sender has finished its +// own benchmark work and will issue no further calls. +type DoneRequest struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_SenderId uint32 `protobuf:"varint,1,opt,name=sender_id,json=senderId"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DoneRequest) Reset() { + *x = DoneRequest{} + mi := &file_benchkit_control_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DoneRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DoneRequest) ProtoMessage() {} + +func (x *DoneRequest) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_control_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *DoneRequest) GetSenderId() uint32 { + if x != nil { + return x.xxx_hidden_SenderId + } + return 0 +} + +func (x *DoneRequest) SetSenderId(v uint32) { + x.xxx_hidden_SenderId = v +} + +type DoneRequest_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // sender_id is the sender's Gorums node ID, used to de-duplicate signals + // and to name peers that have not yet signaled when diagnosing a timeout. + SenderId uint32 +} + +func (b0 DoneRequest_builder) Build() *DoneRequest { + m0 := &DoneRequest{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_SenderId = b.SenderId + return m0 +} + +// DoneResponse is an empty message; Done is a one-way advisory signal, but a +// response type must be defined to satisfy the gRPC schema. +type DoneResponse struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DoneResponse) Reset() { + *x = DoneResponse{} + mi := &file_benchkit_control_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DoneResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DoneResponse) ProtoMessage() {} + +func (x *DoneResponse) ProtoReflect() protoreflect.Message { + mi := &file_benchkit_control_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type DoneResponse_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 DoneResponse_builder) Build() *DoneResponse { + m0 := &DoneResponse{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +var File_benchkit_control_proto protoreflect.FileDescriptor + +const file_benchkit_control_proto_rawDesc = "" + + "\n" + + "\x16benchkit/control.proto\x12\bbenchkit\x1a\fgorums.proto\x1a\x17benchkit/benchkit.proto\"B\n" + + "\fStartRequest\x122\n" + + "\n" + + "stats_mode\x18\x01 \x01(\x0e2\x13.benchkit.StatsModeR\tstatsMode\"\x0f\n" + + "\rStartResponse\"\r\n" + + "\vStopRequest\"\x12\n" + + "\x10ClockSyncRequest\"4\n" + + "\x11ClockSyncResponse\x12\x1f\n" + + "\vserver_time\x18\x01 \x01(\x03R\n" + + "serverTime\"*\n" + + "\vDoneRequest\x12\x1b\n" + + "\tsender_id\x18\x01 \x01(\rR\bsenderId\"\x0e\n" + + "\fDoneResponse2\x89\x02\n" + + "\aControl\x12>\n" + + "\x05Start\x12\x16.benchkit.StartRequest\x1a\x17.benchkit.StartResponse\"\x04\xa0\xb5\x18\x01\x125\n" + + "\x04Stop\x12\x15.benchkit.StopRequest\x1a\x10.benchkit.Result\"\x04\xa0\xb5\x18\x01\x12J\n" + + "\tClockSync\x12\x1a.benchkit.ClockSyncRequest\x1a\x1b.benchkit.ClockSyncResponse\"\x04\xa0\xb5\x18\x01\x12;\n" + + "\x04Done\x12\x15.benchkit.DoneRequest\x1a\x16.benchkit.DoneResponse\"\x04\x98\xb5\x18\x01B'Z github.com/relab/gorums/benchkit\x92\x03\x02\b\x02b\beditionsp\xe9\a" + +var file_benchkit_control_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_benchkit_control_proto_goTypes = []any{ + (*StartRequest)(nil), // 0: benchkit.StartRequest + (*StartResponse)(nil), // 1: benchkit.StartResponse + (*StopRequest)(nil), // 2: benchkit.StopRequest + (*ClockSyncRequest)(nil), // 3: benchkit.ClockSyncRequest + (*ClockSyncResponse)(nil), // 4: benchkit.ClockSyncResponse + (*DoneRequest)(nil), // 5: benchkit.DoneRequest + (*DoneResponse)(nil), // 6: benchkit.DoneResponse + (StatsMode)(0), // 7: benchkit.StatsMode + (*Result)(nil), // 8: benchkit.Result +} +var file_benchkit_control_proto_depIdxs = []int32{ + 7, // 0: benchkit.StartRequest.stats_mode:type_name -> benchkit.StatsMode + 0, // 1: benchkit.Control.Start:input_type -> benchkit.StartRequest + 2, // 2: benchkit.Control.Stop:input_type -> benchkit.StopRequest + 3, // 3: benchkit.Control.ClockSync:input_type -> benchkit.ClockSyncRequest + 5, // 4: benchkit.Control.Done:input_type -> benchkit.DoneRequest + 1, // 5: benchkit.Control.Start:output_type -> benchkit.StartResponse + 8, // 6: benchkit.Control.Stop:output_type -> benchkit.Result + 4, // 7: benchkit.Control.ClockSync:output_type -> benchkit.ClockSyncResponse + 6, // 8: benchkit.Control.Done:output_type -> benchkit.DoneResponse + 5, // [5:9] is the sub-list for method output_type + 1, // [1:5] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_benchkit_control_proto_init() } +func file_benchkit_control_proto_init() { + if File_benchkit_control_proto != nil { + return + } + file_benchkit_benchkit_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_benchkit_control_proto_rawDesc), len(file_benchkit_control_proto_rawDesc)), + NumEnums: 0, + NumMessages: 7, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_benchkit_control_proto_goTypes, + DependencyIndexes: file_benchkit_control_proto_depIdxs, + MessageInfos: file_benchkit_control_proto_msgTypes, + }.Build() + File_benchkit_control_proto = out.File + file_benchkit_control_proto_goTypes = nil + file_benchkit_control_proto_depIdxs = nil +} diff --git a/benchkit/control_gorums.pb.go b/benchkit/control_gorums.pb.go new file mode 100644 index 00000000..d9118a84 --- /dev/null +++ b/benchkit/control_gorums.pb.go @@ -0,0 +1,135 @@ +// Code generated by protoc-gen-gorums. DO NOT EDIT. +// versions: +// protoc-gen-gorums v0.11.0-devel +// protoc v7.35.1 +// source: benchkit/control.proto + +package benchkit + +import ( + gorums "github.com/relab/gorums" + gorumsimpl "github.com/relab/gorums/runtime/gorumsimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = gorumsimpl.EnforceVersion(11 - gorumsimpl.MinVersion) + // Verify that the gorums runtime is sufficiently up-to-date. + _ = gorumsimpl.EnforceVersion(gorumsimpl.MaxVersion - 11) +) + +// The type aliases below are useful Gorums types that we make accessible +// from generated code. These names therefore become reserved identifiers, +// meaning that proto message types with these names would collide with the +// generated aliases and cause a compile error. +// +// The bundler (gorums_bundle.go) is responsible for discovering these +// aliases and any other identifiers defined herein, and adding them to +// the reserved identifiers list. +// +// If necessary, additional aliases and other identifiers should be added in +// the generator's cmd/protoc-gen-gorums/dev directory, and the bundler will +// automatically discover them and add them to the reserved identifiers list. + +type ( + Config = gorums.Config + Node = gorums.Node + NodeContext = gorums.NodeContext + ConfigContext = gorums.ConfigContext +) + +// AsyncClockSyncResponse is a future for async quorum calls returning *ClockSyncResponse. +type AsyncClockSyncResponse = *gorums.Async[*ClockSyncResponse] + +// AsyncResult is a future for async quorum calls returning *Result. +type AsyncResult = *gorums.Async[*Result] + +// AsyncStartResponse is a future for async quorum calls returning *StartResponse. +type AsyncStartResponse = *gorums.Async[*StartResponse] + +// CorrectableClockSyncResponse is a correctable object for quorum calls returning *ClockSyncResponse. +type CorrectableClockSyncResponse = *gorums.Correctable[*ClockSyncResponse] + +// CorrectableResult is a correctable object for quorum calls returning *Result. +type CorrectableResult = *gorums.Correctable[*Result] + +// CorrectableStartResponse is a correctable object for quorum calls returning *StartResponse. +type CorrectableStartResponse = *gorums.Correctable[*StartResponse] + +// Start resets the server-side op counter and Stats baseline. +func Start(ctx *ConfigContext, in *StartRequest) *gorums.Call[*StartRequest, *StartResponse] { + return gorumsimpl.QuorumCall[*StartRequest, *StartResponse]( + ctx, in, "benchkit.Control.Start", + ) +} + +// Stop ends measurement and returns this server's Result. For server-measured +// benchmarks (e.g. Multicast) the Result carries latency samples; for +// client-measured benchmarks (e.g. QuorumCall) it carries only memory stats. +func Stop(ctx *ConfigContext, in *StopRequest) *gorums.Call[*StopRequest, *Result] { + return gorumsimpl.QuorumCall[*StopRequest, *Result]( + ctx, in, "benchkit.Control.Stop", + ) +} + +// ClockSync returns the server's wall-clock time, enabling NTP-style +// clock-offset estimation between peers for one-way latency correction. +func ClockSync(ctx *ConfigContext, in *ClockSyncRequest) *gorums.Call[*ClockSyncRequest, *ClockSyncResponse] { + return gorumsimpl.QuorumCall[*ClockSyncRequest, *ClockSyncResponse]( + ctx, in, "benchkit.Control.ClockSync", + ) +} + +// Done is an advisory, one-way signal: a node multicasts it to all peers +// once it has finished its own benchmark work and trailing flush. It is +// not a barrier — a missing or delayed Done never blocks or fails a run, +// it only means the waiting peer falls back to its own timeout. See +// benchmark.AwaitPeersDoneOrGrace. +// +// Example: +// +// err := Done(ctx, in).Send() +// h := Done(ctx, in).Async(); err := h.Wait() +func Done(ctx *ConfigContext, in *DoneRequest) *gorums.OnewayCall[*DoneRequest] { + return gorumsimpl.Multicast(ctx, in, "benchkit.Control.Done") +} + +// Control is the server-side API for the Control Service +type ControlServer interface { + Start(gorums.ServerContext, *StartRequest) (*StartResponse, error) + Stop(gorums.ServerContext, *StopRequest) (*Result, error) + ClockSync(gorums.ServerContext, *ClockSyncRequest) (*ClockSyncResponse, error) + Done(gorums.ServerContext, *DoneRequest) +} + +func RegisterControlServer(srv *gorums.Server, impl ControlServer) { + srv.RegisterHandler("benchkit.Control.Start", func(ctx gorums.ServerContext, in *gorums.Message) (*gorums.Message, error) { + req := gorums.AsProto[*StartRequest](in) + resp, err := impl.Start(ctx, req) + if err != nil { + return nil, err + } + return gorums.NewResponseMessage(in, resp), nil + }) + srv.RegisterHandler("benchkit.Control.Stop", func(ctx gorums.ServerContext, in *gorums.Message) (*gorums.Message, error) { + req := gorums.AsProto[*StopRequest](in) + resp, err := impl.Stop(ctx, req) + if err != nil { + return nil, err + } + return gorums.NewResponseMessage(in, resp), nil + }) + srv.RegisterHandler("benchkit.Control.ClockSync", func(ctx gorums.ServerContext, in *gorums.Message) (*gorums.Message, error) { + req := gorums.AsProto[*ClockSyncRequest](in) + resp, err := impl.ClockSync(ctx, req) + if err != nil { + return nil, err + } + return gorums.NewResponseMessage(in, resp), nil + }) + srv.RegisterHandler("benchkit.Control.Done", func(ctx gorums.ServerContext, in *gorums.Message) (*gorums.Message, error) { + req := gorums.AsProto[*DoneRequest](in) + impl.Done(ctx, req) + return nil, nil + }) +} diff --git a/benchkit/control_server.go b/benchkit/control_server.go new file mode 100644 index 00000000..a0d45983 --- /dev/null +++ b/benchkit/control_server.go @@ -0,0 +1,160 @@ +package benchkit + +import ( + "sync/atomic" + "time" + + "github.com/relab/gorums" +) + +// Control is the [Stats]-backed implementation of the generated +// ControlServer interface: the protocol-neutral measurement control plane. A +// protocol binary registers a Control alongside its own workload server on +// the same listener (see [RegisterControlServer]), and the workload handlers +// record into the same Stats instance via [Control.Stats] and +// [Control.RecordOp] so the Stop reply observes their work. +type Control struct { + stats *Stats + ops atomic.Uint64 + selfID uint32 // this node's gorums node ID + + // Done tracking (see [Control.ArmDone]): doneSeen[id] marks that sender + // id has signaled, doneLeft counts remaining distinct signals, and + // doneCh closes once doneLeft reaches zero. Nil/zero until ArmDone is + // called; Done is then a no-op, which is the default for modes that + // never need the signal. + doneSeen []atomic.Bool + doneLeft atomic.Int32 + doneCh chan struct{} +} + +// NewControl creates a Control server backed by a fresh Stats instance. +func NewControl() *Control { + return &Control{stats: new(Stats)} +} + +// SetID records this node's gorums node ID. Workload senders tag messages with +// this ID so the receiving server can attribute samples per sender. Call it +// from the server registration closure, once the node ID is known. +func (c *Control) SetID(id uint32) { c.selfID = id } + +// Stats returns the Stats instance backing this control server. The protocol's +// workload handlers record server-measured latencies here so the Stop reply and +// the handlers observe the same samples. +func (c *Control) Stats() *Stats { return c.stats } + +// SelfID returns this node's gorums node ID. +func (c *Control) SelfID() uint32 { return c.selfID } + +// ArmDone configures advisory Done tracking for `total` distinct peer +// signals (sender IDs 1..total) and returns the channel that closes once +// every one of them has signaled. Call sites that never need the signal +// (e.g. local or coordinator mode) simply never call ArmDone: Done is a +// no-op and DoneCh returns nil until armed. Arming for no peers at all +// returns an already-closed channel, since there is nothing to wait for. +func (c *Control) ArmDone(total int) <-chan struct{} { + c.doneSeen = make([]atomic.Bool, max(total, 0)+1) // index 0 unused; IDs are 1..total + c.doneLeft.Store(int32(total)) + c.doneCh = make(chan struct{}) + if total <= 0 { + // Done closes doneCh only when a signal drives doneLeft to zero, and no + // signal can arrive: every sender ID is out of doneSeen's range. + close(c.doneCh) + } + return c.doneCh +} + +// DoneCh returns the channel armed by ArmDone, or nil if Done tracking was +// never armed. +func (c *Control) DoneCh() <-chan struct{} { return c.doneCh } + +// DoneCount returns the number of distinct peers that have signaled Done, or 0 +// if Done tracking was never armed. A straggler uses it to tell whether a +// failed cross-node call is the expected consequence of peers finishing and +// exiting rather than a fault. +func (c *Control) DoneCount() int { + var n int + for id := 1; id < len(c.doneSeen); id++ { + if c.doneSeen[id].Load() { + n++ + } + } + return n +} + +// MissingDone returns the sender IDs that have not yet signaled Done, for +// diagnostics when a caller's wait falls back to its own timeout instead of +// observing DoneCh close. Returns nil if Done tracking was never armed. +func (c *Control) MissingDone() []uint32 { + var missing []uint32 + for id := 1; id < len(c.doneSeen); id++ { + if !c.doneSeen[id].Load() { + missing = append(missing, uint32(id)) + } + } + return missing +} + +// RecordOp increments the server-side operation counter. Workload handlers call +// it once per handled operation so client-measured benchmarks can derive per-op +// memory stats in Stop. +func (c *Control) RecordOp() { c.ops.Add(1) } + +// Reset clears the op counter, reconfigures the Stats for the given aggregate +// store mode, and starts a fresh measurement window. This is the body of the +// Start RPC, also reused by server-measured benchmarks to reset counters before +// the measurement window; mode carries the client's -stats-mode so a +// server-measured benchmark bounds memory in StatsMode_HDR like the client path. +func (c *Control) Reset(mode StatsMode) { + c.ops.Store(0) + c.stats.Reset(mode) + c.stats.Start() +} + +// ClockSync returns the server's current wall-clock time in nanoseconds. +// Peers use it for NTP-style clock-offset estimation when correcting the +// one-way latencies measured by server-measured benchmarks. +func (c *Control) ClockSync(_ gorums.ServerContext, _ *ClockSyncRequest) (*ClockSyncResponse, error) { + return ClockSyncResponse_builder{ServerTime: time.Now().UnixNano()}.Build(), nil +} + +// Start resets the server's op counter and Stats baseline, building the +// aggregate store in the mode requested by the client (see [StartRequest]). +func (c *Control) Start(_ gorums.ServerContext, req *StartRequest) (*StartResponse, error) { + c.Reset(req.GetStatsMode()) + return &StartResponse{}, nil +} + +// Stop ends the benchmark and returns a Result. For server-measured benchmarks +// (e.g. Multicast) the Result carries latency samples; for client-measured +// benchmarks (e.g. QuorumCall) it carries only memory stats derived from the +// op counter. +func (c *Control) Stop(_ gorums.ServerContext, _ *StopRequest) (*Result, error) { + c.stats.End() + n := c.ops.Load() + r := c.stats.GetResult() + if r.GetTotalOps() == 0 && n > 0 { + mallocs, totalAlloc := c.stats.MemDelta() + r.SetTotalOps(n) + r.SetAllocsPerOp(mallocs / n) + r.SetMemPerOp(totalAlloc / n) + } + return r, nil +} + +// Done records that the sender has finished its own benchmark work and will +// issue no further calls. It is advisory only: a missing or duplicate signal +// never blocks or errors anything here, it only means the caller waiting on +// DoneCh falls back to its own timeout. See benchmark.AwaitPeersDoneOrGrace. +func (c *Control) Done(_ gorums.ServerContext, req *DoneRequest) { + id := req.GetSenderId() + if c.doneCh == nil || id == 0 || int(id) >= len(c.doneSeen) { + return + } + if c.doneSeen[id].Swap(true) { + return // duplicate signal, already counted + } + if c.doneLeft.Add(-1) == 0 { + close(c.doneCh) + } +} diff --git a/benchkit/control_server_test.go b/benchkit/control_server_test.go new file mode 100644 index 00000000..ac7cbcd8 --- /dev/null +++ b/benchkit/control_server_test.go @@ -0,0 +1,178 @@ +package benchkit + +import ( + "fmt" + "slices" + "testing" + "time" + + "github.com/relab/gorums" +) + +// TestControlStartHonorsStatsMode verifies that the Start RPC builds the server's +// aggregate store in the mode carried by StartRequest, so a server-measured HDR +// run returns a histogram from Stop, and that a later Start with a different mode +// reconfigures the same Control. +func TestControlStartHonorsStatsMode(t *testing.T) { + ctrl := NewControl() + + if _, err := ctrl.Start(gorums.ServerContext{}, StartRequest_builder{StatsMode: StatsMode_HDR}.Build()); err != nil { + t.Fatalf("Start(HDR): %v", err) + } + ctrl.Stats().AddLatency(5 * time.Microsecond) + r, err := ctrl.Stop(gorums.ServerContext{}, &StopRequest{}) + if err != nil { + t.Fatalf("Stop: %v", err) + } + if got := r.GetLatencies(); got != nil { + t.Errorf("Latencies in HDR mode = %v, want nil", got) + } + if r.GetHistogram() == nil { + t.Error("Histogram in HDR mode = nil, want non-nil") + } + + // The zero-value StartRequest selects StatsMode_EXACT, reconfiguring the + // same Control back to raw-sample storage. + if _, err := ctrl.Start(gorums.ServerContext{}, &StartRequest{}); err != nil { + t.Fatalf("Start(EXACT): %v", err) + } + ctrl.Stats().AddLatency(5 * time.Microsecond) + r2, err := ctrl.Stop(gorums.ServerContext{}, &StopRequest{}) + if err != nil { + t.Fatalf("Stop: %v", err) + } + if got := r2.GetLatencies(); len(got) != 1 { + t.Errorf("Latencies after EXACT restart = %v, want one sample", got) + } + if r2.GetHistogram() != nil { + t.Error("Histogram after EXACT restart != nil, want nil") + } +} + +func TestControlDoneCount(t *testing.T) { + ctrl := NewControl() + if got := ctrl.DoneCount(); got != 0 { + t.Errorf("DoneCount before ArmDone = %d, want 0", got) + } + ctrl.ArmDone(3) + if got := ctrl.DoneCount(); got != 0 { + t.Errorf("DoneCount after ArmDone = %d, want 0", got) + } + ctrl.Done(gorums.ServerContext{}, DoneRequest_builder{SenderId: 2}.Build()) + ctrl.Done(gorums.ServerContext{}, DoneRequest_builder{SenderId: 2}.Build()) // duplicate: counted once + ctrl.Done(gorums.ServerContext{}, DoneRequest_builder{SenderId: 3}.Build()) + ctrl.Done(gorums.ServerContext{}, DoneRequest_builder{SenderId: 9}.Build()) // out of range: ignored + if got := ctrl.DoneCount(); got != 2 { + t.Errorf("DoneCount = %d, want 2", got) + } +} + +func TestControlDoneClosesChannelWhenAllSendersSignal(t *testing.T) { + ctrl := NewControl() + doneCh := ctrl.ArmDone(3) + + ctrl.Done(gorums.ServerContext{}, DoneRequest_builder{SenderId: 1}.Build()) + select { + case <-doneCh: + t.Fatal("DoneCh closed after 1/3 signals, want open") + default: + } + + ctrl.Done(gorums.ServerContext{}, DoneRequest_builder{SenderId: 2}.Build()) + select { + case <-doneCh: + t.Fatal("DoneCh closed after 2/3 signals, want open") + default: + } + + ctrl.Done(gorums.ServerContext{}, DoneRequest_builder{SenderId: 3}.Build()) + select { + case <-doneCh: + default: + t.Fatal("DoneCh open after 3/3 signals, want closed") + } +} + +// TestControlArmDoneWithoutPeers verifies that arming for no peers yields an +// already-closed channel. No sender ID is in range, so Done can never drive +// doneLeft to zero, and a caller waiting on the channel would block forever. +func TestControlArmDoneWithoutPeers(t *testing.T) { + for _, total := range []int{0, -1} { + t.Run(fmt.Sprintf("total=%d", total), func(t *testing.T) { + ctrl := NewControl() + doneCh := ctrl.ArmDone(total) + + select { + case <-doneCh: + default: + t.Fatalf("DoneCh open after ArmDone(%d), want closed", total) + } + + // A stray signal must not close the channel a second time. + ctrl.Done(gorums.ServerContext{}, DoneRequest_builder{SenderId: 1}.Build()) + }) + } +} + +func TestControlDoneIgnoresDuplicateSender(t *testing.T) { + ctrl := NewControl() + doneCh := ctrl.ArmDone(2) + + ctrl.Done(gorums.ServerContext{}, DoneRequest_builder{SenderId: 1}.Build()) + ctrl.Done(gorums.ServerContext{}, DoneRequest_builder{SenderId: 1}.Build()) + ctrl.Done(gorums.ServerContext{}, DoneRequest_builder{SenderId: 1}.Build()) + + select { + case <-doneCh: + t.Fatal("DoneCh closed after repeated signals from one sender, want open") + default: + } + + if got := ctrl.MissingDone(); !slices.Equal(got, []uint32{2}) { + t.Errorf("MissingDone() = %v, want [2]", got) + } +} + +func TestControlDoneTracksMissingSenders(t *testing.T) { + ctrl := NewControl() + ctrl.ArmDone(3) + + ctrl.Done(gorums.ServerContext{}, DoneRequest_builder{SenderId: 2}.Build()) + + if got := ctrl.MissingDone(); !slices.Equal(got, []uint32{1, 3}) { + t.Errorf("MissingDone() = %v, want [1 3]", got) + } +} + +func TestControlDoneNoopWhenUnarmed(t *testing.T) { + ctrl := NewControl() + + if ctrl.DoneCh() != nil { + t.Fatal("DoneCh() != nil before ArmDone, want nil") + } + + // Must not panic. + ctrl.Done(gorums.ServerContext{}, DoneRequest_builder{SenderId: 1}.Build()) + + if ctrl.DoneCh() != nil { + t.Fatal("DoneCh() != nil after Done() without ArmDone, want nil") + } +} + +func TestControlDoneIgnoresSenderIDOutOfRange(t *testing.T) { + ctrl := NewControl() + doneCh := ctrl.ArmDone(2) + + // sender_id 0 and out-of-range IDs must not panic or count toward the total. + ctrl.Done(gorums.ServerContext{}, DoneRequest_builder{SenderId: 0}.Build()) + ctrl.Done(gorums.ServerContext{}, DoneRequest_builder{SenderId: 99}.Build()) + + select { + case <-doneCh: + t.Fatal("DoneCh closed after out-of-range senders, want open") + default: + } + if got := ctrl.MissingDone(); !slices.Equal(got, []uint32{1, 2}) { + t.Errorf("MissingDone() = %v, want [1 2]", got) + } +} diff --git a/benchkit/dist.go b/benchkit/dist.go new file mode 100644 index 00000000..5c4d9890 --- /dev/null +++ b/benchkit/dist.go @@ -0,0 +1,200 @@ +package benchkit + +import ( + "iter" + "maps" + "slices" + + "golang.org/x/exp/stats" +) + +// LatencyDist is a latency distribution accumulated from one or more results. +// It answers the same questions — sample count, mean, standard deviation, +// quantiles — whether the underlying runs retained raw per-op samples (exact +// mode) or only a bucketed distribution (HDR mode), so a consumer merging +// results across the nodes of a cluster does not branch on which it got. +// +// A distribution that holds both raw samples and histogram pairs answers from +// the raw samples: they are the exact record, and the histogram is present only +// because some other contributing node retained nothing better. The zero value +// is an empty distribution ready to merge into. +// +// LatencyDist keeps the units of what it was given, which for every benchkit +// result is nanoseconds. It retains the sample slices it is given rather than +// copying them — a run's merged samples are the largest thing a presentation +// tool holds — and never writes through them, so a caller must not mutate a +// Result's latencies after contributing them. +type LatencyDist struct { + // batches holds each contributing result's raw samples without copying + // them; a merge across many nodes appends a batch rather than growing one + // flat slice, so the merged samples are materialized once, at query time, + // instead of once per merge step. + batches [][]int64 + count uint64 // total samples across batches + hist map[int64]uint64 // merged weighted (value, count) pairs + floats []float64 // cached flat float64 view of batches; dropped on add +} + +// Dist returns the summary's latencies as a distribution: its raw samples when +// they are valid, and its whole-run histogram otherwise. Merge the results to +// aggregate a benchmark across the nodes of a run. +func (s Summary) Dist() *LatencyDist { + d := &LatencyDist{} + if s.LatencyValid { + d.addSamples(s.Latencies) + } + d.addHistogram(s.Histogram) + return d +} + +// resultDist returns the whole-run distribution recorded in r, without the +// read-time trim [Summarize] applies. Raw samples are taken as recorded, +// whatever the run's stats mode, so the untrimmed statistics on Result report +// exactly what the run stored. +func resultDist(r *Result) *LatencyDist { + d := &LatencyDist{} + d.addSamples(r.GetLatencies()) + d.addHistogram(r.GetHistogram()) + return d +} + +// addSamples adds one result's raw samples. It retains ns rather than copying +// it, and never appends into it. +func (d *LatencyDist) addSamples(ns []int64) { + if len(ns) == 0 { + return + } + d.batches = append(d.batches, ns) + d.count += uint64(len(ns)) + d.floats = nil +} + +// addHistogram adds one result's bucketed distribution. Only the aligned pairs +// carry weight (see [LatencyHistogram.pairs]), so a malformed message with more +// counts than values contributes no unmatched tail weight. Nil-safe. +func (d *LatencyDist) addHistogram(h *LatencyHistogram) { + d.addPairs(h.pairs()) +} + +// addPairs adds weighted (value, count) pairs to the histogram side. +func (d *LatencyDist) addPairs(pairs iter.Seq2[int64, uint64]) { + for v, c := range pairs { + if d.hist == nil { + d.hist = make(map[int64]uint64) + } + d.hist[v] += c + } +} + +// Merge adds every sample and histogram pair of other into d, leaving other +// unchanged. Use it to aggregate one benchmark across the nodes of a run. +func (d *LatencyDist) Merge(other *LatencyDist) { + if other == nil { + return + } + for _, batch := range other.batches { + d.addSamples(batch) + } + d.addPairs(maps.All(other.hist)) +} + +// Count returns the number of samples the distribution answers from: the raw +// sample count when raw samples were contributed, and the total histogram +// weight otherwise. +func (d *LatencyDist) Count() uint64 { + if d == nil { + return 0 + } + if d.count > 0 { + return d.count + } + var total uint64 + for _, c := range d.hist { + total += c + } + return total +} + +// Empty reports whether the distribution holds no samples. The statistics of an +// empty distribution are not meaningful, so callers test this rather than +// reading a zero mean or a nil quantile slice as a measurement. +func (d *LatencyDist) Empty() bool { + return d.Count() == 0 +} + +// MeanAndStdDev returns the mean and standard deviation of the distribution, +// or (0, 0) when it is empty. Over raw samples this is the sample standard +// deviation; over histogram pairs it is the population standard deviation, +// matching [Histogram.Mean] and [Histogram.StdDev]'s HdrHistogram-mirroring +// convention. +func (d *LatencyDist) MeanAndStdDev() (mean, stddev float64) { + if d == nil { + return 0, 0 + } + if d.count == 0 { + return weightedMeanStdDev(d.pairs()) + } + return stats.MeanAndStdDev(d.samples()) +} + +// Quantiles returns the requested quantile values (in [0, 1]) in the units of +// the recorded samples, or nil when the distribution is empty. Over raw samples +// the quantiles are interpolated; over histogram pairs each is the recorded +// bucket value on which the quantile's cumulative rank falls, so a reported +// quantile is always a value the run actually observed. +func (d *LatencyDist) Quantiles(quantiles ...float64) []float64 { + if d == nil { + return nil + } + if d.count > 0 { + return stats.Quantiles(d.samples(), quantiles...) + } + total := d.Count() + if total == 0 { + return nil + } + out := make([]float64, len(quantiles)) + for i, q := range quantiles { + target := quantileRank(total, q) + var cum uint64 + for v, c := range d.pairs() { + cum += c + if cum >= target { + out[i] = float64(v) + break + } + } + } + return out +} + +// samples returns the merged raw samples as one float64 slice, as the +// golang.org/x/exp/stats functions require. The conversion is cached, since +// summarizing one distribution asks several questions of it; adding to the +// distribution drops the cache. +func (d *LatencyDist) samples() []float64 { + if d.floats != nil { + return d.floats + } + d.floats = make([]float64, 0, d.count) + for _, batch := range d.batches { + for _, v := range batch { + d.floats = append(d.floats, float64(v)) + } + } + return d.floats +} + +// pairs yields the merged weighted (value, count) pairs in ascending value +// order, which the cumulative-rank quantile scan requires. The sequence is +// re-iterable, so weightedMeanStdDev may range over it twice. +func (d *LatencyDist) pairs() iter.Seq2[int64, uint64] { + values := slices.Sorted(maps.Keys(d.hist)) + return func(yield func(int64, uint64) bool) { + for _, v := range values { + if !yield(v, d.hist[v]) { + return + } + } + } +} diff --git a/benchkit/dist_test.go b/benchkit/dist_test.go new file mode 100644 index 00000000..2c7bffd0 --- /dev/null +++ b/benchkit/dist_test.go @@ -0,0 +1,260 @@ +package benchkit + +import ( + "math" + "slices" + "testing" +) + +// closeTo reports whether got is within tol of want. +func closeTo(got, want, tol float64) bool { + return math.Abs(got-want) <= tol +} + +// latencyHist builds a LatencyHistogram from parallel value and count slices, which +// need not be the same length: a malformed message is exactly what the +// aligned-pair rule exists for. +func latencyHist(values []int64, counts []uint64) *LatencyHistogram { + return LatencyHistogram_builder{Value: values, Count: counts}.Build() +} + +// TestLatencyDistSamples verifies the raw-sample path: the count, the sample +// standard deviation, and the interpolated quantiles. +func TestLatencyDistSamples(t *testing.T) { + d := Summary{Latencies: []int64{1, 2, 3, 4, 5}, LatencyValid: true}.Dist() + + if got := d.Count(); got != 5 { + t.Errorf("Count() = %d, want 5", got) + } + if d.Empty() { + t.Error("Empty() = true, want false") + } + mean, stddev := d.MeanAndStdDev() + if !closeTo(mean, 3, 1e-9) { + t.Errorf("mean = %v, want 3", mean) + } + // Sample standard deviation over 1..5: sqrt(10/4). + if !closeTo(stddev, math.Sqrt(2.5), 1e-9) { + t.Errorf("stddev = %v, want %v", stddev, math.Sqrt(2.5)) + } + // R-7 interpolation: p50 of 1..5 lands exactly on 3. + if qs := d.Quantiles(0.5); len(qs) != 1 || !closeTo(qs[0], 3, 1e-9) { + t.Errorf("Quantiles(0.5) = %v, want [3]", qs) + } +} + +// TestLatencyDistHistogram verifies the weighted path: the total weight, the +// population standard deviation, and the cumulative-rank quantiles, which +// return a recorded bucket value rather than an interpolated one. +func TestLatencyDistHistogram(t *testing.T) { + d := Summary{Histogram: latencyHist([]int64{100, 200}, []uint64{5, 15})}.Dist() + + if got := d.Count(); got != 20 { + t.Errorf("Count() = %d, want 20", got) + } + // Weighted mean: (100·5 + 200·15) / 20. + mean, stddev := d.MeanAndStdDev() + if !closeTo(mean, 175, 1e-9) { + t.Errorf("mean = %v, want 175", mean) + } + // Population stddev: sqrt((75²·5 + 25²·15) / 20) = sqrt(1875). + if !closeTo(stddev, math.Sqrt(1875), 1e-9) { + t.Errorf("stddev = %v, want %v", stddev, math.Sqrt(1875)) + } + // The 10th of 20 samples falls in the 200 bucket. + if qs := d.Quantiles(0.5); len(qs) != 1 || qs[0] != 200 { + t.Errorf("Quantiles(0.5) = %v, want [200]", qs) + } +} + +// TestLatencyDistPrefersSamples verifies that a distribution holding both raw +// samples and histogram pairs answers from the samples, which are the exact +// record; the histogram is present only because some other contributing node +// retained nothing better. +func TestLatencyDistPrefersSamples(t *testing.T) { + d := Summary{ + Latencies: []int64{10, 10, 10}, + LatencyValid: true, + Histogram: latencyHist([]int64{500}, []uint64{97}), + }.Dist() + + if got := d.Count(); got != 3 { + t.Errorf("Count() = %d, want 3 (raw samples), not 100", got) + } + if mean, _ := d.MeanAndStdDev(); !closeTo(mean, 10, 1e-9) { + t.Errorf("mean = %v, want 10", mean) + } + if qs := d.Quantiles(0.5); len(qs) != 1 || !closeTo(qs[0], 10, 1e-9) { + t.Errorf("Quantiles(0.5) = %v, want [10]", qs) + } +} + +// TestLatencyDistInvalidLatenciesIgnored verifies that a summary whose latency +// samples are not valid (an HDR run retains none) contributes only its +// histogram, so the two never mix. +func TestLatencyDistInvalidLatenciesIgnored(t *testing.T) { + d := Summary{ + Latencies: []int64{1, 2, 3}, // stale field; LatencyValid says otherwise + Histogram: latencyHist([]int64{400}, []uint64{7}), + }.Dist() + + if got := d.Count(); got != 7 { + t.Errorf("Count() = %d, want 7 (histogram only)", got) + } + if qs := d.Quantiles(0.5); len(qs) != 1 || qs[0] != 400 { + t.Errorf("Quantiles(0.5) = %v, want [400]", qs) + } +} + +// TestLatencyDistMerge verifies that merging aggregates samples and histogram +// pairs across nodes, that a merge of a mixed run answers from the raw samples, +// and that merging leaves the source distribution and its sample slices +// unchanged. +func TestLatencyDistMerge(t *testing.T) { + t.Run("samples", func(t *testing.T) { + nodeA := []int64{1, 2, 3} + a := Summary{Latencies: nodeA, LatencyValid: true}.Dist() + b := Summary{Latencies: []int64{4, 5}, LatencyValid: true}.Dist() + + var merged LatencyDist + merged.Merge(a) + merged.Merge(b) + + if got := merged.Count(); got != 5 { + t.Errorf("Count() = %d, want 5", got) + } + if mean, _ := merged.MeanAndStdDev(); !closeTo(mean, 3, 1e-9) { + t.Errorf("mean = %v, want 3", mean) + } + if !slices.Equal(nodeA, []int64{1, 2, 3}) { + t.Errorf("source samples = %v, want them left unchanged", nodeA) + } + if got := a.Count(); got != 3 { + t.Errorf("source Count() = %d, want it left unchanged at 3", got) + } + }) + + t.Run("histograms", func(t *testing.T) { + a := Summary{Histogram: latencyHist([]int64{100, 200}, []uint64{5, 15})}.Dist() + b := Summary{Histogram: latencyHist([]int64{200, 300}, []uint64{5, 5})}.Dist() + + var merged LatencyDist + merged.Merge(a) + merged.Merge(b) + + if got := merged.Count(); got != 30 { + t.Errorf("Count() = %d, want 30", got) + } + // Merged weights: 100×5, 200×20, 300×5. The 15th of 30 is in the 200 bucket. + if qs := merged.Quantiles(0.5); len(qs) != 1 || qs[0] != 200 { + t.Errorf("Quantiles(0.5) = %v, want [200]", qs) + } + // p99 rank 30 falls on the last bucket. + if qs := merged.Quantiles(0.99); len(qs) != 1 || qs[0] != 300 { + t.Errorf("Quantiles(0.99) = %v, want [300]", qs) + } + }) + + t.Run("mixed", func(t *testing.T) { + exact := Summary{Latencies: []int64{7, 7}, LatencyValid: true}.Dist() + hdr := Summary{Histogram: latencyHist([]int64{900}, []uint64{50})}.Dist() + + var merged LatencyDist + merged.Merge(exact) + merged.Merge(hdr) + + if got := merged.Count(); got != 2 { + t.Errorf("Count() = %d, want 2 (raw samples win over the merged histogram)", got) + } + if mean, _ := merged.MeanAndStdDev(); !closeTo(mean, 7, 1e-9) { + t.Errorf("mean = %v, want 7", mean) + } + }) + + t.Run("nil", func(t *testing.T) { + var merged LatencyDist + merged.Merge(nil) + if !merged.Empty() { + t.Error("Empty() = false after merging nil, want true") + } + }) +} + +// TestLatencyDistAlignedPairs verifies that only the aligned (value, count) +// pairs of a malformed histogram carry weight, in both directions. An unmatched +// value must not become a zero-weight reading and an unmatched count must not +// become a fabricated zero-nanosecond one, since a report cannot tell either +// from a real measurement. +func TestLatencyDistAlignedPairs(t *testing.T) { + tests := []struct { + name string + values []int64 + counts []uint64 + wantCount uint64 + wantP50 float64 + }{ + {"aligned", []int64{100, 200}, []uint64{5, 15}, 20, 200}, + {"more counts than values", []int64{100, 200}, []uint64{5, 15, 99}, 20, 200}, + {"more values than counts", []int64{100, 200, 300}, []uint64{5, 15}, 20, 200}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := Summary{Histogram: latencyHist(tt.values, tt.counts)}.Dist() + if got := d.Count(); got != tt.wantCount { + t.Errorf("Count() = %d, want %d", got, tt.wantCount) + } + if qs := d.Quantiles(0.5); len(qs) != 1 || qs[0] != tt.wantP50 { + t.Errorf("Quantiles(0.5) = %v, want [%v]", qs, tt.wantP50) + } + }) + } +} + +// TestLatencyDistEmpty verifies that an empty distribution reports itself as +// empty and yields no statistics, rather than a zero mean or a zero-filled +// quantile slice a caller could mistake for a measurement. A nil distribution +// behaves the same, so a consumer holding one for a node that recorded nothing +// needs no presence check. +func TestLatencyDistEmpty(t *testing.T) { + for name, d := range map[string]*LatencyDist{ + "zero value": {}, + "nil": nil, + "empty summary": Summary{}.Dist(), + "empty samples": Summary{Latencies: []int64{}, LatencyValid: true}.Dist(), + "zero-count bin": Summary{Histogram: latencyHist([]int64{100}, []uint64{0})}.Dist(), + } { + t.Run(name, func(t *testing.T) { + if !d.Empty() { + t.Error("Empty() = false, want true") + } + if got := d.Count(); got != 0 { + t.Errorf("Count() = %d, want 0", got) + } + if mean, stddev := d.MeanAndStdDev(); mean != 0 || stddev != 0 { + t.Errorf("MeanAndStdDev() = (%v, %v), want (0, 0)", mean, stddev) + } + if qs := d.Quantiles(0.5); qs != nil { + t.Errorf("Quantiles(0.5) = %v, want nil", qs) + } + }) + } +} + +// TestLatencyDistMergeAfterQuery verifies that a distribution queried before it +// is fully merged still answers from everything it holds afterwards, so the +// cached sample conversion cannot go stale. +func TestLatencyDistMergeAfterQuery(t *testing.T) { + var d LatencyDist + d.Merge(Summary{Latencies: []int64{1, 1}, LatencyValid: true}.Dist()) + if mean, _ := d.MeanAndStdDev(); !closeTo(mean, 1, 1e-9) { + t.Fatalf("mean = %v, want 1", mean) + } + + d.Merge(Summary{Latencies: []int64{7, 7}, LatencyValid: true}.Dist()) + if got := d.Count(); got != 4 { + t.Errorf("Count() = %d, want 4", got) + } + if mean, _ := d.MeanAndStdDev(); !closeTo(mean, 4, 1e-9) { + t.Errorf("mean = %v, want 4", mean) + } +} diff --git a/benchkit/doc.go b/benchkit/doc.go new file mode 100644 index 00000000..b82f5b05 --- /dev/null +++ b/benchkit/doc.go @@ -0,0 +1,9 @@ +// Package benchkit is a toolkit for load-testing and measuring distributed +// systems. +// +// It provides the building blocks for a benchmark run: load generation +// (pacer, ticker), latency and throughput measurement (HDR histograms, time +// series, summary statistics), fault injection, clock synchronization, a +// control server for coordinating remote workers, and aggregation and +// reporting of the collected results. +package benchkit diff --git a/benchkit/event_buffer.go b/benchkit/event_buffer.go new file mode 100644 index 00000000..c6637f7d --- /dev/null +++ b/benchkit/event_buffer.go @@ -0,0 +1,93 @@ +package benchkit + +import ( + "sync" + "time" +) + +// eventBuffer accumulates time-series events in memory during a run. The +// harness attaches the buffered events to the per-benchmark Result (via +// Result.SetEvents); there is no separate events file. All methods are no-ops +// on a nil receiver, so callers can pass a nil *eventBuffer when event +// collection is disabled (-interval=0). The Ticker owns the buffer; its +// background goroutine emits throughput and latency events while RateStep +// emits phase markers from the caller's goroutine, so all access is +// serialized through mu. Consumers that need synthetic event streams +// construct Event values directly via the generated builders. +type eventBuffer struct { + mu sync.Mutex + start time.Time // monotonic offset base (set on the START phase or first emit) + events []*Event +} + +// newEventBuffer returns an empty eventBuffer ready to accumulate events. +func newEventBuffer() *eventBuffer { + return &eventBuffer{} +} + +// emitOffsetLocked returns the nanosecond offset from the start base. Because +// now carries Go's monotonic clock reading, now.Sub(b.start) is +// non-decreasing; the max(..., 0) is a defensive guard for fabricated +// wall-clock-only times. Callers must hold b.mu. +func (b *eventBuffer) emitOffsetLocked(now time.Time) int64 { + if b.start.IsZero() { + b.start = now + } + return max(now.Sub(b.start).Nanoseconds(), 0) +} + +// emitPhase records a lifecycle phase transition. When phase is START the +// supplied instant is used as the monotonic base for all subsequent offsets. +func (b *eventBuffer) emitPhase(now time.Time, phase PhaseMarker_Phase, rate int64) { + if b == nil { + return + } + b.mu.Lock() + defer b.mu.Unlock() + if phase == PhaseMarker_START { + b.start = now + } + b.events = append(b.events, Event_builder{ + Offset: b.emitOffsetLocked(now), + Phase: PhaseMarker_builder{Phase: phase, Rate: rate}.Build(), + }.Build()) +} + +// emitThroughput records an ops-completed count and interval duration for one +// ticker period. +func (b *eventBuffer) emitThroughput(now time.Time, ops uint64, duration time.Duration) { + if b == nil { + return + } + b.mu.Lock() + defer b.mu.Unlock() + b.events = append(b.events, Event_builder{ + Offset: b.emitOffsetLocked(now), + Throughput: ThroughputInterval_builder{Ops: ops, Duration: duration.Nanoseconds()}.Build(), + }.Build()) +} + +// emitLatency records the Welford accumulator state (mean, stddev, count) for +// one ticker period, all in nanoseconds. +func (b *eventBuffer) emitLatency(now time.Time, mean, stddev float64, count uint64) { + if b == nil { + return + } + b.mu.Lock() + defer b.mu.Unlock() + b.events = append(b.events, Event_builder{ + Offset: b.emitOffsetLocked(now), + Latency: LatencyInterval_builder{Mean: mean, Stddev: stddev, Count: count}.Build(), + }.Build()) +} + +// Events returns the buffered events in emission order, or nil on a nil +// receiver. The harness attaches them to the Result via Result.SetEvents. +func (b *eventBuffer) Events() []*Event { + if b == nil { + return nil + } + b.mu.Lock() + defer b.mu.Unlock() + return b.events +} diff --git a/benchkit/event_buffer_test.go b/benchkit/event_buffer_test.go new file mode 100644 index 00000000..93229394 --- /dev/null +++ b/benchkit/event_buffer_test.go @@ -0,0 +1,93 @@ +package benchkit + +import ( + "testing" + "time" +) + +func TestEventBufferRoundTrip(t *testing.T) { + b := newEventBuffer() + start := time.Unix(0, 1000) + + b.emitPhase(start, PhaseMarker_START, 100) + b.emitThroughput(start.Add(500*time.Millisecond), 50, 500*time.Millisecond) + b.emitLatency(start.Add(500*time.Millisecond), 200.0, 30.5, 50) + b.emitThroughput(start.Add(1500*time.Millisecond), 100, 500*time.Millisecond) + b.emitPhase(start.Add(2000*time.Millisecond), PhaseMarker_STOP, 0) + + events := b.Events() + if len(events) != 5 { + t.Fatalf("len(events) = %d, want 5", len(events)) + } + + // First event: START phase marker at offset 0. + ev0 := events[0] + if ev0.GetOffset() != 0 { + t.Errorf("events[0].offset = %d, want 0", ev0.GetOffset()) + } + ph0 := ev0.GetPhase() + if ph0 == nil { + t.Fatal("events[0].phase is nil") + } + if got, want := ph0.GetPhase(), PhaseMarker_START; got != want { + t.Errorf("events[0].phase = %v, want %v", got, want) + } + if got, want := ph0.GetRate(), int64(100); got != want { + t.Errorf("events[0].rate = %d, want 100", got) + } + + // Second event: throughput interval at offset 500ms. + ev1 := events[1] + if ev1.GetOffset() != 500_000_000 { + t.Errorf("events[1].offset = %d, want 500000000", ev1.GetOffset()) + } + tp := ev1.GetThroughput() + if tp == nil { + t.Fatal("events[1].throughput is nil") + } + if got, want := tp.GetOps(), uint64(50); got != want { + t.Errorf("throughput.ops = %d, want 50", got) + } + + // Third event: latency interval. + lat := events[2].GetLatency() + if lat == nil { + t.Fatal("events[2].latency is nil") + } + if got, want := lat.GetMean(), 200.0; got != want { + t.Errorf("latency.mean = %f, want %f", got, want) + } + if got, want := lat.GetCount(), uint64(50); got != want { + t.Errorf("latency.count = %d, want 50", got) + } +} + +func TestEventBufferNilSafe(t *testing.T) { + var b *eventBuffer + // All methods must be no-ops on a nil receiver. + b.emitPhase(time.Time{}, PhaseMarker_START, 0) + b.emitThroughput(time.Time{}, 10, time.Microsecond) + b.emitLatency(time.Time{}, 100.0, 10.0, 10) + if b.Events() != nil { + t.Error("nil eventBuffer Events() should return nil") + } +} + +func TestEventBufferOffsetBase(t *testing.T) { + b := newEventBuffer() + // Emit without a phase marker first: base anchored to the first call. + base := time.Unix(0, 1000) + b.emitThroughput(base, 5, 100) + b.emitThroughput(base.Add(500), 5, 100) + + events := b.Events() + if len(events) != 2 { + t.Fatalf("len(events) = %d, want 2", len(events)) + } + if got := events[0].GetOffset(); got != 0 { + t.Errorf("events[0].offset = %d, want 0", got) + } + if got := events[1].GetOffset(); got != 500 { + t.Errorf("events[1].offset = %d, want 500", got) + } +} diff --git a/benchkit/fault.go b/benchkit/fault.go new file mode 100644 index 00000000..0bf01422 --- /dev/null +++ b/benchkit/fault.go @@ -0,0 +1,23 @@ +package benchkit + +import ( + "os" + "time" +) + +// ArmFaultInjection schedules an abrupt, unannounced process exit after d, +// simulating a node crash mid-run for fault-injection experiments (the +// -fault-kill-after flag, see StandardFlags). The exit is clean (status 0) so +// remote launchers do not flag it; the node simply writes no result file, and +// sweep reports the missing file and summarizes the surviving nodes. A +// non-positive d disables the fault and returns nil; otherwise the returned +// timer can stop the scheduled exit (used by tests). +func ArmFaultInjection(d time.Duration) *time.Timer { + if d <= 0 { + return nil + } + return time.AfterFunc(d, func() { + Logf("fault injection: exiting after %v\n", d) + os.Exit(0) // skipcq: RVV-A0003 + }) +} diff --git a/benchkit/fault_test.go b/benchkit/fault_test.go new file mode 100644 index 00000000..dec0177d --- /dev/null +++ b/benchkit/fault_test.go @@ -0,0 +1,27 @@ +package benchkit + +import ( + "testing" + "time" +) + +// TestArmFaultInjection verifies the arming guard: a non-positive duration +// disables the fault, and a positive duration returns a stoppable timer. The +// timer is stopped long before it could fire, so the test never exits. +func TestArmFaultInjection(t *testing.T) { + if timer := ArmFaultInjection(0); timer != nil { + timer.Stop() + t.Error("ArmFaultInjection(0) = non-nil timer, want nil") + } + if timer := ArmFaultInjection(-time.Second); timer != nil { + timer.Stop() + t.Error("ArmFaultInjection(-1s) = non-nil timer, want nil") + } + timer := ArmFaultInjection(time.Hour) + if timer == nil { + t.Fatal("ArmFaultInjection(1h) = nil, want armed timer") + } + if !timer.Stop() { + t.Error("timer already fired or stopped, want active timer") + } +} diff --git a/benchkit/flags.go b/benchkit/flags.go new file mode 100644 index 00000000..e2833c3f --- /dev/null +++ b/benchkit/flags.go @@ -0,0 +1,98 @@ +package benchkit + +import ( + "flag" + "fmt" + "regexp" + "strings" + "time" +) + +// StandardFlags holds the CLI flag contract that every sweep-driven benchmark +// binary must accept (doc/benchkit.html, sections 9 and 11). +// A binary built on benchkit registers exactly this set via RegisterFlags, so +// it complies with the contract automatically; sweep launches it without knowing +// what the workload does. +type StandardFlags struct { + Benchmarks *regexp.Regexp // -benchmarks: regexp selecting benchmarks to run + Self string // -self: this node's listen address; non-empty triggers distributed mode + Remotes []string // -remotes: comma-separated peer addresses + Workers int // -workers: concurrent worker goroutines + Payload int // -payload: request/response payload size in bytes + Rate int // -rate: target sends/sec per node; 0 = unlimited (saturating) + Duration time.Duration // -time: measurement duration + Output string // -output: result file path + Verbose bool // -verbose: log connection progress + StatsMode StatsMode // -stats-mode: aggregate latency backing store + Interval time.Duration // -interval: ticker interval for per-interval metrics; 0 = disabled + RateStep int // -rate-step: rate increment per ramp step; 0 = disabled + RateStepMax int // -rate-step-max: maximum target rate for ramp; 0 = disabled + StreamMode string // -stream-mode: symmetric stream topology (dual or dedup) + CPUProfile string // -cpuprofile: CPU profile output path; empty = disabled + MemProfile string // -memprofile: heap profile output path; empty = disabled + Trace string // -trace: execution trace output path; empty = disabled + FaultKillAfter time.Duration // -fault-kill-after: exit cleanly after this duration; 0 = disabled + CallTimeout time.Duration // -call-timeout: per-call deadline for quorum-call workloads; 0 = disabled +} + +// RegisterFlags registers the standard flag contract on fs and returns a pointer +// to the values, populated after fs.Parse. A protocol binary calls this, adds any +// protocol-specific flags to the same FlagSet, parses, and passes Options() to +// Run. The default selector matches every benchmark. +func RegisterFlags(fs *flag.FlagSet) *StandardFlags { + f := &StandardFlags{Benchmarks: regexp.MustCompile(".*")} + fs.Func("benchmarks", "A `regexp` matching the benchmarks to run.", func(v string) (err error) { + f.Benchmarks, err = regexp.Compile(v) + return + }) + fs.Func("remotes", "A comma-separated `list` of remote addresses to connect to.", func(v string) error { + f.Remotes = strings.Split(v, ",") + return nil + }) + fs.IntVar(&f.Workers, "workers", 1, "Number of goroutines that can make calls concurrently.") + fs.IntVar(&f.Payload, "payload", 0, "Size of the payload in request and response messages (in bytes).") + fs.IntVar(&f.Rate, "rate", 0, "Target sends per second per node; 0 means unlimited (saturating).") + fs.DurationVar(&f.Duration, "time", 1*time.Second, "The duration of each benchmark.") + fs.StringVar(&f.Output, "output", "", "Write results to this `file`.") + fs.StringVar(&f.Self, "self", "", "This node's listen `address`; triggers distributed mode.") + fs.BoolVar(&f.Verbose, "verbose", false, "Log connection progress in distributed mode.") + fs.DurationVar(&f.Interval, "interval", 500*time.Millisecond, "Ticker interval for per-interval metrics; 0 = disabled.") + fs.IntVar(&f.RateStep, "rate-step", 0, "Rate increment per ramp step (ops/s); 0 = disabled (no ramp).") + fs.IntVar(&f.RateStepMax, "rate-step-max", 0, "Maximum target rate for ramp (ops/s); 0 = disabled (no ramp).") + fs.StringVar(&f.StreamMode, "stream-mode", "dual", "Symmetric stream topology: dual or dedup.") + fs.StringVar(&f.CPUProfile, "cpuprofile", "", "A `file` to write cpu profile to.") + fs.StringVar(&f.MemProfile, "memprofile", "", "A `file` to write memory profile to.") + fs.StringVar(&f.Trace, "trace", "", "A `file` to write trace to.") + fs.DurationVar(&f.FaultKillAfter, "fault-kill-after", 0, "Fault injection: exit cleanly after this duration; 0 = disabled (see ArmFaultInjection).") + fs.DurationVar(&f.CallTimeout, "call-timeout", 0, "Per-call deadline for quorum-call workloads; a call stalled behind an unresponsive peer fails with DeadlineExceeded instead of hanging until run end. 0 = disabled.") + fs.Func("stats-mode", "Aggregate latency backing store: `exact` (default) or hdr.", func(v string) error { + switch v { + case "exact": + f.StatsMode = StatsMode_EXACT + case "hdr": + f.StatsMode = StatsMode_HDR + default: + return fmt.Errorf("unknown stats mode %q (want: exact or hdr)", v) + } + return nil + }) + return f +} + +// Options builds the run Options from the standard flags. Fields outside the CLI +// contract (NumNodes, Remote, QuorumSize, MaxAsync) are left zero for the caller +// to set from the run topology. +func (f *StandardFlags) Options() Options { + return Options{ + Workers: f.Workers, + Payload: f.Payload, + Rate: f.Rate, + Duration: f.Duration, + StatsMode: f.StatsMode, + Interval: f.Interval, + RateStep: f.RateStep, + RateStepMax: f.RateStepMax, + StreamMode: f.StreamMode, + CallTimeout: f.CallTimeout, + } +} diff --git a/benchkit/flags_test.go b/benchkit/flags_test.go new file mode 100644 index 00000000..60f77962 --- /dev/null +++ b/benchkit/flags_test.go @@ -0,0 +1,32 @@ +package benchkit + +import ( + "flag" + "testing" + "time" +) + +// TestRegisterFlagsCallTimeout verifies that -call-timeout is parsed into +// StandardFlags and carried into Options, and that it defaults to disabled. +func TestRegisterFlagsCallTimeout(t *testing.T) { + tests := []struct { + name string + args []string + want time.Duration + }{ + {name: "DefaultDisabled", args: nil, want: 0}, + {name: "Set", args: []string{"-call-timeout=2s"}, want: 2 * time.Second}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fs := flag.NewFlagSet(t.Name(), flag.ContinueOnError) + f := RegisterFlags(fs) + if err := fs.Parse(tt.args); err != nil { + t.Fatalf("Parse(%v) failed: %v", tt.args, err) + } + if got := f.Options().CallTimeout; got != tt.want { + t.Errorf("Options().CallTimeout = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/benchkit/go.mod b/benchkit/go.mod new file mode 100644 index 00000000..921b1e2f --- /dev/null +++ b/benchkit/go.mod @@ -0,0 +1,32 @@ +module github.com/relab/gorums/benchkit + +go 1.26.2 + +require ( + github.com/google/pprof v0.0.0-20260604005048-7023385849c0 + github.com/relab/gorums v0.11.0 + github.com/relab/iago v0.0.0-20260702190239-acea5b94dd97 + golang.org/x/exp v0.0.0-20260611194520-c48552f49976 + golang.org/x/sync v0.21.0 + google.golang.org/grpc v1.82.0 + google.golang.org/protobuf v1.36.11 +) + +require ( + github.com/kevinburke/ssh_config v1.6.0 // indirect + github.com/kr/fs v0.1.0 // indirect + github.com/pkg/sftp v1.13.10 // indirect + github.com/relab/wrfs v0.0.0-20220416082020-a641cd350078 // indirect + go.uber.org/goleak v1.3.0 // indirect + golang.org/x/crypto v0.53.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/sys v0.46.0 // indirect + golang.org/x/text v0.38.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 // indirect +) + +// benchkit tracks the gorums source in this repository rather than a released +// version, so that a change to the gorums API and its benchkit follow-up land +// together. The target is inside this repository, so every clone resolves it. +// Extracting benchkit to its own repository replaces this with a version pin. +replace github.com/relab/gorums => ../ diff --git a/benchkit/go.sum b/benchkit/go.sum new file mode 100644 index 00000000..ccc9e755 --- /dev/null +++ b/benchkit/go.sum @@ -0,0 +1,70 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20260604005048-7023385849c0 h1:h1QTMDl6q9wDvDCJVpKQSjgleGFYnd2fOxmg2K+6BGE= +github.com/google/pprof v0.0.0-20260604005048-7023385849c0/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kevinburke/ssh_config v1.6.0 h1:J1FBfmuVosPHf5GRdltRLhPJtJpTlMdKTBjRgTaQBFY= +github.com/kevinburke/ssh_config v1.6.0/go.mod h1:q2RIzfka+BXARoNexmF9gkxEX7DmvbW9P4hIVx2Kg4M= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/pkg/sftp v1.13.10 h1:+5FbKNTe5Z9aspU88DPIKJ9z2KZoaGCu6Sr6kKR/5mU= +github.com/pkg/sftp v1.13.10/go.mod h1:bJ1a7uDhrX/4OII+agvy28lzRvQrmIQuaHrcI1HbeGA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/relab/container v0.0.0-20260109140004-4adfae874bb5 h1:ImfKSqvvsCWtQ2ibKkZZMgE8+incQzgfdRKEcxTW3g4= +github.com/relab/container v0.0.0-20260109140004-4adfae874bb5/go.mod h1:oLZXG1NirJWzF2fMEeMUC6OLiMn6RtCChzUz1jtF/qs= +github.com/relab/iago v0.0.0-20260702190239-acea5b94dd97 h1:+Vq22taQ4bCB4h5T0yEce4MXEfRCrSWl6bWZInEd+QU= +github.com/relab/iago v0.0.0-20260702190239-acea5b94dd97/go.mod h1:I2XJuORTq0tDooHs3WJh9PSUs9i9womJZ3OzFs4PjVA= +github.com/relab/wrfs v0.0.0-20220416082020-a641cd350078 h1:JN5qn8C/HZoyMAycX6z6O0SeX+09CV3w3GcVNW70OZA= +github.com/relab/wrfs v0.0.0-20220416082020-a641cd350078/go.mod h1:8BTalsvE1BexSfZZFmZfmJKqfoXPzsP/ixpVOmg7Udk= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= +go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= +go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto= +golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio= +golang.org/x/exp v0.0.0-20260611194520-c48552f49976 h1:X8Hz2ImujgbmetVuW+w2YkyZChE3cBpZi2P158rTG9M= +golang.org/x/exp v0.0.0-20260611194520-c48552f49976/go.mod h1:vnf4pv9iKZXY58sQE1L86zmNWJ4159e1RkcWiLCkeEY= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc= +golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y= +golang.org/x/text v0.38.0 h1:sXmwo9DwP3OK9EZ7PqAdaooSGozfl/3a6/xJcbzPRhE= +golang.org/x/text v0.38.0/go.mod h1:YXZt3QhHUKYT53r2lLKFIVi6Ao1jdzrTR/KQ09qyxF4= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7 h1:eM/YSd5bBFagF51o1E745Ta7RwzpW0h+z+QDNZOgmQ8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260630182238-925bb5da69e7/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU= +google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/benchkit/harness.go b/benchkit/harness.go new file mode 100644 index 00000000..03449c9c --- /dev/null +++ b/benchkit/harness.go @@ -0,0 +1,738 @@ +package benchkit + +import ( + "context" + "errors" + "fmt" + "io" + "iter" + "os" + "regexp" + "runtime" + "slices" + "sort" + "sync/atomic" + "text/tabwriter" + "time" + + "github.com/relab/gorums" + "golang.org/x/sync/errgroup" + "google.golang.org/protobuf/proto" +) + +// memSnapshot captures heap allocation counters for computing client-side +// per-op memory stats across a measurement window. +type memSnapshot struct { + ms runtime.MemStats +} + +// read records the current heap allocation counters. +func (m *memSnapshot) read() { runtime.ReadMemStats(&m.ms) } + +// delta returns the allocations and bytes allocated per operation between m and +// end, or zero when totalOps is zero. +func (m memSnapshot) delta(end memSnapshot, totalOps uint64) (allocsPerOp, memPerOp uint64) { + if totalOps == 0 { + return 0, 0 + } + return (end.ms.Mallocs - m.ms.Mallocs) / totalOps, + (end.ms.TotalAlloc - m.ms.TotalAlloc) / totalOps +} + +// benchSlack bounds the scheduling and shutdown slack on top of the run +// duration for a single benchmark invocation. A hung RPC is thus surfaced as a +// timeout rather than a hang. +const benchSlack = 30 * time.Second + +// Options controls the parameters shared by every benchmark the harness runs. +type Options struct { + Workers int // Number of concurrent worker goroutines + Duration time.Duration // Duration of benchmark + MaxAsync int // Max async calls at once + NumNodes int // Number of nodes to include in configuration + Payload int // Size of message payload + QuorumSize int // Number of messages to wait for + Rate int // Target sends per second per node; 0 means unlimited (saturate) + Remote bool // Whether the servers are remote (true) or local (false) + StatsMode StatsMode // Aggregate latency backing store; 0 = StatsMode_EXACT (default) + Interval time.Duration // Ticker interval for per-interval metrics; 0 = disabled + BenchName string // Benchmark name, stamped by Run before invoking Bench.Run + RateStep int // Rate increment per step during the ramp; 0 = disabled + // RateStepMax is the maximum target rate during the ramp; 0 = disabled. The + // duration of each ramp step is derived: Duration divided evenly across the + // rate levels (rampSteps), so a ramp always spans exactly Duration and ends + // exactly at RateStepMax. + RateStepMax int + StreamMode string // Symmetric stream topology: "dual" (default) or "dedup" + // CallTimeout is the per-call deadline for quorum-call workloads; 0 = + // disabled. With a deadline, a call stalled behind an unresponsive peer + // fails the run with DeadlineExceeded — an attributable error in the + // manifest — instead of silently zeroing the node's throughput until the + // run-scoped context expires. + CallTimeout time.Duration + // SendBuffer and RecvBuffer record the buffer capacities the run is + // configured with, so results differing only by buffer size stay + // distinguishable. Zero selects Gorums' own default for that buffer, the + // same substitution [gorums.WithSendBufferSize] and [gorums.WithBufferSizes] + // apply internally. + SendBuffer uint + RecvBuffer uint +} + +// rampEnabled reports whether rate ramping is active: both ramp options +// (RateStep, RateStepMax) must be set. +func (o Options) rampEnabled() bool { + return o.RateStep > 0 && o.RateStepMax > 0 +} + +// rampSteps returns the number of offered-load levels in a ramp: one per +// RateStep increment from the start rate up to and including RateStepMax, where +// a partial final increment still counts as a level. A start rate at or above +// RateStepMax yields a single level. +func (o Options) rampSteps() int { + span := o.RateStepMax - o.startRate() + if span <= 0 { + return 1 + } + return (span+o.RateStep-1)/o.RateStep + 1 +} + +// startRate returns the offered rate of the first measurement phase: Rate +// normally, but RateStep when ramping is enabled and Rate is unset, so a +// ramped run climbs from the first step instead of starting unlimited and +// dropping at the first transition. +func (o Options) startRate() int { + if o.rampEnabled() && o.Rate <= 0 { + return o.RateStep + } + return o.Rate +} + +// StreamDedupOption returns the [gorums.WithStreamDedup] server option when the +// stream mode requests deduplication, or nil otherwise. +func (o Options) StreamDedupOption() gorums.ServerOption { + if o.StreamMode == "dedup" { + return gorums.WithStreamDedup() + } + return nil +} + +// BufferSizesOption returns the [gorums.WithBufferSizes] server option carrying +// the run's configured buffer capacities. +func (o Options) BufferSizesOption() gorums.ServerOption { + return gorums.WithBufferSizes(o.RecvBuffer, o.SendBuffer) +} + +// ServerOptions returns the server options this run's configuration implies, +// with any that do not apply omitted. +func (o Options) ServerOptions() []gorums.ServerOption { + var opts []gorums.ServerOption + for _, opt := range []gorums.ServerOption{o.StreamDedupOption(), o.BufferSizesOption()} { + if opt != nil { + opts = append(opts, opt) + } + } + return opts +} + +// Validate checks the generic constraints every benchkit binary shares, +// independent of workload topology; topology-specific checks (e.g. node +// counts) belong to the caller. [Run] calls this before executing any +// benchmark. +func (o Options) Validate() error { + switch { + case o.Workers < 1: + return fmt.Errorf("workers must be >= 1, got %d", o.Workers) + case o.Duration <= 0: + return fmt.Errorf("duration must be > 0, got %v", o.Duration) + case o.Payload < 0: + return fmt.Errorf("payload must be >= 0, got %d", o.Payload) + case o.Rate < 0: + return fmt.Errorf("rate must be >= 0, got %d", o.Rate) + case o.Interval < 0: + return fmt.Errorf("interval must be >= 0, got %v", o.Interval) + case o.CallTimeout < 0: + return fmt.Errorf("call timeout must be >= 0, got %v", o.CallTimeout) + case o.RateStep < 0: + return fmt.Errorf("rate step must be >= 0, got %d", o.RateStep) + case o.RateStepMax < 0: + return fmt.Errorf("rate step max must be >= 0, got %d", o.RateStepMax) + case (o.RateStep > 0) != (o.RateStepMax > 0): + return fmt.Errorf("rate-step and rate-step-max must both be set or both be zero, got rate-step=%d rate-step-max=%d", o.RateStep, o.RateStepMax) + case o.rampEnabled() && o.RateStepMax < o.RateStep: + return fmt.Errorf("rate-step-max (%d) must be >= rate-step (%d)", o.RateStepMax, o.RateStep) + } + switch o.StreamMode { + case "", "dual", "dedup": + default: + return fmt.Errorf("invalid stream mode %q (want: dual or dedup)", o.StreamMode) + } + return nil +} + +// Bench is a named, runnable benchmark. Run performs one measured campaign with +// the given options and returns this node's Result. The harness (Run) stamps the +// shared run metadata (name, node count, mode, duration, …) onto the returned +// Result, so a Bench.Run closure only fills in the measured fields. +type Bench struct { + Name string + Description string + Run func(Options) (*Result, error) +} + +// ListBenches writes a name/description listing of benches to w, one per +// line, aligned by tab. +func ListBenches(w io.Writer, benches []Bench) { + tw := tabwriter.NewWriter(w, 0, 0, 4, ' ', 0) + for _, b := range benches { + fmt.Fprintf(tw, "%s:\t%s\n", b.Name, b.Description) + } + tw.Flush() +} + +// BenchContext returns a context bounded by the benchmark's run duration plus a +// fixed slack, so a stuck RPC fails the benchmark instead of blocking +// indefinitely. +func BenchContext(opts Options) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), opts.Duration+benchSlack) +} + +// CollectReplies drains a ResponseSeq into a map keyed by node ID. It returns +// the joined errors observed from any node so that a single failing server is +// not silently dropped (as CollectAll does by storing the zero value) and so +// the caller sees every failure, not just the first. +func CollectReplies[T proto.Message](seq gorums.ResponseSeq[T]) (map[uint32]T, error) { + replies := make(map[uint32]T) + var errs []error + for r := range seq { + if r.Err != nil { + errs = append(errs, fmt.Errorf("node %d: %w", r.NodeID, r.Err)) + continue + } + replies[r.NodeID] = r.Value + } + return replies, errors.Join(errs...) +} + +// ErrRunOver is returned by a workload op to signal that the measurement run is +// over (e.g. a straggler's peers have already finished and closed their +// listeners): the worker that receives it stops cleanly, and the op is recorded +// neither as a success nor as a failure. RunPhase treats it as a clean stop +// rather than a fault, so MeasureLatency returns the samples gathered before +// the run ended instead of failing. +var ErrRunOver = errors.New("benchmark run over") + +// RunPhase launches numG goroutines per doOp, each calling its doOp in a tight +// loop from start until endTime or until ctx is cancelled, collecting the first +// non-nil error. When rate > 0, the goroutines for a single doOp are paced to a +// combined rate sends per second (each worker takes an equal, staggered share); +// rate <= 0 saturates. A doOp returning ErrRunOver stops its worker cleanly +// without failing the phase. Passing more than one doOp fans the phase out +// across several independent send targets that run concurrently in one shared +// errgroup, each with its own pool of numG paced workers; this is how the +// symmetric runners drive every system at once. It is the shared measurement +// skeleton used by the benchmark runners. +func RunPhase(ctx context.Context, numG, rate int, start, endTime time.Time, doOps ...func() error) error { + var g errgroup.Group + for _, doOp := range doOps { + for w := range numG { + g.Go(func() error { + p := NewPacer(rate, numG, w, start) + // The loop checks ctx itself: a closed-loop (nil-pacer) worker + // never blocks in Wait, so cancellation would otherwise go + // unnoticed and the worker would spin until endTime. + for time.Now().Before(endTime) && ctx.Err() == nil { + if !p.Wait(ctx) { + return nil + } + if err := doOp(); err != nil { + if errors.Is(err, ErrRunOver) { + return nil + } + return err + } + } + return nil + }) + } + } + return g.Wait() +} + +// RunOffsetCorrected runs the server-measured-latency sequence: estimate peer +// clock offsets, run the measurement send, estimate offsets again, then build +// the per-node corrected results and aggregate them into one cluster-wide +// Result. +// +// estimate samples the per-peer clock offsets and is called once before and once +// after the measurement; the offset type O differs between callers (a single +// offset map coordinator-side, one map per system in symmetric mode), so it is a +// type parameter. measure brackets the send window tightly between the two +// estimate calls: it starts the run, runs the send loop, and stops the run, +// so neither client memory accounting nor the server-measured throughput +// window includes clock-offset estimation time. buildReplies is the +// correction step: it turns the before/after offsets and the replies measure +// collected into per-node corrected Results. +func RunOffsetCorrected[O any]( + estimate func() (O, error), + measure func() error, + buildReplies func(before, after O) (map[uint32]*Result, error), +) (*Result, error) { + // Estimate offsets just before the measurement window; servers measure + // one-way latency against their own clock, so samples carry this offset. + before, err := estimate() + if err != nil { + return nil, err + } + if err := measure(); err != nil { + return nil, err + } + // Estimate offsets again after the window; averaging before and after + // tolerates small clock drift over the run. + after, err := estimate() + if err != nil { + return nil, err + } + replies, err := buildReplies(before, after) + if err != nil { + return nil, err + } + return AggregateServerResults(replies) +} + +// OfferedOps returns the number of sends the offered-load schedule asks of one +// send target over the run: rate × duration, summed per level when ramping. It +// returns 0 for an unlimited (closed-loop) run, which has no schedule. +func (o Options) OfferedOps() float64 { + if o.rampEnabled() { + numSteps := o.rampSteps() + stepSec := (o.Duration / time.Duration(numSteps)).Seconds() + rate := o.startRate() + var total float64 + for range numSteps { + total += float64(rate) * stepSec + rate = min(rate+o.RateStep, o.RateStepMax) + } + return total + } + if o.Rate <= 0 { + return 0 + } + return float64(o.Rate) * o.Duration.Seconds() +} + +// paceTolerance is the fraction of the offered sends below which a paced run +// is reported as having fallen behind its open-loop schedule. +const paceTolerance = 0.95 + +// PaceWarning returns a warning when a paced run attempted markedly fewer +// sends than its offered-load schedule asked for, and "" otherwise. Falling +// behind means the workers could not sustain the offered rate — sustaining +// rate R at per-op latency L needs roughly R × L in-flight ops — so the run +// degraded toward closed-loop saturation while the recorded rate markers +// still claim the offered load. +func PaceWarning(sent uint64, offered float64) string { + if offered <= 0 || float64(sent) >= paceTolerance*offered { + return "" + } + return fmt.Sprintf("warning: offered rate not sustained: %d of %.0f scheduled sends (%.0f%%); "+ + "raise -workers (sustaining a rate needs about rate x latency in-flight ops) or lower the rate", + sent, offered, 100*float64(sent)/offered) +} + +// runMeasure executes the measurement phase, applying rate ramping if both +// ramp options (RateStep, RateStepMax) are set. When the run is paced, it +// counts the attempted sends and warns on stderr when the run fell behind the +// offered-load schedule (see [PaceWarning]); the count is one atomic add per +// send, taken only on the paced path, where each send already pays a timer +// wait. The unlimited (closed-loop) path is untouched. +func runMeasure(ctx context.Context, ticker *Ticker, opts Options, doOps ...func() error) error { + offered := opts.OfferedOps() * float64(len(doOps)) + if offered <= 0 { + return runSchedule(ctx, ticker, opts, doOps...) + } + var sent atomic.Uint64 + counted := make([]func() error, len(doOps)) + for i, doOp := range doOps { + counted[i] = func() error { + sent.Add(1) + return doOp() + } + } + if err := runSchedule(ctx, ticker, opts, counted...); err != nil { + return err + } + if msg := PaceWarning(sent.Load(), offered); msg != "" { + fmt.Fprintln(os.Stderr, msg) + } + return nil +} + +// runSchedule drives the offered-load schedule. It emits a RateStep event on +// the ticker before each rate transition. When ramping is disabled, the entire +// Duration runs at opts.Rate. A ramp starts at opts.startRate (RateStep when +// Rate is unset), climbs by RateStep per level up to RateStepMax, and divides +// Duration evenly across the levels, so it always spans exactly Duration and +// ends exactly at RateStepMax. Passing more than one doOp fans the phase out +// across independent send targets, as the symmetric runners do (see [RunPhase]). +func runSchedule(ctx context.Context, ticker *Ticker, opts Options, doOps ...func() error) error { + if !opts.rampEnabled() { + start := time.Now() + return RunPhase(ctx, opts.Workers, opts.Rate, start, start.Add(opts.Duration), doOps...) + } + // Rate-ramp mode: one phase per offered-load level. + numSteps := opts.rampSteps() + stepDur := opts.Duration / time.Duration(numSteps) + currentRate := opts.startRate() + for i := range numSteps { + dur := stepDur + if i == numSteps-1 { + // The last level absorbs the integer-division remainder. + dur = opts.Duration - time.Duration(i)*stepDur + } + stepStart := time.Now() + if err := RunPhase(ctx, opts.Workers, currentRate, stepStart, stepStart.Add(dur), doOps...); err != nil { + return err + } + if i < numSteps-1 { + currentRate = min(currentRate+opts.RateStep, opts.RateStepMax) + ticker.RateStep(int64(currentRate)) + } + } + return nil +} + +// StartRemote resets the remote servers' op counters and Stats baselines at the +// start of a client-measured run, so the Stop reply observes only the work done +// in the measurement window. It is a no-op in local mode, where the in-process +// servers share the client's heap and their memory stats cannot be separated. +func StartRemote(cc *gorums.ConfigContext, opts Options) error { + if !opts.Remote { + return nil + } + _, err := Start(cc, StartRequest_builder{StatsMode: opts.StatsMode}.Build()).All() + return err +} + +// StopRemote collects each remote server's per-op memory stats via +// [Control.Stop] and appends them to result. It returns the per-server Stop +// replies keyed by node ID so callers can verify them (see [WithVerify]). It +// is the counterpart to [StartRemote] and is likewise a no-op in local mode, +// where it returns a nil map. +func StopRemote(cc *gorums.ConfigContext, opts Options, result *Result) (map[uint32]*Result, error) { + if !opts.Remote { + return nil, nil + } + replies, err := CollectReplies(Stop(cc, &StopRequest{}).Results()) + if err != nil { + return nil, err + } + AppendServerStats(result, replies) + return replies, nil +} + +// MeasureLatency runs the client-measured measurement window over one or more +// configurations and returns the Result built from the accumulated latency +// samples. setup is invoked once per yielded configuration (outside the loop) to +// build the request message and bind the context; the returned closure is the +// tight per-operation send, which MeasureLatency times and records via +// [Stats.AddLatency]. Yielding a single configuration drives the coordinator +// case (one send target); yielding one configuration per peer system drives +// the symmetric case (every system sends concurrently in one shared phase). +// +// MeasureLatency owns only the measurement window; any control-plane +// lifecycle ([StartRemote]/[StopRemote], local server resets) is the +// caller's concern. +func MeasureLatency(ctx context.Context, opts Options, configs iter.Seq[gorums.Config], setup func(opts Options, cc *gorums.ConfigContext) func() error) (*Result, error) { + m := StartMeasurement(opts) + var doOps []func() error + for cfg := range configs { + op := setup(opts, cfg.Context(ctx)) + doOps = append(doOps, func() error { + start := time.Now() + err := op() + if err != nil { + if errors.Is(err, ErrRunOver) { + // The workload declared the run over: pass the sentinel + // through to RunPhase so this worker stops, recording the + // op neither as a success nor as a failure. + return err + } + // Count the failed op and continue: a saturating workload can + // see failed quorum calls, and aborting the run on the first + // one turns graceful degradation into a crash. FailedOps records + // them; only latencies of successful ops feed TotalOps and the + // latency distribution. + m.RecordError() + return nil + } + m.Stats.AddLatency(time.Since(start)) + return nil + }) + } + if err := runMeasure(ctx, m.ticker, opts, doOps...); err != nil { + m.stop() + return nil, err + } + return m.Finish(), nil +} + +// MeasureOneWay builds the client-side send window for a server-measured run. It +// wraps each one-way send into a doOp that records a client op on success (the +// op count feeds only the throughput time-series; latency is measured server +// side) and returns the Measurement together with a window closure that starts +// the measurement, runs the paced phase via runMeasure, and stops the +// measurement. Server-measured runners therefore get rate ramping and ticker +// rate-step events without counting work outside the send window. Passing more +// than one send fans the window out across independent targets, as the +// symmetric multicast runner does. +// +// MeasureOneWay owns only the send window: the caller feeds window into +// RunOffsetCorrected as the measure step and Attaches the resulting Result to +// the returned Measurement. Any pre/post-window work (server resets, client +// memory snapshots, outbound flushes) stays with the caller. +func MeasureOneWay(ctx context.Context, opts Options, sends ...func() error) (*Measurement, func() error) { + m := newMeasurement(opts) + doOps := make([]func() error, len(sends)) + for i, send := range sends { + doOps[i] = func() error { + if err := send(); err != nil { + return err + } + m.Stats.AddOp() + return nil + } + } + window := func() error { + m.start() + defer m.stop() + return runMeasure(ctx, m.ticker, opts, doOps...) + } + return m, window +} + +// LifecycleOption customizes the reusable lifecycles built by ClientMeasured +// and ServerMeasured with optional hooks. +type LifecycleOption func(*lifecycleHooks) + +// lifecycleHooks holds the optional hooks of one lifecycle. The zero value +// disables every hook. +type lifecycleHooks struct { + quiesce func(context.Context) error + verify func(map[uint32]*Result) error +} + +// newLifecycleHooks applies opts to a zero hook set. +func newLifecycleHooks(opts []LifecycleOption) lifecycleHooks { + var h lifecycleHooks + for _, o := range opts { + o(&h) + } + return h +} + +// runQuiesce invokes the quiesce hook when one is registered. +func (h lifecycleHooks) runQuiesce(ctx context.Context) error { + if h.quiesce == nil { + return nil + } + return h.quiesce(ctx) +} + +// WithQuiesce registers a drain hook invoked after the measurement window +// closes and before Control.Stop collects the server-side statistics, letting +// a benchmark drain in-flight operations so the boundary measurement observes +// them. The context is the run's BenchContext; a non-nil error fails the run. +func WithQuiesce(f func(context.Context) error) LifecycleOption { + return func(h *lifecycleHooks) { h.quiesce = f } +} + +// runVerify invokes the verify hook when one is registered. +func (h lifecycleHooks) runVerify(replies map[uint32]*Result) error { + if h.verify == nil { + return nil + } + return h.verify(replies) +} + +// WithVerify registers a correctness check over the per-server Control.Stop +// replies, keyed by node ID, invoked after the measurement completes. A +// non-nil error fails the run, so no result file is written for a run that +// does not verify (e.g. reject when per-server TotalOps diverge beyond a +// tolerance). In a local client-measured run there are no remote replies and +// the hook receives a nil map. +func WithVerify(f func(map[uint32]*Result) error) LifecycleOption { + return func(h *lifecycleHooks) { h.verify = f } +} + +// ClientMeasured builds a [Bench].Run closure for the standard +// client-measured lifecycle: (in remote mode) [Control.Start], a paced +// measurement window timed on the client via [Stats] from t=0, and finally +// [Control.Stop] to collect each server's memory stats. The protocol author +// supplies setup, which binds a configuration context and returns the tight +// per-operation closure; building the request message and binding the +// context happen once, outside the measurement loop. Optional lifecycle +// hooks (e.g. [WithQuiesce]) run between the window and Control.Stop. +func ClientMeasured(cfg gorums.Config, setup func(opts Options, cc *gorums.ConfigContext) func() error, lifecycleOpts ...LifecycleOption) func(Options) (*Result, error) { + hooks := newLifecycleHooks(lifecycleOpts) + return func(opts Options) (*Result, error) { + ctx, cancel := BenchContext(opts) + defer cancel() + cc := cfg.Context(ctx) + if err := StartRemote(cc, opts); err != nil { + return nil, err + } + result, err := MeasureLatency(ctx, opts, slices.Values([]gorums.Config{cfg}), setup) + if err != nil { + return nil, err + } + if err := hooks.runQuiesce(ctx); err != nil { + return nil, err + } + replies, err := StopRemote(cc, opts, result) + if err != nil { + return nil, err + } + if err := hooks.runVerify(replies); err != nil { + return nil, err + } + return result, nil + } +} + +// ServerMeasured builds a [Bench].Run closure for the standard +// server-measured lifecycle: [Control.Start], a paced one-way send window +// from t=0 with clock-offset correction around it, and [Control.Stop] to +// collect each server's latency samples. The protocol author supplies setup, +// which binds a configuration context and returns the tight per-send closure +// (it stamps and sends one one-way message). A failed send aborts the run and +// is not counted as an operation, as in [ClientMeasured]. Optional lifecycle +// hooks (e.g. [WithQuiesce]) run between the send window and Control.Stop, +// where a drain ensures the servers observe every in-flight one-way send. +func ServerMeasured(cfg gorums.Config, setup func(opts Options, cc *gorums.ConfigContext) func() error, lifecycleOpts ...LifecycleOption) func(Options) (*Result, error) { + hooks := newLifecycleHooks(lifecycleOpts) + return func(opts Options) (*Result, error) { + ctx, cancel := BenchContext(opts) + defer cancel() + cc := cfg.Context(ctx) + send := setup(opts, cc) + var startMem, endMem memSnapshot + var replies map[uint32]*Result + + m, window := MeasureOneWay(ctx, opts, send) + measure := func() error { + // Start, the memory snapshots, and Stop are issued here, inside the + // window estimate() brackets, so the two clock-sync phases (50 + // sequential RPC rounds each) never inflate client per-op memory + // accounting or the server-measured throughput window. + if _, err := Start(cc, StartRequest_builder{StatsMode: opts.StatsMode}.Build()).All(); err != nil { + return err + } + startMem.read() + if err := window(); err != nil { + return err + } + if err := hooks.runQuiesce(ctx); err != nil { + return err + } + endMem.read() + var err error + replies, err = CollectReplies(Stop(cc, &StopRequest{}).Results()) + return err + } + estimate := func() (map[uint32]int64, error) { + return EstimateOffsets(ctx, cfg) + } + buildReplies := func(before, after map[uint32]int64) (map[uint32]*Result, error) { + LogOffsets("servers", before, after) + offsets := AverageOffsets(before, after) + for id, reply := range replies { + CorrectLatencies(reply, offsets[id]) + } + // Verify the corrected per-server replies: exactly what the + // aggregation step below will combine. + if err := hooks.runVerify(replies); err != nil { + return nil, err + } + return replies, nil + } + resp, err := RunOffsetCorrected(estimate, measure, buildReplies) + if err != nil { + m.stop() + return nil, err + } + m.Attach(resp) + + // Divide the client-side memory delta by the client's own send count; + // resp.GetTotalOps() aggregates over all servers (N x the sends for + // multicast) and would under-report the per-send client cost. + clientAllocs, clientMem := startMem.delta(endMem, m.Stats.Ops()) + resp.SetAllocsPerOp(clientAllocs) + resp.SetMemPerOp(clientMem) + return resp, nil + } +} + +// Run selects the benchmarks whose name matches sel, runs each with opts, stamps +// the shared run metadata onto each Result, and returns the results sorted by +// benchmark name. The mode metadata ("local"/"remote") is derived from +// opts.Remote. Run rejects invalid options (see [Options.Validate]) and a +// selector matching no benchmark, instead of returning an empty successful +// result set. +func Run(sel *regexp.Regexp, opts Options, benches []Bench) ([]*Result, error) { + if err := opts.Validate(); err != nil { + return nil, fmt.Errorf("invalid options: %w", err) + } + if opts.StreamMode == "" { + opts.StreamMode = "dual" + } + mode := "local" + if opts.Remote { + mode = "remote" + } + var matched bool + var results []*Result + for _, b := range benches { + if !sel.MatchString(b.Name) { + continue + } + matched = true + opts.BenchName = b.Name + result, err := b.Run(opts) + if err != nil { + return nil, err + } + // The runner stamped the measurement mode via Finish/Attach; preserve it + // while filling in the rest of the metadata from opts. + result.SetConfig(RunConfig_builder{ + Name: b.Name, + NumNodes: int32(opts.NumNodes), + Mode: mode, + Duration: int64(opts.Duration), + Workers: int32(opts.Workers), + Payload: int32(opts.Payload), + Rate: int64(opts.Rate), + Interval: int64(opts.Interval), + MeasurementMode: result.GetConfig().GetMeasurementMode(), + StatsMode: opts.StatsMode, + StreamMode: opts.StreamMode, + QuorumSize: int32(opts.QuorumSize), + MaxAsync: int32(opts.MaxAsync), + RateStep: int64(opts.RateStep), + RateStepMax: int64(opts.RateStepMax), + CallTimeout: int64(opts.CallTimeout), + SendBuffer: int32(opts.SendBuffer), + RecvBuffer: int32(opts.RecvBuffer), + }.Build()) + i := sort.Search(len(results), func(i int) bool { + return results[i].GetConfig().GetName() >= result.GetConfig().GetName() + }) + results = append(results, nil) + copy(results[i+1:], results[i:]) + results[i] = result + } + if !matched { + return nil, fmt.Errorf("no benchmarks match %q", sel) + } + return results, nil +} diff --git a/benchkit/harness_test.go b/benchkit/harness_test.go new file mode 100644 index 00000000..53bedb9b --- /dev/null +++ b/benchkit/harness_test.go @@ -0,0 +1,738 @@ +package benchkit + +import ( + "context" + "errors" + "regexp" + "slices" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/relab/gorums" + "github.com/relab/gorums/gorumstest" +) + +// TestRunPhase exercises the generalized RunPhase skeleton: error propagation, +// the no-op boundary conditions (zero workers, past deadline), and the +// multi-doOp fan-out where each doOp gets its own pool of workers. +func TestRunPhase(t *testing.T) { + errBoom := errors.New("boom") + + tests := []struct { + name string + numG int + numOps int // number of doOps to fan out across + past bool // endTime already elapsed before the phase starts + failing bool // doOps return errBoom on first call + wantErr error + // wantInvoked is true when at least one doOp must have run. + wantInvoked bool + }{ + {name: "PastDeadlineRunsNothing", numG: 4, numOps: 1, past: true, wantErr: nil, wantInvoked: false}, + {name: "ZeroWorkersRunsNothing", numG: 0, numOps: 3, wantErr: nil, wantInvoked: false}, + {name: "SingleDoOpRuns", numG: 1, numOps: 1, wantErr: nil, wantInvoked: true}, + {name: "FanOutRunsEveryDoOp", numG: 2, numOps: 3, wantErr: nil, wantInvoked: true}, + {name: "ErrorPropagates", numG: 1, numOps: 1, failing: true, wantErr: errBoom, wantInvoked: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // invoked is a bit vector: bit i is set once doOp i has run. + // allBits is the value with every doOp's bit set. + var invoked atomic.Int32 + allBits := int32(1)< time.Second { + t.Errorf("RunPhase returned after %v; want prompt stop on cancel, not spin to endTime", elapsed) + } + if n := ops.Load(); n > 1000 { + t.Errorf("ops after cancel = %d, want a handful (workers must stop, not spin)", n) + } +} + +// TestRunPhaseRunOverStopsWorker verifies that a doOp returning ErrRunOver +// stops its worker cleanly: the phase ends promptly without treating the +// sentinel as a fault, so RunPhase returns nil. +func TestRunPhaseRunOverStopsWorker(t *testing.T) { + var ops atomic.Int64 + start := time.Now() + err := RunPhase(context.Background(), 2, 0, start, start.Add(2*time.Second), func() error { + ops.Add(1) + return ErrRunOver + }) + if err != nil { + t.Fatalf("RunPhase err = %v, want nil (ErrRunOver is a clean stop, not a fault)", err) + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Errorf("RunPhase returned after %v; want prompt stop on ErrRunOver", elapsed) + } + // Each of the 2 workers stops on its first ErrRunOver. + if n := ops.Load(); n != 2 { + t.Errorf("ops = %d, want 2 (one per worker before stopping)", n) + } +} + +// TestMeasureLatencyRunOverNotRecorded verifies that ErrRunOver stops its +// worker without changing the success or failure counts. +func TestMeasureLatencyRunOverNotRecorded(t *testing.T) { + opts := Options{Workers: 2, Duration: 2 * time.Second} + var calls atomic.Int64 + setup := func(_ Options, _ *gorums.ConfigContext) func() error { + return func() error { + if calls.Add(1) > 3 { + return ErrRunOver + } + return nil + } + } + start := time.Now() + configs := slices.Values([]gorums.Config{gorumstest.NoDialedConfig(t)}) + result, err := MeasureLatency(context.Background(), opts, configs, setup) + if err != nil { + t.Fatalf("MeasureLatency err = %v, want nil", err) + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Errorf("MeasureLatency returned after %v; want prompt stop on ErrRunOver", elapsed) + } + if got := result.GetTotalOps(); got != 3 { + t.Errorf("TotalOps = %d, want 3 (only pre-run-over ops recorded)", got) + } + if got := result.GetFailedOps(); got != 0 { + t.Errorf("FailedOps = %d, want 0 (run over is not a failure)", got) + } +} + +// TestRunOffsetCorrected verifies the estimate→measure→estimate→aggregate +// sequencing and that an error at any stage short-circuits the remaining +// stages, leaving later stages uncalled. +func TestRunOffsetCorrected(t *testing.T) { + errEst := errors.New("estimate failed") + errMeasure := errors.New("measure failed") + errReplies := errors.New("buildReplies failed") + + // validReplies yields a single non-empty reply so AggregateServerResults + // succeeds on the happy path. + validReplies := func() map[uint32]*Result { + return map[uint32]*Result{1: Result_builder{TotalOps: 4, Latencies: []int64{1, 2, 3, 4}}.Build()} + } + + tests := []struct { + name string + // failAtEstimate: 0 never, 1 first call, 2 second call. + failAtEstimate int + failMeasure bool + failReplies bool + emptyReplies bool // buildReplies returns an empty map → aggregate ErrIncomplete + wantErr error + wantEstimate int // expected estimate() call count + wantMeasure int // expected measure() call count + wantReplies int // expected buildReplies() call count + wantTotalOps uint64 + }{ + {name: "HappyPath", wantErr: nil, wantEstimate: 2, wantMeasure: 1, wantReplies: 1, wantTotalOps: 4}, + {name: "EstimateBeforeErrors", failAtEstimate: 1, wantErr: errEst, wantEstimate: 1, wantMeasure: 0, wantReplies: 0}, + {name: "MeasureErrors", failMeasure: true, wantErr: errMeasure, wantEstimate: 1, wantMeasure: 1, wantReplies: 0}, + {name: "EstimateAfterErrors", failAtEstimate: 2, wantErr: errEst, wantEstimate: 2, wantMeasure: 1, wantReplies: 0}, + {name: "BuildRepliesErrors", failReplies: true, wantErr: errReplies, wantEstimate: 2, wantMeasure: 1, wantReplies: 1}, + {name: "EmptyRepliesIncomplete", emptyReplies: true, wantErr: gorums.ErrIncomplete, wantEstimate: 2, wantMeasure: 1, wantReplies: 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var estimateCalls, measureCalls, repliesCalls int + estimate := func() (int, error) { + estimateCalls++ + if tt.failAtEstimate == estimateCalls { + return 0, errEst + } + return estimateCalls, nil + } + measure := func() error { + measureCalls++ + if tt.failMeasure { + return errMeasure + } + return nil + } + buildReplies := func(before, after int) (map[uint32]*Result, error) { + repliesCalls++ + if tt.failReplies { + return nil, errReplies + } + if tt.emptyReplies { + return nil, nil + } + return validReplies(), nil + } + + r, err := RunOffsetCorrected(estimate, measure, buildReplies) + if !errors.Is(err, tt.wantErr) { + t.Fatalf("RunOffsetCorrected err = %v, want %v", err, tt.wantErr) + } + if estimateCalls != tt.wantEstimate { + t.Errorf("estimate calls = %d, want %d", estimateCalls, tt.wantEstimate) + } + if measureCalls != tt.wantMeasure { + t.Errorf("measure calls = %d, want %d", measureCalls, tt.wantMeasure) + } + if repliesCalls != tt.wantReplies { + t.Errorf("buildReplies calls = %d, want %d", repliesCalls, tt.wantReplies) + } + if tt.wantErr == nil && r.GetTotalOps() != tt.wantTotalOps { + t.Errorf("TotalOps = %d, want %d", r.GetTotalOps(), tt.wantTotalOps) + } + }) + } +} + +// TestRunMeasureSinglePhase verifies that runMeasure runs a single phase when +// rate ramp options are zero. +func TestRunMeasureSinglePhase(t *testing.T) { + var ops atomic.Int64 + s := NewStats(StatsMode_EXACT) + tk := NewTicker(0, s) + tk.Start(0) + opts := Options{Workers: 2, Rate: 0, Duration: 100 * time.Millisecond} + if err := runMeasure(context.Background(), tk, opts, func() error { + ops.Add(1) + return nil + }); err != nil { + t.Fatalf("runMeasure: %v", err) + } + tk.Stop() + if ops.Load() == 0 { + t.Error("runMeasure: no ops executed in single-phase mode") + } +} + +// TestOptionsStartRate verifies the offered rate of the first measurement +// phase: opts.Rate normally, but RateStep when ramping is enabled and Rate is +// unset, so a ramped run climbs from the first step instead of starting +// unlimited and dropping. +func TestOptionsStartRate(t *testing.T) { + ramp := Options{RateStep: 50, RateStepMax: 200} + rampWithRate := ramp + rampWithRate.Rate = 100 + + tests := []struct { + name string + opts Options + want int + }{ + {"NoRampUnlimitedStaysUnlimited", Options{}, 0}, + {"NoRampUsesRate", Options{Rate: 7}, 7}, + {"RampWithRateKeepsRate", rampWithRate, 100}, + {"RampUnlimitedStartsAtRateStep", ramp, 50}, + {"PartialRampStaysUnlimited", Options{RateStep: 50}, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.opts.startRate(); got != tt.want { + t.Errorf("startRate() = %d, want %d", got, tt.want) + } + }) + } +} + +// TestRunMeasureRampUnlimitedStartsAtRateStep verifies that a ramp configured +// with Rate=0 starts at RateStep and climbs to RateStepMax: the START marker +// carries RateStep and the RATE_STEP markers carry increasing rates above it. +func TestRunMeasureRampUnlimitedStartsAtRateStep(t *testing.T) { + opts := Options{ + Workers: 1, + Rate: 0, // unlimited; ramp must override the start rate + Duration: 300 * time.Millisecond, + Interval: time.Hour, // capture phase markers without background ticks + RateStep: 50, + RateStepMax: 200, + } + m := StartMeasurement(opts) + if err := runMeasure(context.Background(), m.ticker, opts, func() error { return nil }); err != nil { + t.Fatalf("runMeasure: %v", err) + } + m.ticker.Stop() + + events := m.ticker.Events() + if ph := events[0].GetPhase(); ph == nil || ph.GetPhase() != PhaseMarker_START || ph.GetRate() != 50 { + t.Errorf("events[0] = %v, want START with rate 50", events[0]) + } + var rates []int64 + for _, ev := range events { + if ph := ev.GetPhase(); ph != nil && ph.GetPhase() == PhaseMarker_RATE_STEP { + rates = append(rates, ph.GetRate()) + } + } + // 4 levels (50, 100, 150, 200) over 300ms → 3 transitions. + if len(rates) != 3 || rates[0] != 100 || rates[1] != 150 || rates[2] != 200 { + t.Errorf("RATE_STEP rates = %v, want [100 150 200]", rates) + } +} + +// TestOptionsOfferedOps verifies the number of sends the offered-load schedule +// asks of one send target: rate × duration, summed per level when ramping, and +// 0 for an unlimited (closed-loop) run. +func TestOptionsOfferedOps(t *testing.T) { + tests := []struct { + name string + opts Options + want float64 + }{ + {"Unlimited", Options{Duration: time.Second}, 0}, + {"Paced", Options{Rate: 100, Duration: 2 * time.Second}, 200}, + // Levels 100, 200, 300 × 1s each. + {"Ramp", Options{RateStep: 100, RateStepMax: 300, Duration: 3 * time.Second}, 600}, + // Levels 200, 300, 400 × 1s each. + {"RampWithStartRate", Options{Rate: 200, RateStep: 100, RateStepMax: 400, Duration: 3 * time.Second}, 900}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.opts.OfferedOps(); got != tt.want { + t.Errorf("OfferedOps() = %v, want %v", got, tt.want) + } + }) + } +} + +// TestPaceWarning verifies that a warning is produced exactly when a paced run +// attempted markedly fewer sends than its offered-load schedule asked for, and +// that the message names the worker-sizing fix. +func TestPaceWarning(t *testing.T) { + tests := []struct { + name string + sent uint64 + offered float64 + wantWarn bool + }{ + {"OnSchedule", 100, 100, false}, + {"WithinTolerance", 96, 100, false}, + {"Behind", 50, 100, true}, + {"JustBelowTolerance", 94, 100, true}, + {"NoSchedule", 0, 0, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + msg := PaceWarning(tt.sent, tt.offered) + if got := msg != ""; got != tt.wantWarn { + t.Fatalf("PaceWarning(%d, %v) = %q, want warning: %v", tt.sent, tt.offered, msg, tt.wantWarn) + } + if tt.wantWarn && !strings.Contains(msg, "-workers") { + t.Errorf("warning %q does not mention -workers", msg) + } + }) + } +} + +// TestOptionsRampSteps verifies the number of offered-load levels derived from +// the ramp options: one per RateStep increment from the start rate up to and +// including RateStepMax, with a partial final increment still counting as a +// level. +func TestOptionsRampSteps(t *testing.T) { + tests := []struct { + name string + opts Options + want int + }{ + {"ExactSpan", Options{RateStep: 50, RateStepMax: 200}, 4}, // 50,100,150,200 + {"WithStartRate", Options{Rate: 100, RateStep: 100, RateStepMax: 300}, 3}, // 100,200,300 + {"PartialLastStep", Options{RateStep: 100, RateStepMax: 250}, 3}, // 100,200,250 + {"StartAtMax", Options{Rate: 200, RateStep: 50, RateStepMax: 200}, 1}, // 200 + {"StartAboveMax", Options{Rate: 500, RateStep: 50, RateStepMax: 200}, 1}, // 500 + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.opts.rampSteps(); got != tt.want { + t.Errorf("rampSteps() = %d, want %d", got, tt.want) + } + }) + } +} + +// TestRunMeasureRampEmitsRateStepEvents verifies that runMeasure emits the +// expected number of RATE_STEP phase markers when ramping is configured. +func TestRunMeasureRampEmitsRateStepEvents(t *testing.T) { + s := NewStats(StatsMode_EXACT) + // Interval longer than the run: the buffer captures the synchronously emitted + // phase markers (START, RATE_STEP, STOP) without any background ticks firing. + tk := NewTicker(time.Hour, s) + + tk.Start(100) // start marker at rate 100 + + opts := Options{ + Workers: 1, + Rate: 100, + Duration: 300 * time.Millisecond, + RateStep: 100, + RateStepMax: 300, + } + var opsCount atomic.Int64 + if err := runMeasure(context.Background(), tk, opts, func() error { + opsCount.Add(1) + return nil + }); err != nil { + t.Fatalf("runMeasure: %v", err) + } + tk.Stop() // STOP + + // Count RATE_STEP phase markers: 3 levels (100, 200, 300) → 2 transitions. + var rateSteps int + for _, ev := range tk.Events() { + if ph := ev.GetPhase(); ph != nil && ph.GetPhase() == PhaseMarker_RATE_STEP { + rateSteps++ + } + } + if rateSteps != 2 { + t.Errorf("rateSteps = %d, want 2", rateSteps) + } + if opsCount.Load() == 0 { + t.Error("no ops executed during ramp") + } +} + +// validOptions returns a minimal Options value that passes Validate, so each +// test case below only needs to override the field under test. +func validOptions() Options { + return Options{Workers: 1, Duration: time.Second} +} + +// TestOptionsValidate verifies the generic option constraints [Run] enforces +// before executing any benchmark: bad flag values must fail fast with a +// specific message instead of panicking or silently producing a zero-work +// run. +func TestOptionsValidate(t *testing.T) { + tests := []struct { + name string + opts Options + wantErr bool + }{ + {"Valid", validOptions(), false}, + {"ZeroWorkers", func() Options { o := validOptions(); o.Workers = 0; return o }(), true}, + {"NegativeWorkers", func() Options { o := validOptions(); o.Workers = -1; return o }(), true}, + {"ZeroDuration", func() Options { o := validOptions(); o.Duration = 0; return o }(), true}, + {"NegativeDuration", func() Options { o := validOptions(); o.Duration = -time.Second; return o }(), true}, + {"NegativePayload", func() Options { o := validOptions(); o.Payload = -1; return o }(), true}, + {"NegativeRate", func() Options { o := validOptions(); o.Rate = -1; return o }(), true}, + {"NegativeInterval", func() Options { o := validOptions(); o.Interval = -time.Second; return o }(), true}, + {"NegativeCallTimeout", func() Options { o := validOptions(); o.CallTimeout = -time.Second; return o }(), true}, + {"NegativeRateStep", func() Options { o := validOptions(); o.RateStep = -1; return o }(), true}, + {"NegativeRateStepMax", func() Options { o := validOptions(); o.RateStepMax = -1; return o }(), true}, + {"RateStepWithoutMax", func() Options { o := validOptions(); o.RateStep = 10; return o }(), true}, + {"RateStepMaxWithoutStep", func() Options { o := validOptions(); o.RateStepMax = 10; return o }(), true}, + {"RateStepMaxBelowStep", func() Options { o := validOptions(); o.RateStep = 100; o.RateStepMax = 50; return o }(), true}, + {"ValidRamp", func() Options { o := validOptions(); o.RateStep = 50; o.RateStepMax = 200; return o }(), false}, + {"EmptyStreamMode", func() Options { o := validOptions(); o.StreamMode = ""; return o }(), false}, + {"DualStreamMode", func() Options { o := validOptions(); o.StreamMode = "dual"; return o }(), false}, + {"DedupStreamMode", func() Options { o := validOptions(); o.StreamMode = "dedup"; return o }(), false}, + {"InvalidStreamMode", func() Options { o := validOptions(); o.StreamMode = "bogus"; return o }(), true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.opts.Validate() + if (err != nil) != tt.wantErr { + t.Errorf("Validate() = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +// TestRunRejectsInvalidOptions verifies that Run refuses to execute any +// benchmark when opts fails Validate. +func TestRunRejectsInvalidOptions(t *testing.T) { + opts := validOptions() + opts.Workers = 0 + var ran bool + benches := []Bench{{Name: "Bench", Run: func(Options) (*Result, error) { + ran = true + return &Result{}, nil + }}} + _, err := Run(regexp.MustCompile(".*"), opts, benches) + if err == nil { + t.Fatal("Run with invalid options = nil error, want error") + } + if ran { + t.Error("Run invoked the benchmark despite invalid options") + } +} + +// TestRunNoMatchingBenchmark verifies that a selector matching no benchmark +// fails instead of returning an empty, successful result set — a selector +// typo should not look like a clean zero-benchmark run. +func TestRunNoMatchingBenchmark(t *testing.T) { + benches := []Bench{{Name: "QuorumCall", Run: func(Options) (*Result, error) { + return &Result{}, nil + }}} + _, err := Run(regexp.MustCompile("^NoSuchBenchmark$"), validOptions(), benches) + if err == nil { + t.Fatal("Run with no matching benchmark = nil error, want error") + } +} + +// TestRunStampsFullConfig verifies that [Run] stamps every semantic option +// onto the result's [RunConfig], including QuorumSize, MaxAsync, RateStep, +// RateStepMax, and CallTimeout: two runs with different quorum size or +// rate-ramp settings must not serialize identical configs. +func TestRunStampsFullConfig(t *testing.T) { + opts := Options{ + Workers: 2, + Duration: time.Second, + Payload: 16, + Rate: 100, + Interval: 50 * time.Millisecond, + QuorumSize: 3, + MaxAsync: 500, + RateStep: 50, + RateStepMax: 200, + CallTimeout: 20 * time.Millisecond, + StatsMode: StatsMode_HDR, + StreamMode: "dedup", + NumNodes: 4, + Remote: true, + } + benches := []Bench{{Name: "Bench", Run: func(Options) (*Result, error) { return &Result{}, nil }}} + results, err := Run(regexp.MustCompile(".*"), opts, benches) + if err != nil { + t.Fatalf("Run: %v", err) + } + if len(results) != 1 { + t.Fatalf("len(results) = %d, want 1", len(results)) + } + cfg := results[0].GetConfig() + checks := []struct { + name string + got any + want any + }{ + {"NumNodes", cfg.GetNumNodes(), int32(4)}, + {"Mode", cfg.GetMode(), "remote"}, + {"Workers", cfg.GetWorkers(), int32(2)}, + {"Payload", cfg.GetPayload(), int32(16)}, + {"Rate", cfg.GetRate(), int64(100)}, + {"Interval", cfg.GetInterval(), int64(50 * time.Millisecond)}, + {"QuorumSize", cfg.GetQuorumSize(), int32(3)}, + {"MaxAsync", cfg.GetMaxAsync(), int32(500)}, + {"RateStep", cfg.GetRateStep(), int64(50)}, + {"RateStepMax", cfg.GetRateStepMax(), int64(200)}, + {"CallTimeout", cfg.GetCallTimeout(), int64(20 * time.Millisecond)}, + {"StatsMode", cfg.GetStatsMode(), StatsMode_HDR}, + {"StreamMode", cfg.GetStreamMode(), "dedup"}, + } + for _, c := range checks { + if c.got != c.want { + t.Errorf("RunConfig.%s = %v, want %v", c.name, c.got, c.want) + } + } +} + +// TestClientMeasuredCountsErrorsWithoutAborting verifies that client-measured +// operations count errors while allowing the run to complete. +func TestClientMeasuredCountsErrorsWithoutAborting(t *testing.T) { + var calls atomic.Int64 + run := ClientMeasured(gorumstest.NoDialedConfig(t), + func(_ Options, _ *gorums.ConfigContext) func() error { + return func() error { + // Fail every other op; persistent errors must not abort. + if calls.Add(1)%2 == 0 { + return errors.New("quorum call failed") + } + return nil + } + }) + result, err := run(Options{Workers: 1, Duration: 30 * time.Millisecond}) + if err != nil { + t.Fatalf("run aborted on op error: %v", err) + } + if result.GetTotalOps() == 0 { + t.Error("TotalOps = 0, want > 0 (successful ops must still be recorded)") + } + if result.GetFailedOps() == 0 { + t.Error("FailedOps = 0, want > 0 (op errors must be counted)") + } +} + +// TestClientMeasuredQuiesce verifies that the WithQuiesce hook runs after the +// measurement window closes (every op already recorded) and that a quiesce +// error fails the run. +func TestClientMeasuredQuiesce(t *testing.T) { + var opsDone atomic.Int64 + var opsAtQuiesce int64 + quiesceCalls := 0 + run := ClientMeasured(gorumstest.NoDialedConfig(t), + func(_ Options, _ *gorums.ConfigContext) func() error { + return func() error { + opsDone.Add(1) + return nil + } + }, + WithQuiesce(func(context.Context) error { + quiesceCalls++ + opsAtQuiesce = opsDone.Load() + return nil + })) + + result, err := run(Options{Workers: 2, Duration: 20 * time.Millisecond}) + if err != nil { + t.Fatalf("run: %v", err) + } + if quiesceCalls != 1 { + t.Fatalf("quiesce calls = %d, want 1", quiesceCalls) + } + if opsAtQuiesce != opsDone.Load() { + t.Errorf("ops at quiesce = %d, want %d (window must be closed before quiesce)", + opsAtQuiesce, opsDone.Load()) + } + if result.GetTotalOps() == 0 { + t.Error("TotalOps = 0, want > 0") + } + + errDrain := errors.New("drain failed") + failing := ClientMeasured(gorumstest.NoDialedConfig(t), + func(_ Options, _ *gorums.ConfigContext) func() error { + return func() error { return nil } + }, + WithQuiesce(func(context.Context) error { return errDrain })) + if _, err := failing(Options{Workers: 1, Duration: time.Millisecond}); !errors.Is(err, errDrain) { + t.Errorf("run with failing quiesce = %v, want %v", err, errDrain) + } +} + +// TestClientMeasuredVerify verifies the WithVerify hook: in a local run there +// are no remote Stop replies, so the hook receives a nil map, and a non-nil +// verify error fails the run. +func TestClientMeasuredVerify(t *testing.T) { + verifyCalls := 0 + var gotReplies map[uint32]*Result + setup := func(_ Options, _ *gorums.ConfigContext) func() error { + return func() error { return nil } + } + run := ClientMeasured(gorumstest.NoDialedConfig(t), setup, + WithVerify(func(replies map[uint32]*Result) error { + verifyCalls++ + gotReplies = replies + return nil + })) + if _, err := run(Options{Workers: 1, Duration: time.Millisecond}); err != nil { + t.Fatalf("run: %v", err) + } + if verifyCalls != 1 { + t.Fatalf("verify calls = %d, want 1", verifyCalls) + } + if gotReplies != nil { + t.Errorf("verify replies = %v, want nil in local mode", gotReplies) + } + + errVerify := errors.New("ops diverged") + failing := ClientMeasured(gorumstest.NoDialedConfig(t), setup, + WithVerify(func(map[uint32]*Result) error { return errVerify })) + if _, err := failing(Options{Workers: 1, Duration: time.Millisecond}); !errors.Is(err, errVerify) { + t.Errorf("run with failing verify = %v, want %v", err, errVerify) + } +} + +// TestOptionsServerOptions verifies which server options a run's configuration +// implies. These carry the stream topology and the buffer capacities to the +// server; BufferSizesOption is unconditional, so only StreamDedupOption varies +// the count. +func TestOptionsServerOptions(t *testing.T) { + tests := []struct { + name string + opts Options + wantCount int + wantDedup bool + }{ + { + name: "NothingConfigured", + wantCount: 1, + }, + { + name: "DedupOnly", + opts: Options{StreamMode: "dedup"}, + wantCount: 2, wantDedup: true, + }, + { + // Zero is a real receive-buffer size, and still produces an option. + name: "RecvBufferZero", + opts: Options{RecvBuffer: 0}, + wantCount: 1, + }, + { + name: "DedupAndBuffers", + opts: Options{StreamMode: "dedup", SendBuffer: 256, RecvBuffer: 16}, + wantCount: 2, wantDedup: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := len(tt.opts.ServerOptions()); got != tt.wantCount { + t.Errorf("ServerOptions() length = %d, want %d", got, tt.wantCount) + } + if got := tt.opts.StreamDedupOption() != nil; got != tt.wantDedup { + t.Errorf("StreamDedupOption() non-nil = %v, want %v", got, tt.wantDedup) + } + if tt.opts.BufferSizesOption() == nil { + t.Error("BufferSizesOption() = nil, want non-nil") + } + }) + } +} diff --git a/benchkit/hdr.go b/benchkit/hdr.go new file mode 100644 index 00000000..35c09c05 --- /dev/null +++ b/benchkit/hdr.go @@ -0,0 +1,250 @@ +package benchkit + +import ( + "fmt" + "iter" + "math" + "math/bits" +) + +// Histogram is a bounded-memory, log-linear histogram of non-negative int64 +// values (latencies in nanoseconds), after the HdrHistogram design +// (hdrhistogram.org): bucket widths grow with the value's magnitude so every +// recorded value is resolved to the configured number of significant figures, +// in O(1) record time and constant memory. It backs StatsMode_HDR (see +// store.go). +// +// The API mirrors the common HdrHistogram bindings (RecordValue, +// ValueAtQuantile, Mean, StdDev, TotalCount, Min, Max), so switching to a +// library implementation later is mechanical. Histogram is not thread-safe; +// callers serialize access (Stats records under its mutex). +type Histogram struct { + lowest, highest int64 + unitMagnitude int + subBucketHalfCountMagnitude int + subBucketCount int + subBucketHalfCount int + subBucketMask int64 + bucketCount int + counts []uint64 + totalCount uint64 + minValue, maxValue int64 +} + +// NewHistogram returns a histogram tracking values in [lowest, highest] with +// sigfigs significant figures (1..5). lowest is the smallest value resolved at +// full precision (1 for nanosecond resolution); smaller recorded values are +// not lost but resolve coarsely toward zero. NewHistogram panics on invalid +// parameters, mirroring the HdrHistogram constructors: the parameters are +// compile-time constants of the caller, not runtime input. +func NewHistogram(lowest, highest int64, sigfigs int) *Histogram { + if lowest < 1 || highest < 2*lowest { + panic(fmt.Sprintf("benchkit.NewHistogram: need 1 <= lowest and highest >= 2*lowest, got [%d, %d]", lowest, highest)) + } + if sigfigs < 1 || sigfigs > 5 { + panic(fmt.Sprintf("benchkit.NewHistogram: sigfigs must be in [1, 5], got %d", sigfigs)) + } + // The sub-bucket count is the smallest power of two resolving sigfigs + // decimal digits across one power-of-two range. + largestSingleUnit := 2 * math.Pow10(sigfigs) + subBucketCountMagnitude := int(math.Ceil(math.Log2(largestSingleUnit))) + subBucketHalfCountMagnitude := max(subBucketCountMagnitude-1, 0) + unitMagnitude := max(int(math.Floor(math.Log2(float64(lowest)))), 0) + subBucketCount := 1 << (subBucketHalfCountMagnitude + 1) + + h := &Histogram{ + lowest: lowest, + highest: highest, + unitMagnitude: unitMagnitude, + subBucketHalfCountMagnitude: subBucketHalfCountMagnitude, + subBucketCount: subBucketCount, + subBucketHalfCount: subBucketCount / 2, + subBucketMask: int64(subBucketCount-1) << unitMagnitude, + minValue: math.MaxInt64, + } + // One bucket spans [0, subBucketCount) units; each further bucket doubles + // the range, reusing the upper half of the sub-buckets at double width. + // The loop bound is <=, not <: highest itself must be trackable, so a + // smallestUntrackable that lands exactly on highest still needs one more + // bucket to cover it. + h.bucketCount = 1 + for smallestUntrackable := int64(subBucketCount) << unitMagnitude; smallestUntrackable <= highest; smallestUntrackable <<= 1 { + h.bucketCount++ + } + h.counts = make([]uint64, (h.bucketCount+1)*h.subBucketHalfCount) + return h +} + +// bucketIndex returns the power-of-two bucket holding v. +func (h *Histogram) bucketIndex(v int64) int { + pow2Ceiling := bits.Len64(uint64(v | h.subBucketMask)) + return pow2Ceiling - h.unitMagnitude - (h.subBucketHalfCountMagnitude + 1) +} + +// subBucketIndex returns v's linear sub-bucket within bucket bucketIdx. +func (h *Histogram) subBucketIndex(v int64, bucketIdx int) int { + return int(v >> (bucketIdx + h.unitMagnitude)) +} + +// countsIndex maps a (bucket, sub-bucket) pair to its slot in counts. Buckets +// beyond the first use only the upper half of their sub-buckets (the lower +// half aliases the previous bucket at half the width), so each contributes +// subBucketHalfCount slots. +func (h *Histogram) countsIndex(bucketIdx, subBucketIdx int) int { + base := (bucketIdx + 1) << h.subBucketHalfCountMagnitude + return base + subBucketIdx - h.subBucketHalfCount +} + +// valueFromIndex returns the lowest value of a (bucket, sub-bucket) pair. +func (h *Histogram) valueFromIndex(bucketIdx, subBucketIdx int) int64 { + return int64(subBucketIdx) << (bucketIdx + h.unitMagnitude) +} + +// rangeSize returns the width of the buckets at bucketIdx. +func (h *Histogram) rangeSize(bucketIdx int) int64 { + return int64(1) << (bucketIdx + h.unitMagnitude) +} + +// RecordValue records one value. It returns an error when v is outside the +// trackable range [0, highest], mirroring the HdrHistogram bindings; callers +// that must not lose samples clamp first (see [hdrStore]). +func (h *Histogram) RecordValue(v int64) error { + return h.RecordValueN(v, 1) +} + +// RecordValueN records n occurrences of v in O(1), for replaying a weighted +// (value, count) pair when re-quantizing one histogram onto another (see +// [Histogram.recordPairs]). It returns an error when v is outside the trackable +// range [0, highest], mirroring the HdrHistogram bindings; callers that must not +// lose samples clamp first. Recording zero occurrences is a no-op. +func (h *Histogram) RecordValueN(v int64, n uint64) error { + if v < 0 || v > h.highest { + return fmt.Errorf("value %d outside trackable range [0, %d]", v, h.highest) + } + if n == 0 { + return nil + } + bucketIdx := h.bucketIndex(v) + h.counts[h.countsIndex(bucketIdx, h.subBucketIndex(v, bucketIdx))] += n + h.totalCount += n + h.minValue = min(h.minValue, v) + h.maxValue = max(h.maxValue, v) + return nil +} + +// recordPairs adds each (value+delta, count) pair of the weighted sequence into +// h, clamping each shifted value into h's trackable range [0, highest]. It is +// the shared core of clock-offset correction and histogram merging: delta is a +// per-source clock offset (0 when merging already-corrected histograms). +// Clamping matches [hdrStore.Add], so a correction that pushes a value below +// zero or above the ceiling never drops the sample, whose count feeds +// throughput and the distribution. +func (h *Histogram) recordPairs(pairs iter.Seq2[int64, uint64], delta int64) { + for v, c := range pairs { + _ = h.RecordValueN(min(max(v+delta, 0), h.highest), c) + } +} + +// snapshot renders the histogram's occupied buckets as a LatencyHistogram +// message: ascending (value, count) pairs consumers treat as a weighted sample +// set. Returns nil when empty. +func (h *Histogram) snapshot() *LatencyHistogram { + if h.totalCount == 0 { + return nil + } + var values []int64 + var counts []uint64 + for v, c := range h.buckets() { + values = append(values, v) + counts = append(counts, c) + } + return LatencyHistogram_builder{Value: values, Count: counts}.Build() +} + +// TotalCount returns the number of recorded values. +func (h *Histogram) TotalCount() uint64 { return h.totalCount } + +// Min returns the lowest recorded value, exact (0 when empty). +func (h *Histogram) Min() int64 { + if h.totalCount == 0 { + return 0 + } + return h.minValue +} + +// Max returns the highest recorded value, exact (0 when empty). +func (h *Histogram) Max() int64 { return h.maxValue } + +// buckets yields each occupied bucket in ascending value order as the pair +// (median-equivalent value, count). The median-equivalent value — the middle +// of the bucket's range, as in HdrHistogram — represents every value recorded +// into the bucket within the configured precision, so consumers can treat the +// pairs as a weighted sample set. +func (h *Histogram) buckets() iter.Seq2[int64, uint64] { + return func(yield func(int64, uint64) bool) { + for bucketIdx := range h.bucketCount { + subLo := 0 + if bucketIdx > 0 { + subLo = h.subBucketHalfCount + } + for subBucketIdx := subLo; subBucketIdx < h.subBucketCount; subBucketIdx++ { + c := h.counts[h.countsIndex(bucketIdx, subBucketIdx)] + if c == 0 { + continue + } + median := h.valueFromIndex(bucketIdx, subBucketIdx) + h.rangeSize(bucketIdx)/2 + if !yield(median, c) { + return + } + } + } + } +} + +// ValueAtQuantile returns the highest value of the bucket below which q +// percent (q in [0, 100], as in the HdrHistogram bindings) of the recorded +// values fall, or 0 when empty. The result is within the configured +// significant figures of the exact quantile. +func (h *Histogram) ValueAtQuantile(q float64) int64 { + if h.totalCount == 0 { + return 0 + } + target := quantileRank(h.totalCount, q/100) + var cum uint64 + for bucketIdx := range h.bucketCount { + subLo := 0 + if bucketIdx > 0 { + subLo = h.subBucketHalfCount + } + for subBucketIdx := subLo; subBucketIdx < h.subBucketCount; subBucketIdx++ { + cum += h.counts[h.countsIndex(bucketIdx, subBucketIdx)] + if cum >= target { + // The highest value equivalent to this bucket. + return h.valueFromIndex(bucketIdx, subBucketIdx) + h.rangeSize(bucketIdx) - 1 + } + } + } + return h.maxValue +} + +// Mean returns the mean of the recorded values, computed over the bucket +// median-equivalent values (0 when empty). +func (h *Histogram) Mean() float64 { + mean, _ := weightedMeanStdDev(h.buckets()) + return mean +} + +// StdDev returns the population standard deviation of the recorded values, +// computed over the bucket median-equivalent values (0 when empty). +func (h *Histogram) StdDev() float64 { + _, stddev := weightedMeanStdDev(h.buckets()) + return stddev +} + +// Reset discards all recorded values, keeping the bucket layout. +func (h *Histogram) Reset() { + clear(h.counts) + h.totalCount = 0 + h.minValue = math.MaxInt64 + h.maxValue = 0 +} diff --git a/benchkit/hdr_test.go b/benchkit/hdr_test.go new file mode 100644 index 00000000..316b0c01 --- /dev/null +++ b/benchkit/hdr_test.go @@ -0,0 +1,200 @@ +package benchkit + +import ( + "math" + "math/rand/v2" + "slices" + "testing" + + "golang.org/x/exp/stats" +) + +// TestHistogramInvalidParameters verifies that NewHistogram rejects parameters +// outside the supported ranges, mirroring the HdrHistogram constructors. +func TestHistogramInvalidParameters(t *testing.T) { + tests := []struct { + name string + lowest, highest int64 + sigfigs int + }{ + {"LowestZero", 0, 1000, 3}, + {"HighestBelowTwiceLowest", 1000, 1500, 3}, + {"SigfigsZero", 1, 1000, 0}, + {"SigfigsTooLarge", 1, 1000, 6}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer func() { + if recover() == nil { + t.Errorf("NewHistogram(%d, %d, %d) did not panic", tt.lowest, tt.highest, tt.sigfigs) + } + }() + NewHistogram(tt.lowest, tt.highest, tt.sigfigs) + }) + } +} + +// TestHistogramRecordValue verifies basic recording: total count, exact +// min/max, range errors for untrackable values, and Reset. +func TestHistogramRecordValue(t *testing.T) { + h := NewHistogram(1, 1_000_000, 3) + for _, v := range []int64{1, 500, 999_999, 42, 42} { + if err := h.RecordValue(v); err != nil { + t.Fatalf("RecordValue(%d): %v", v, err) + } + } + if got := h.TotalCount(); got != 5 { + t.Errorf("TotalCount() = %d, want 5", got) + } + if got := h.Min(); got != 1 { + t.Errorf("Min() = %d, want 1", got) + } + if got := h.Max(); got != 999_999 { + t.Errorf("Max() = %d, want 999999", got) + } + if err := h.RecordValue(-1); err == nil { + t.Error("RecordValue(-1) = nil, want error") + } + if err := h.RecordValue(2_000_000); err == nil { + t.Error("RecordValue(2000000) = nil, want error") + } + + h.Reset() + if h.TotalCount() != 0 || h.Min() != 0 || h.Max() != 0 || h.ValueAtQuantile(50) != 0 { + t.Errorf("after Reset: count=%d min=%d max=%d p50=%d, want all zero", + h.TotalCount(), h.Min(), h.Max(), h.ValueAtQuantile(50)) + } +} + +// TestHistogramRecordValueAtHighestBoundary verifies that recording exactly at +// the configured highest value never panics, including when highest lands +// exactly on a bucket-count doubling boundary (a power-of-two multiple of the +// sub-bucket range), which previously undercounted the needed buckets by one. +func TestHistogramRecordValueAtHighestBoundary(t *testing.T) { + for _, tc := range []struct { + name string + lowest, highest int64 + sigfigs int + }{ + {"boundary-2sigfigs", 1, 1 << 20, 2}, + {"boundary-3sigfigs", 1, 2048, 3}, + {"non-boundary", 1, 1_000_000, 3}, + } { + t.Run(tc.name, func(t *testing.T) { + h := NewHistogram(tc.lowest, tc.highest, tc.sigfigs) + if err := h.RecordValue(tc.highest); err != nil { + t.Fatalf("RecordValue(highest=%d): %v", tc.highest, err) + } + if got := h.Max(); got != tc.highest { + t.Errorf("Max() = %d, want %d", got, tc.highest) + } + }) + } +} + +// TestHistogramRecordValueN verifies that RecordValueN records the given number +// of occurrences in one call, equivalent to that many RecordValue calls, that a +// zero count is a no-op, and that an out-of-range value errors without recording. +func TestHistogramRecordValueN(t *testing.T) { + bulk := NewHistogram(1, 1_000_000, 3) + one := NewHistogram(1, 1_000_000, 3) + + if err := bulk.RecordValueN(500, 4); err != nil { + t.Fatalf("RecordValueN(500, 4): %v", err) + } + for range 4 { + if err := one.RecordValue(500); err != nil { + t.Fatalf("RecordValue(500): %v", err) + } + } + if got := bulk.TotalCount(); got != 4 { + t.Errorf("TotalCount() = %d, want 4", got) + } + if bulk.Min() != one.Min() || bulk.Max() != one.Max() { + t.Errorf("bulk min/max = %d/%d, want %d/%d", bulk.Min(), bulk.Max(), one.Min(), one.Max()) + } + if !slices.Equal(bulk.counts, one.counts) { + t.Error("RecordValueN(v, n) counts differ from n * RecordValue(v)") + } + + if err := bulk.RecordValueN(700, 0); err != nil { + t.Fatalf("RecordValueN(700, 0): %v", err) + } + if got := bulk.TotalCount(); got != 4 { + t.Errorf("TotalCount() after zero-count record = %d, want 4", got) + } + if err := bulk.RecordValueN(2_000_000, 3); err == nil { + t.Error("RecordValueN(2000000, 3) = nil, want out-of-range error") + } + if got := bulk.TotalCount(); got != 4 { + t.Errorf("TotalCount() after out-of-range record = %d, want 4", got) + } +} + +// TestHistogramAccuracy verifies that quantiles, mean, and stddev computed +// from the histogram match the exact statistics of the recorded samples within +// the configured significant figures (3 sigfigs → 0.1% relative error per +// value, plus quantile granularity of one bucket). +func TestHistogramAccuracy(t *testing.T) { + const n = 100_000 + h := NewHistogram(1, 60_000_000_000, 3) + rng := rand.New(rand.NewPCG(1, 2)) + samples := make([]float64, n) + for i := range samples { + // Log-normal-ish latencies spanning ~1µs to ~100ms. + v := int64(1000 * math.Exp(rng.NormFloat64()*1.5+3)) + samples[i] = float64(v) + if err := h.RecordValue(v); err != nil { + t.Fatalf("RecordValue(%d): %v", v, err) + } + } + + for _, q := range []float64{50, 90, 99, 99.9} { + exact := stats.Quantiles(samples, q/100)[0] + got := float64(h.ValueAtQuantile(q)) + if relErr := math.Abs(got-exact) / exact; relErr > 0.005 { + t.Errorf("ValueAtQuantile(%v) = %.0f, exact %.0f (rel err %.4f > 0.005)", q, got, exact, relErr) + } + } + exactMean, exactStdDev := stats.MeanAndStdDev(samples) + if relErr := math.Abs(h.Mean()-exactMean) / exactMean; relErr > 0.001 { + t.Errorf("Mean() = %.0f, exact %.0f (rel err %.4f > 0.001)", h.Mean(), exactMean, relErr) + } + if relErr := math.Abs(h.StdDev()-exactStdDev) / exactStdDev; relErr > 0.005 { + t.Errorf("StdDev() = %.0f, exact %.0f (rel err %.4f > 0.005)", h.StdDev(), exactStdDev, relErr) + } +} + +// TestHistogramBuckets verifies that buckets yields (value, count) pairs in +// ascending value order, that the counts sum to the total, and that each +// recorded value is represented within the configured precision. +func TestHistogramBuckets(t *testing.T) { + h := NewHistogram(1, 1_000_000_000, 2) + recorded := []int64{100, 100, 5_000, 123_456, 999_999_999} + for _, v := range recorded { + if err := h.RecordValue(v); err != nil { + t.Fatalf("RecordValue(%d): %v", v, err) + } + } + var values []int64 + var total uint64 + for v, c := range h.buckets() { + values = append(values, v) + total += c + } + if !slices.IsSorted(values) { + t.Errorf("bucket values not ascending: %v", values) + } + if total != h.TotalCount() { + t.Errorf("bucket counts sum = %d, want %d", total, h.TotalCount()) + } + // Every recorded value must have a representative within 1% (2 sigfigs). + for _, want := range recorded { + ok := slices.ContainsFunc(values, func(v int64) bool { + return math.Abs(float64(v-want)) <= max(0.01*float64(want), 1) + }) + if !ok { + t.Errorf("no bucket value within 1%% of recorded %d: %v", want, values) + } + } +} diff --git a/benchkit/log.go b/benchkit/log.go new file mode 100644 index 00000000..6e5bd63f --- /dev/null +++ b/benchkit/log.go @@ -0,0 +1,33 @@ +package benchkit + +import ( + "fmt" + "os" +) + +var logVerbose bool + +// SetVerbose enables diagnostic output to stderr. Call once at program startup, +// before any goroutines that call Logf are started. +func SetVerbose(v bool) { logVerbose = v } + +// Logf writes a formatted diagnostic message to stderr when verbose logging is +// enabled. All diagnostic output in a sweep-launched binary must use Logf rather +// than writing to os.Stdout: sweep only drains stderr, and unread stdout fills +// the SSH channel window and blocks goroutines (see the diagnostic-output rule +// in doc/benchkit-troubleshooting.html). +func Logf(format string, args ...any) { + if logVerbose { + fmt.Fprintf(os.Stderr, format, args...) + } +} + +// Printf writes a formatted diagnostic message to stderr unconditionally. Use it +// for low-volume, per-run diagnostics that must always be recorded regardless of +// -verbose (for example the clock-offset summary that documents how a +// server-measured latency was corrected), so they survive in a sweep's collected +// per-run logs. Like Logf it writes only to stderr, never stdout, which sweep +// does not drain. +func Printf(format string, args ...any) { + fmt.Fprintf(os.Stderr, format, args...) +} diff --git a/benchkit/measurement.go b/benchkit/measurement.go new file mode 100644 index 00000000..6efa3231 --- /dev/null +++ b/benchkit/measurement.go @@ -0,0 +1,118 @@ +package benchkit + +import "sync/atomic" + +// Measurement bundles a run's statistics and time-series ticker so every +// benchmark runner wires up observability the same way, honoring opts.StatsMode +// (the aggregate latency store) and opts.Interval (the time-series event +// stream). Record samples through the exported Stats field during the run, then +// call Finish (client-measured: the Result is built from Stats) or Attach +// (server-measured: the Result is built from server replies). +// +// The lifecycle is StartMeasurement -> run workload -> Finish, Attach, or +// Abandon. There is no warmup phase: measurement runs continuously from t=0 +// and the startup transient is trimmed by read-time tools, not here. +type Measurement struct { + Stats *Stats // record samples via Stats.AddLatency / Stats.AddOp + ticker *Ticker // owns the event buffer; drives the interval event stream + initialRate int64 + started bool + stopped bool + failed atomic.Uint64 +} + +// RecordError counts one operation that returned an error. Client-measured +// runners call it instead of aborting so a saturating workload records failed +// operations (surfaced as Result.FailedOps by Finish) rather than terminating +// the run on the first failure. +func (m *Measurement) RecordError() { + m.failed.Add(1) +} + +// newMeasurement creates a dormant measurement whose clock and ticker start +// when [Measurement.start] is called. +func newMeasurement(opts Options) *Measurement { + s := NewStats(opts.StatsMode) + return &Measurement{ + Stats: s, + ticker: NewTicker(opts.Interval, s), + initialRate: int64(opts.startRate()), + } +} + +// StartMeasurement creates the Stats and Ticker for a run from opts, emits the +// START phase marker at the run's starting rate (opts.Rate, or RateStep for a +// ramp with Rate unset), and starts the measurement clock. Call it at t=0, +// immediately before the measurement loop. +func StartMeasurement(opts Options) *Measurement { + m := newMeasurement(opts) + m.start() + return m +} + +// start emits the START marker and starts the measurement clock and ticker. +func (m *Measurement) start() { + if m.started { + return + } + m.started = true + m.ticker.Start(m.initialRate) + m.Stats.Start() +} + +// stop ends the measurement clock and stops the ticker. The whole-run CV that +// Ticker.Stop returns is recomputed at read time over the trimmed intervals, so +// it is not persisted here. +func (m *Measurement) stop() { + if !m.started || m.stopped { + return + } + m.stopped = true + m.Stats.End() + m.ticker.Stop() +} + +// Finish ends the measurement and returns the client-measured Result built from +// the accumulated latency samples, with the time-series events attached. Use it +// for benchmarks where the client times each operation via Stats.AddLatency. +func (m *Measurement) Finish() *Result { + m.stop() + r := m.Stats.GetResult() + r.SetEvents(m.ticker.Events()) + r.SetFailedOps(m.failed.Load()) + setMeasurementMode(r, MeasurementMode_CLIENT_MEASURED) + return r +} + +// Attach ends the measurement and attaches the time-series events to result, a +// Result built elsewhere. Use it for server-measured benchmarks where latency +// comes from server replies rather than the client-side Stats; the client Stats +// then carries only the op count (via Stats.AddOp) for the throughput stream. +func (m *Measurement) Attach(result *Result) { + m.stop() + result.SetEvents(m.ticker.Events()) + setMeasurementMode(result, MeasurementMode_SERVER_MEASURED) +} + +// Abandon stops the measurement clock and ticker without producing a Result. +// Call it from another package when a workload fails at a point that +// precedes where Finish or Attach would normally run, so the ticker's +// background goroutine and its time.Ticker are not leaked. Callers within +// benchkit itself can call the unexported stop directly. +func (m *Measurement) Abandon() { + m.stop() +} + +// setMeasurementMode records how a Result's latency was produced. Finish and +// Attach are the single source of truth for this: Finish builds the result from +// client Stats (client-measured) while Attach takes a server-built result +// (server-measured). The harness Run preserves the mode when it stamps the rest +// of the RunConfig metadata, so consumers never have to infer it. +func setMeasurementMode(r *Result, mode MeasurementMode) { + cfg := r.GetConfig() + if cfg == nil { + cfg = &RunConfig{} + r.SetConfig(cfg) + } + cfg.SetMeasurementMode(mode) +} diff --git a/benchkit/measurement_test.go b/benchkit/measurement_test.go new file mode 100644 index 00000000..dbb4f0f5 --- /dev/null +++ b/benchkit/measurement_test.go @@ -0,0 +1,27 @@ +package benchkit + +import ( + "testing" + "time" +) + +// TestMeasurementAbandonStopsTicker verifies that Abandon stops the ticker's +// background goroutine instead of leaking it. Regression test for the +// pattern where MeasureLatency, ServerMeasured, and the symmetric multicast +// runner all returned early on a workload failure without stopping the +// ticker that StartMeasurement had already started; Abandon (or the +// unexported stop, for callers within benchkit) closes the ticker's done +// channel and waits for its goroutine to exit before returning. +func TestMeasurementAbandonStopsTicker(t *testing.T) { + m := StartMeasurement(Options{Interval: time.Millisecond}) + m.Abandon() + + select { + case _, open := <-m.ticker.done: + if open { + t.Error("ticker.done received a value instead of being closed") + } + default: + t.Error("ticker.done is not closed; Abandon did not stop the ticker goroutine") + } +} diff --git a/benchkit/pacer.go b/benchkit/pacer.go new file mode 100644 index 00000000..3c3cf666 --- /dev/null +++ b/benchkit/pacer.go @@ -0,0 +1,101 @@ +package benchkit + +import ( + "context" + "sync" + "time" +) + +// Pacer paces a single sending goroutine to an open-loop, fixed-rate schedule. +// Sends are scheduled at absolute deadlines start, start+interval, ... so the +// cadence does not drift even if individual sends are briefly delayed. This +// keeps a one-way latency benchmark below saturation, avoiding the queue +// backlog that otherwise dominates the measured latency. +type Pacer struct { + interval time.Duration + next time.Time +} + +// NewPacer returns a pacer that, together with the other workers, sustains a +// combined rate of ratePerNode sends per second. Each of the workers gets an +// equal share of the rate, staggered within one inter-send interval so their +// sends do not all fire at the same instant. NewPacer returns nil when +// ratePerNode <= 0, signalling unlimited (saturating) sends. +func NewPacer(ratePerNode, workers, worker int, start time.Time) *Pacer { + if ratePerNode <= 0 || workers <= 0 { + return nil + } + perWorker := float64(ratePerNode) / float64(workers) + interval := time.Duration(float64(time.Second) / perWorker) + offset := interval * time.Duration(worker) / time.Duration(workers) + return &Pacer{interval: interval, next: start.Add(offset)} +} + +// Wait blocks until this worker's next scheduled send time, then advances the +// schedule. It returns false if ctx is cancelled while waiting. A nil pacer +// never waits, so unlimited senders call Wait without a branch at the call site. +func (p *Pacer) Wait(ctx context.Context) bool { + if p == nil { + return true + } + if d := time.Until(p.next); d > 0 { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-t.C: + case <-ctx.Done(): + return false + } + } + p.next = p.next.Add(p.interval) + return true +} + +// RatedGate is a concurrency-safe, open-loop rate limiter shared by many +// sending goroutines. Unlike Pacer, which is owned by a single worker, one +// RatedGate enforces a combined rate across an arbitrary, dynamically sized set +// of senders. This is used by the async benchmark, where new sends are fired +// from completion callbacks rather than a fixed worker pool, so a per-worker +// pacer cannot pace them. Slots are handed out at absolute deadlines start, +// start+interval, ... so the cadence does not drift. +type RatedGate struct { + interval time.Duration + mu sync.Mutex + next time.Time +} + +// NewRatedGate returns a gate that hands out slots at a combined rate sends per +// second. It returns nil when rate <= 0, signalling unlimited (saturating) +// sends, mirroring NewPacer. +func NewRatedGate(rate int, start time.Time) *RatedGate { + if rate <= 0 { + return nil + } + interval := time.Duration(float64(time.Second) / float64(rate)) + return &RatedGate{interval: interval, next: start} +} + +// Wait claims the next slot in the shared schedule and blocks until its +// deadline, then returns true. It returns false if ctx is cancelled while +// waiting. A nil gate never waits, so unlimited senders call Wait without a +// branch at the call site. The schedule is advanced under the lock, but the +// wait itself happens unlocked so concurrent senders are not serialized. +func (g *RatedGate) Wait(ctx context.Context) bool { + if g == nil { + return true + } + g.mu.Lock() + when := g.next + g.next = g.next.Add(g.interval) + g.mu.Unlock() + if d := time.Until(when); d > 0 { + t := time.NewTimer(d) + defer t.Stop() + select { + case <-t.C: + case <-ctx.Done(): + return false + } + } + return true +} diff --git a/benchkit/pacer_test.go b/benchkit/pacer_test.go new file mode 100644 index 00000000..b3ec9c3a --- /dev/null +++ b/benchkit/pacer_test.go @@ -0,0 +1,151 @@ +package benchkit + +import ( + "context" + "sync" + "testing" + "time" +) + +func TestNewPacerUnlimited(t *testing.T) { + start := time.Now() + if p := NewPacer(0, 4, 0, start); p != nil { + t.Errorf("NewPacer(rate=0) = %v, want nil (unlimited)", p) + } + if p := NewPacer(1000, 0, 0, start); p != nil { + t.Errorf("NewPacer(workers=0) = %v, want nil", p) + } + // A nil pacer must not block and must report success. + var p *Pacer + if !p.Wait(context.Background()) { + t.Error("nil pacer Wait() = false, want true") + } +} + +func TestNewPacerInterval(t *testing.T) { + start := time.Now() + tests := []struct { + name string + rate, workers, worker int + wantInterval time.Duration + wantOffset time.Duration + }{ + {name: "SingleWorker", rate: 1000, workers: 1, worker: 0, wantInterval: time.Millisecond, wantOffset: 0}, + {name: "FourWorkersW0", rate: 1000, workers: 4, worker: 0, wantInterval: 4 * time.Millisecond, wantOffset: 0}, + {name: "FourWorkersW1", rate: 1000, workers: 4, worker: 1, wantInterval: 4 * time.Millisecond, wantOffset: time.Millisecond}, + {name: "FourWorkersW3", rate: 1000, workers: 4, worker: 3, wantInterval: 4 * time.Millisecond, wantOffset: 3 * time.Millisecond}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := NewPacer(tt.rate, tt.workers, tt.worker, start) + if p == nil { + t.Fatal("NewPacer returned nil") + } + if p.interval != tt.wantInterval { + t.Errorf("interval = %v, want %v", p.interval, tt.wantInterval) + } + if got := p.next.Sub(start); got != tt.wantOffset { + t.Errorf("start offset = %v, want %v", got, tt.wantOffset) + } + }) + } +} + +func TestPacerWaitAdvances(t *testing.T) { + start := time.Now() + p := NewPacer(1000, 1, 0, start) // 1ms interval + next := p.next + if !p.Wait(context.Background()) { + t.Fatal("Wait() = false, want true") + } + if got := p.next.Sub(next); got != time.Millisecond { + t.Errorf("schedule advanced by %v, want 1ms", got) + } +} + +func TestPacerWaitCancelled(t *testing.T) { + // Schedule far in the future so Wait blocks, then cancel. + p := NewPacer(1, 1, 0, time.Now().Add(time.Hour)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if p.Wait(ctx) { + t.Error("Wait() on cancelled ctx = true, want false") + } +} + +func TestNewRatedGateUnlimited(t *testing.T) { + if g := NewRatedGate(0, time.Now()); g != nil { + t.Errorf("NewRatedGate(rate=0) = %v, want nil (unlimited)", g) + } + if g := NewRatedGate(-5, time.Now()); g != nil { + t.Errorf("NewRatedGate(rate=-5) = %v, want nil (unlimited)", g) + } + // A nil gate must not block and must report success. + var g *RatedGate + if !g.Wait(context.Background()) { + t.Error("nil RatedGate Wait() = false, want true") + } +} + +func TestNewRatedGateInterval(t *testing.T) { + tests := []struct { + name string + rate int + wantInterval time.Duration + }{ + {name: "1000/s", rate: 1000, wantInterval: time.Millisecond}, + {name: "500/s", rate: 500, wantInterval: 2 * time.Millisecond}, + {name: "1/s", rate: 1, wantInterval: time.Second}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewRatedGate(tt.rate, time.Now()) + if g == nil { + t.Fatal("NewRatedGate returned nil") + } + if g.interval != tt.wantInterval { + t.Errorf("interval = %v, want %v", g.interval, tt.wantInterval) + } + }) + } +} + +func TestRatedGateWaitAdvances(t *testing.T) { + // Past start so Wait does not block; each call must hand out the next slot. + g := NewRatedGate(1000, time.Now().Add(-time.Hour)) // 1ms interval + first := g.next + if !g.Wait(context.Background()) { + t.Fatal("Wait() = false, want true") + } + if got := g.next.Sub(first); got != time.Millisecond { + t.Errorf("schedule advanced by %v, want 1ms", got) + } +} + +func TestRatedGateConcurrentSlots(t *testing.T) { + // Many goroutines sharing one gate must each receive a distinct slot, so the + // schedule advances exactly once per Wait even under contention. + const rate, calls = 100000, 1000 + g := NewRatedGate(rate, time.Now().Add(-time.Hour)) // all slots in the past + start := g.next + var wg sync.WaitGroup + for range calls { + wg.Go(func() { + g.Wait(context.Background()) + }) + } + wg.Wait() + if got := g.next.Sub(start); got != calls*g.interval { + t.Errorf("schedule advanced by %v after %d waits, want %v", got, calls, calls*g.interval) + } +} + +func TestRatedGateWaitCancelled(t *testing.T) { + // Schedule far in the future so Wait blocks, then cancel. + g := NewRatedGate(1, time.Now().Add(time.Hour)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if g.Wait(ctx) { + t.Error("Wait() on cancelled ctx = true, want false") + } +} diff --git a/benchkit/profiling.go b/benchkit/profiling.go new file mode 100644 index 00000000..c48347d3 --- /dev/null +++ b/benchkit/profiling.go @@ -0,0 +1,115 @@ +package benchkit + +import ( + "errors" + "os" + "runtime" + "runtime/pprof" + "runtime/trace" +) + +// StartProfilers starts the profilers selected by non-empty paths — a CPU +// profile, a heap profile, and an execution trace — and returns a stop +// function that finalizes them. A binary built on benchkit wires the standard +// -cpuprofile/-memprofile/-trace flags (see [StandardFlags]) straight into this: +// +// stop, err := benchkit.StartProfilers(f.CPUProfile, f.MemProfile, f.Trace) +// ... +// defer stop() +// +// The CPU profile and trace run from this call until stop; the heap profile is +// written once at stop time, after a GC, so it reflects live allocations at the +// end of the run. +func StartProfilers(cpuProfilePath, memProfilePath, tracePath string) (stop func() error, err error) { + nilFunc := func() error { return nil } + + var ( + cpuProfileStop = nilFunc + traceStop = nilFunc + ) + + if cpuProfilePath != "" { + cpuProfileStop, err = startCPUProfile(cpuProfilePath) + if err != nil { + return nil, err + } + } + + if tracePath != "" { + traceStop, err = startTrace(tracePath) + if err != nil { + // The CPU profiler, if started above, is still running with its + // file open; stop it now so this setup failure doesn't leak it. + return nil, errors.Join(err, cpuProfileStop()) + } + } + + return func() error { + // Run every finalizer even if an earlier one errors, so a failing CPU + // profile stop cannot skip the trace stop or the heap profile. + return errors.Join(cpuProfileStop(), traceStop(), writeMemProfileIfSet(memProfilePath)) + }, nil +} + +// writeMemProfileIfSet writes a heap profile to memProfilePath, or does +// nothing if it is empty. +func writeMemProfileIfSet(memProfilePath string) error { + if memProfilePath == "" { + return nil + } + return writeMemProfile(memProfilePath) +} + +// startCPUProfile starts a CPU profile that will be written to the given path. +// Returns a function to stop the profiler. +func startCPUProfile(cpuProfilePath string) (stop func() error, err error) { + cpuProfile, err := os.Create(cpuProfilePath) + if err != nil { + return nil, err + } + if err := pprof.StartCPUProfile(cpuProfile); err != nil { + // StartCPUProfile failed, so nothing will ever call Close on this + // file; close it here instead of leaking the descriptor. + _ = cpuProfile.Close() + return nil, err + } + return func() error { + pprof.StopCPUProfile() + return cpuProfile.Close() + }, nil +} + +// writeMemProfile writes a heap profile to the given path. +func writeMemProfile(memProfilePath string) error { + f, err := os.Create(memProfilePath) + if err != nil { + return err + } + runtime.GC() // get up-to-date statistics + if err := pprof.WriteHeapProfile(f); err != nil { + // WriteHeapProfile failed, so nothing will ever call Close on this + // file; close it here instead of leaking the descriptor. + _ = f.Close() + return err + } + return f.Close() +} + +// startTrace starts a program trace using the "runtime/trace" package. +// Returns a function to stop the trace. +func startTrace(tracePath string) (stop func() error, err error) { + traceFile, err := os.Create(tracePath) + if err != nil { + return nil, err + } + if err := trace.Start(traceFile); err != nil { + // trace.Start failed, so nothing will ever call Close on this file; + // close it here instead of leaking the descriptor. + _ = traceFile.Close() + return nil, err + } + return func() error { + trace.Stop() + return traceFile.Close() + }, nil +} diff --git a/benchkit/profiling_test.go b/benchkit/profiling_test.go new file mode 100644 index 00000000..9d2236f8 --- /dev/null +++ b/benchkit/profiling_test.go @@ -0,0 +1,98 @@ +package benchkit + +import ( + "os" + "path/filepath" + "testing" +) + +// TestStartProfilers verifies that StartProfilers writes a CPU profile, a +// memory profile, and an execution trace to the given paths, and that the +// returned stop function finalizes all three files. +func TestStartProfilers(t *testing.T) { + dir := t.TempDir() + cpu := filepath.Join(dir, "cpu.prof") + mem := filepath.Join(dir, "mem.prof") + trace := filepath.Join(dir, "trace.out") + + stop, err := StartProfilers(cpu, mem, trace) + if err != nil { + t.Fatalf("StartProfilers: %v", err) + } + if err := stop(); err != nil { + t.Fatalf("stop: %v", err) + } + + for _, path := range []string{cpu, mem, trace} { + info, err := os.Stat(path) + if err != nil { + t.Errorf("missing profile artifact: %v", err) + continue + } + if info.Size() == 0 { + t.Errorf("%s is empty", filepath.Base(path)) + } + } +} + +// TestStartProfilersDisabled verifies that empty paths disable all profilers: +// no files are created and the stop function is a no-op. +func TestStartProfilersDisabled(t *testing.T) { + stop, err := StartProfilers("", "", "") + if err != nil { + t.Fatalf("StartProfilers: %v", err) + } + if err := stop(); err != nil { + t.Fatalf("stop: %v", err) + } +} + +// TestStartProfilersStopsCPUProfileOnTraceSetupFailure verifies that a +// trace-setup failure does not leak the already-started CPU profiler: the +// runtime only allows one active CPU profile at a time, so a leaked profiler +// would make every subsequent [StartProfilers] call fail with "cpu profiling +// already in use". +func TestStartProfilersStopsCPUProfileOnTraceSetupFailure(t *testing.T) { + dir := t.TempDir() + cpu := filepath.Join(dir, "cpu.prof") + // A trace path under a nonexistent directory makes os.Create fail inside + // startTrace, exercising the setup-failure path in StartProfilers. + badTracePath := filepath.Join(dir, "no-such-dir", "trace.out") + + if _, err := StartProfilers(cpu, "", badTracePath); err == nil { + t.Fatal("StartProfilers with bad trace path = nil error, want error") + } + + // If the CPU profiler were left running, this second call would fail + // with "cpu profiling already in use" instead of succeeding. + stop, err := StartProfilers(cpu, "", "") + if err != nil { + t.Fatalf("StartProfilers after prior failure: %v (CPU profiler was not stopped/cleaned up)", err) + } + if err := stop(); err != nil { + t.Fatalf("stop: %v", err) + } +} + +// TestStartCPUProfileReturnsErrorWhenAlreadyRunning verifies that +// startCPUProfile surfaces pprof.StartCPUProfile's error when a profile is +// already running, exercising the path where startCPUProfile must close the +// file it just created instead of leaking it before returning that error. +func TestStartCPUProfileReturnsErrorWhenAlreadyRunning(t *testing.T) { + dir := t.TempDir() + first := filepath.Join(dir, "first.prof") + stopFirst, err := startCPUProfile(first) + if err != nil { + t.Fatalf("startCPUProfile(first): %v", err) + } + defer func() { + if err := stopFirst(); err != nil { + t.Errorf("stopFirst: %v", err) + } + }() + + second := filepath.Join(dir, "second.prof") + if _, err := startCPUProfile(second); err == nil { + t.Fatal("startCPUProfile while a profile is already running = nil error, want error") + } +} diff --git a/benchkit/proto/benchkit/benchkit.proto b/benchkit/proto/benchkit/benchkit.proto new file mode 100644 index 00000000..e6c2d1da --- /dev/null +++ b/benchkit/proto/benchkit/benchkit.proto @@ -0,0 +1,269 @@ +edition = "2024"; + +package benchkit; + +option features.field_presence = IMPLICIT; +option go_package = "github.com/relab/gorums/benchkit"; + +// MeasurementMode records who timed each operation, which decides whether the +// event-stream index map can cut the latency slice at read time. Client-measured +// runs record one in-order sample per op, so the cut is exact; server-measured +// samples arrive out of band and clock-corrected, so they cannot be cut by +// client op counts (see doc/benchkit.html, section 11). +enum MeasurementMode { + CLIENT_MEASURED = 0; // latency from Stats.AddLatency, one per op (default) + SERVER_MEASURED = 1; // latency from server replies, clock-corrected +} + +// StatsMode records the aggregate latency backing store, so consumers know +// whether raw per-op samples (exact) or a bounded-memory histogram (hdr) is +// available without inferring it from the presence of the latencies field. +enum StatsMode { + EXACT = 0; // every sample retained; exact percentiles (default) + HDR = 2; // log-linear histogram; approximate percentiles, bounded memory + reserved 1; // was WELFORD (online mean/stddev only); removed +} + +// RunConfig holds the configuration metadata for one benchmark run. It is kept +// separate from the measured results so producers and consumers can reason +// about how a run was configured independently of what it measured. Durations +// are nanoseconds (compatible with time.Duration). +message RunConfig { + string name = 1; // benchmark name, e.g. "QuorumCall" + int32 num_nodes = 2; // number of nodes in the configuration + string mode = 3; // "local" or "remote" + int64 duration = 4; // configured run duration, nanoseconds + int32 workers = 5; // concurrent worker goroutines + int32 payload = 6; // request/response payload size, bytes + int64 rate = 7; // target ops/s per node; 0 = unlimited (saturating) + int64 interval = 8; // ticker interval, nanoseconds; 0 = events disabled + + MeasurementMode measurement_mode = 9; // client- vs server-measured + StatsMode stats_mode = 10; // exact vs hdr aggregate store + string stream_mode = 11; // "dual" or "dedup" for symmetric stream topology + + int32 quorum_size = 12; // number of replies awaited per quorum call; 0 if not applicable + int32 max_async = 13; // max in-flight async calls; 0 if not applicable + int64 rate_step = 14; // rate increment per ramp step, ops/s; 0 = no ramp + int64 rate_step_max = 15; // maximum target rate during the ramp, ops/s; 0 = no ramp + int64 call_timeout = 16; // per-call deadline, nanoseconds; 0 = disabled + + int32 send_buffer = 17; // per-node send queue capacity + int32 recv_buffer = 18; // server receive queue capacity +} + +// ThroughputInterval is the ops-completed count and elapsed time for one +// ticker interval (both quantities come from Stats.TickInterval). +message ThroughputInterval { + uint64 ops = 1; // operations completed during this interval + int64 duration = 2; // actual elapsed interval, nanoseconds +} + +// LatencyInterval is the Welford accumulator state for one ticker interval. +// Emitted each tick to summarize latency over time; the values are derived +// online and retain no raw samples. +message LatencyInterval { + double mean = 1; // mean latency, nanoseconds + double stddev = 2; // sample standard deviation, nanoseconds + uint64 count = 3; // number of latency samples in this interval +} + +// PhaseMarker announces a lifecycle transition in a run. There is no warmup +// phase: START fires at t=0, STOP at the end, and RATE_STEP on each rate-ramp +// transition. Consumers use phase offsets to trim and annotate. +message PhaseMarker { + enum Phase { + START = 0; // t=0; rate carries the initial target ops/s (default) + RATE_STEP = 1; // rate ramp step; rate carries the new target ops/s + STOP = 2; // run finished + } + Phase phase = 1; + int64 rate = 2; // target ops/s at START and RATE_STEP; 0 = unlimited +} + +// Event is one time-stamped entry in the per-node event stream. offset is +// nanoseconds since the START phase marker (monotonic). Field 15 is reserved +// for a google.protobuf.Any escape hatch carrying protocol-specific events +// without a schema change. +message Event { + int64 offset = 1; // nanoseconds since START (monotonic) + oneof payload { + ThroughputInterval throughput = 2; + LatencyInterval latency = 3; + PhaseMarker phase = 4; + // 15 reserved for google.protobuf.Any extension + } +} + +// LatencyHistogram is the bounded-memory latency distribution recorded in +// StatsMode_HDR, where no raw samples are retained. Entry i records count[i] +// samples indistinguishable from value[i] (nanoseconds) at the histogram's +// resolution; values are ascending. Consumers treat the pairs as a weighted +// sample set — quantiles, mean, and stddev are computed over them without +// knowing the producer's bucket layout. +message LatencyHistogram { + repeated int64 value = 1; // representative latency, nanoseconds; ascending + repeated uint64 count = 2; // samples recorded at value[i] +} + +// MemoryStat contains memory statistics for a single server. +message MemoryStat { + uint64 allocs = 1; // total heap allocations + uint64 memory = 2; // total heap bytes allocated +} + +// Result is one benchmark's complete output for one node: the run +// configuration, the aggregate measured results, and the time-series event +// stream. The startup transient is not removed here; consumers trim it at read +// time using the event offsets (see doc/benchkit.html, section 11). +message Result { + RunConfig config = 1; // how the run was configured + + // Aggregate measured results over the whole run. + uint64 total_ops = 2; // total operations completed + int64 total_time = 3; // elapsed wall time, nanoseconds + double throughput = 4; // operations per second over the whole run + uint64 allocs_per_op = 5; // heap allocations per operation + uint64 mem_per_op = 6; // heap bytes allocated per operation + repeated MemoryStat server_stats = 7; // per-server memory stats + + // Raw per-op latency samples, nanoseconds; nil in hdr mode. Signed: + // clock-offset correction (server-measured benchmarks) can yield negatives. + repeated int64 latencies = 8; + + // Time-series event stream covering the whole run; empty when interval = 0. + repeated Event events = 9; + + // Latency distribution for StatsMode_HDR runs; nil otherwise. + LatencyHistogram histogram = 10; + + // Field 11 held the retired StreamStats message (stream-topology + // statistics); reserved so older result files never misdecode. + reserved 11; + reserved stream_stats; + + // Operations that returned an error and were counted but not aborted on + // (client-measured runs). Under saturation a workload may see failed quorum + // calls; the run completes and records them here instead of exiting. Total + // attempts is total_ops + failed_ops; a high ratio flags an unhealthy run. + uint64 failed_ops = 12; +} + +// Report is the per-node container: one labeled set of benchmark results, +// written once per node and suitable for later comparison via -compare. The +// name is deliberately distinct from Result to avoid a one-character typo +// silently compiling against the wrong type. +message Report { + string label = 1; // run label, e.g. "baseline" or "experiment" + repeated Result results = 2; // one Result per benchmark that ran +} + +// LatencySummary is a latency distribution reduced to summary statistics, +// microseconds. It is a message rather than inline fields on its parent so an +// absent distribution (a run or node that recorded no latency samples) is a +// nil message instead of a spurious all-zero summary, which would be +// indistinguishable from a real measurement. +message LatencySummary { + double mean_us = 1; // mean latency + double p50_us = 2; // 50th percentile latency + double p95_us = 3; // 95th percentile latency + double p99_us = 4; // 99th percentile latency + uint64 samples = 5; // samples the summary was computed from +} + +// The messages below hold a whole sweep reduced for plotting. A sweep's raw +// per-node Result files are far too large to keep or transfer in full, so a +// consumer reduces each run to summary statistics plus a sampled latency CDF. +// Identity is stored once per run and once per node rather than repeated on +// every CDF point, which is what keeps the reduction compact. + +// PlotNode is one node's reduced contribution to one benchmark of one run. +message PlotNode { + string node = 1; // node label, matching Report.label + double throughput = 2; // this node's operations per second + LatencySummary summary = 3; // this node's latency distribution; nil when it recorded no samples + + // Latency CDF, microseconds, ascending. The cumulative probability of point + // i is i/(n-1) for n points, so the probability grid is implied by position + // and is not stored. + repeated double cdf_us = 4; +} + +// PlotBenchmark is one benchmark's reduced results for one run: the run-wide +// aggregate across nodes, plus each node's own reduction. +message PlotBenchmark { + RunConfig config = 1; // benchmark name and the sweep dimensions of this run + + double throughput = 2; // aggregate operations per second across nodes + uint64 total_ops = 3; // total operations completed + uint64 failed_ops = 4; // total operations that returned an error + + // Per-op cost averaged over the reporting nodes. These are doubles, unlike + // their uint64 counterparts in Result, because they are means rather than a + // single node's count. + double allocs_per_op = 5; // mean heap allocations per operation + double mem_per_op = 6; // mean heap bytes allocated per operation + + int32 nodes_seen = 7; // nodes that reported this benchmark + + // summary covers the merged sample set across all nodes, so it is not + // derivable from the per-node summaries; nil when no node recorded samples. + LatencySummary summary = 8; + + repeated PlotNode nodes = 9; // per-node reductions +} + +// PlotRun is one run's identity and its per-benchmark reductions. The identity +// fields mirror the run's manifest, so the reduction is self-describing. +message PlotRun { + string base = 1; // run base name, shared with its manifest + string label = 2; // sweep label prefix + string status = 3; // run outcome; a consumer may exclude degraded runs from aggregates + int32 rep = 4; // repetition number, 1-based + + repeated PlotBenchmark benchmarks = 5; // one entry per benchmark that ran +} + +// PlotData is a whole sweep reduced for plotting: one entry per run that a +// consumer chose to retain. Having a single repeated field means two encoded +// PlotData messages concatenate into a valid message holding both sweeps' runs. +message PlotData { + repeated PlotRun runs = 1; +} + +// The messages below carry a sweep's event streams, the one part of a raw +// Result the reduction above drops. They are stored beside PlotData rather than +// inside it, so a consumer that wants only the summaries reads the small +// message, and a directory reduced before the event streams existed stays +// readable. Identity nests base -> benchmark -> node, mirroring PlotData, so a +// node is named once however many events it recorded. Every run's events are +// retained regardless of outcome: an event stream is a fraction of a percent of +// a raw result file, and a failed run's throughput trace is the most useful of +// all. + +// PlotNodeEvents is one node's event stream for one benchmark of one run. +message PlotNodeEvents { + string node = 1; // node label, matching Report.label + repeated Event events = 2; // this node's stream, in recorded order and untrimmed +} + +// PlotBenchmarkEvents holds every node's event stream for one benchmark of one run. +message PlotBenchmarkEvents { + string benchmark = 1; // benchmark name, matching PlotBenchmark.config.name + repeated PlotNodeEvents nodes = 2; // per-node streams +} + +// PlotRunEvents holds one run's per-benchmark event streams. The run's identity +// beyond its base name (status, trim, dimensions) lives in its manifest, which +// travels alongside, so it is not duplicated here. +message PlotRunEvents { + string base = 1; // run base name, shared with its manifest + + repeated PlotBenchmarkEvents benchmarks = 2; // one entry per benchmark that ran +} + +// PlotEvents is a whole sweep's event streams. As with PlotData, the single +// repeated field means two encoded messages concatenate into a valid one. +message PlotEvents { + repeated PlotRunEvents runs = 1; +} diff --git a/benchkit/proto/benchkit/control.proto b/benchkit/proto/benchkit/control.proto new file mode 100644 index 00000000..3328accd --- /dev/null +++ b/benchkit/proto/benchkit/control.proto @@ -0,0 +1,80 @@ +edition = "2024"; + +package benchkit; + +import "gorums.proto"; +import "benchkit/benchkit.proto"; + +option features.field_presence = IMPLICIT; +option go_package = "github.com/relab/gorums/benchkit"; + +// StartRequest starts a benchmarking campaign and selects the aggregate +// latency backing store the server builds for the run, so a server-measured +// benchmark honors the client's -stats-mode. +message StartRequest { + StatsMode stats_mode = 1; // exact vs hdr aggregate store; 0 = exact (default) +} + +// StartResponse is an empty message to acknowledge the start of a benchmarking campaign. +message StartResponse {} + +// StopRequest is an empty message for stopping a benchmarking campaign. +message StopRequest {} + +// ClockSyncRequest is an empty message for requesting the server's wall clock. +message ClockSyncRequest {} + +// ClockSyncResponse carries the server's wall-clock reading, used for NTP-style +// clock-offset estimation between peers when correcting one-way latencies. +message ClockSyncResponse { + int64 server_time = 1; // nanoseconds; server's wall clock at handler entry +} + +// DoneRequest is multicast to advise peers that the sender has finished its +// own benchmark work and will issue no further calls. +message DoneRequest { + // sender_id is the sender's Gorums node ID, used to de-duplicate signals + // and to name peers that have not yet signaled when diagnosing a timeout. + uint32 sender_id = 1; +} + +// DoneResponse is an empty message; Done is a one-way advisory signal, but a +// response type must be defined to satisfy the gRPC schema. +message DoneResponse {} + +// The Result, Report, and MemoryStat data messages live in benchkit.proto, a +// gorums-free file that consumers (e.g. sweep) can compile standalone without +// pulling in gorums.proto. The Stop RPC below returns benchkit.Result. + +// Control is the protocol-neutral measurement control plane. Any gorums-based +// protocol embeds it alongside its own workload service on the same listener. +// The four RPCs never reference a workload, so they are shared by every +// protocol that benchkit measures. +service Control { + // Start resets the server-side op counter and Stats baseline. + rpc Start(StartRequest) returns (StartResponse) { + option (gorums.quorumcall) = true; + } + + // Stop ends measurement and returns this server's Result. For server-measured + // benchmarks (e.g. Multicast) the Result carries latency samples; for + // client-measured benchmarks (e.g. QuorumCall) it carries only memory stats. + rpc Stop(StopRequest) returns (Result) { + option (gorums.quorumcall) = true; + } + + // ClockSync returns the server's wall-clock time, enabling NTP-style + // clock-offset estimation between peers for one-way latency correction. + rpc ClockSync(ClockSyncRequest) returns (ClockSyncResponse) { + option (gorums.quorumcall) = true; + } + + // Done is an advisory, one-way signal: a node multicasts it to all peers + // once it has finished its own benchmark work and trailing flush. It is + // not a barrier — a missing or delayed Done never blocks or fails a run, + // it only means the waiting peer falls back to its own timeout. See + // benchmark.AwaitPeersDoneOrGrace. + rpc Done(DoneRequest) returns (DoneResponse) { + option (gorums.multicast) = true; + } +} diff --git a/benchkit/proto/benchmark/benchmark.proto b/benchkit/proto/benchmark/benchmark.proto new file mode 100644 index 00000000..ef60dc00 --- /dev/null +++ b/benchkit/proto/benchmark/benchmark.proto @@ -0,0 +1,43 @@ +edition = "2024"; + +package benchmark; +option go_package = "github.com/relab/gorums/benchkit/benchmark"; +option features.field_presence = IMPLICIT; + +import "google/protobuf/empty.proto"; + +import "gorums.proto"; + +// Echo is a simple message used for echo benchmarks. +message Echo { + bytes payload = 1; +} + +// TimedMsg is a message with a send time and a payload used for multicast benchmarks. +message TimedMsg { + int64 send_time = 1; + bytes payload = 2; + uint32 sender_id = 3; // gorums node ID of the sender; lets the receiver bucket + // one-way latency samples per sender for clock-offset + // correction. Zero means untagged (no per-sender bucketing). +} + +// Benchmark is the gorums workload service benchmarked by benchkit. The +// measurement control plane (Start, Stop, ClockSync, Done) lives in benchkit's +// Control service; this service carries only the operations under test. +service Benchmark { + // QuorumCall performs an echo quorum call on all servers. + rpc QuorumCall(Echo) returns (Echo) { + option (gorums.quorumcall) = true; + } + + // SlowServer performs an echo quorum call on slow servers. + rpc SlowServer(Echo) returns (Echo) { + option (gorums.quorumcall) = true; + } + + // Multicast performs a multicast call to all servers. + rpc Multicast(TimedMsg) returns (google.protobuf.Empty) { + option (gorums.multicast) = true; + } +} diff --git a/benchkit/report.go b/benchkit/report.go new file mode 100644 index 00000000..312de52d --- /dev/null +++ b/benchkit/report.go @@ -0,0 +1,199 @@ +package benchkit + +import ( + "fmt" + "io" + "os" + "strings" + "text/tabwriter" + "time" + + "google.golang.org/protobuf/proto" +) + +// binaryMagic is the 8-byte sentinel written at the start of every result file: +// "BKRS" (benchkit results) and "v2", followed by a newline and a NUL byte. Its +// job is identification: proto.Unmarshal accepts almost any bytes without error, +// so the sentinel is what lets [DecodeReport] reject a non-benchkit file +// cleanly. Schema evolution is handled by protobuf rules (add fields with new +// numbers), not by this version: bump "v2" only if the on-disk framing itself +// changes. The full contract is in doc/benchkit.html, section 12. +const binaryMagic = "BKRSv2\n\x00" + +// WriteReport serializes a labeled report to filename as binary proto: an +// 8-byte magic header followed by the binary-encoded Report message (see +// doc/benchkit.html, section 12). LoadReport reads the file back. +func WriteReport(report *Report, filename string) error { + reportBytes, err := proto.Marshal(report) + if err != nil { + return fmt.Errorf("marshal Report: %w", err) + } + buf := make([]byte, 0, len(binaryMagic)+len(reportBytes)) + buf = append(buf, binaryMagic...) + buf = append(buf, reportBytes...) + return os.WriteFile(filename, buf, 0o644) +} + +// LoadReport reads a labeled report from filename written by [WriteReport]. It +// returns an error if the file does not carry the magic header. +func LoadReport(filename string) (*Report, error) { + data, err := os.ReadFile(filename) + if err != nil { + return nil, err + } + report, err := DecodeReport(data) + if err != nil { + return nil, fmt.Errorf("%s: %w", filename, err) + } + return report, nil +} + +// DecodeReport decodes the contents of a result file written by [WriteReport]: +// an 8-byte magic header followed by the binary-encoded Report message. It +// returns an error if the data does not carry the magic header. Use it in place +// of [LoadReport] when the caller reads the file itself, for instance to +// classify read failures of its own (a consumer of a sweep's compact-transfer +// directory expects some result files to be absent). +func DecodeReport(data []byte) (*Report, error) { + if len(data) < len(binaryMagic) || string(data[:len(binaryMagic)]) != binaryMagic { + return nil, fmt.Errorf("not a benchkit binary result file") + } + var report Report + if err := proto.Unmarshal(data[len(binaryMagic):], &report); err != nil { + return nil, fmt.Errorf("unmarshal Report: %w", err) + } + return &report, nil +} + +// WriteLabeledReport wraps results in a labeled [Report] message and writes +// it to filename via [WriteReport]. The label is typically the run's -label +// flag, falling back to a topology-derived identifier (e.g. -self) when unset. +func WriteLabeledReport(results []*Result, label, filename string) error { + report := Report_builder{Label: label, Results: results}.Build() + if err := WriteReport(report, filename); err != nil { + return fmt.Errorf("write report: %w", err) + } + return nil +} + +// CompareWithBaseline loads the report at baselineFile, wraps results in a +// labeled [Report], and writes the side-by-side comparison to w via +// [PrintComparison]. +func CompareWithBaseline(baselineFile, label string, results []*Result, w io.Writer) error { + baseline, err := LoadReport(baselineFile) + if err != nil { + return fmt.Errorf("load comparison file: %w", err) + } + experiment := Report_builder{Label: label, Results: results}.Build() + PrintComparison(baseline, experiment, w) + return nil +} + +// PrintComparison writes a side-by-side latency and throughput comparison +// of two reports matched by benchmark name. No statistical tests are +// performed; percentage change is relative to baseline. A benchmark whose +// baseline and experiment configs differ in a field that changes result +// semantics (e.g. quorum size, rate ramp, stream mode) is still compared — +// rejecting outright would break intentional comparisons like dual vs dedup +// stream mode — but the differing fields are printed as a warning +// (see [ConfigDelta]) so the comparison is never silently misleading. +func PrintComparison(baseline, experiment *Report, w io.Writer) { + fmt.Fprintf(w, "Comparison: %q (baseline) vs %q\n\n", + baseline.GetLabel(), experiment.GetLabel()) + + byName := make(map[string]*Result, len(experiment.GetResults())) + for _, r := range experiment.GetResults() { + byName[r.GetConfig().GetName()] = r + } + + tw := tabwriter.NewWriter(w, 0, 0, 3, ' ', 0) + fmt.Fprintln(tw, "Benchmark\tBaseline latency\tExperiment latency\tΔ latency\tBaseline tput\tExperiment tput\tΔ tput") + for _, b := range baseline.GetResults() { + e, ok := byName[b.GetConfig().GetName()] + if !ok { + continue + } + bMean, bStd := b.LatencyMeanAndStdDev() + eMean, eStd := e.LatencyMeanAndStdDev() + bTput := b.GetThroughput() + eTput := e.GetThroughput() + + latDelta := "" + if bMean > 0 { + pct := (float64(eMean-bMean) / float64(bMean)) * 100 + latDelta = fmt.Sprintf("%+.1f%%", pct) + } + tputDelta := "" + if bTput > 0 { + pct := (eTput - bTput) / bTput * 100 + tputDelta = fmt.Sprintf("%+.1f%%", pct) + } + + fmt.Fprintf(tw, "%s\t%s ± %s\t%s ± %s\t%s\t%.0f ops/s\t%.0f ops/s\t%s\n", + b.GetConfig().GetName(), + formatDuration(bMean), formatDuration(bStd), + formatDuration(eMean), formatDuration(eStd), + latDelta, + bTput, eTput, + tputDelta, + ) + } + tw.Flush() + + for _, b := range baseline.GetResults() { + e, ok := byName[b.GetConfig().GetName()] + if !ok { + continue + } + if delta := ConfigDelta(b.GetConfig(), e.GetConfig()); len(delta) > 0 { + fmt.Fprintf(w, "warning: %s: baseline and experiment configs differ: %s\n", + b.GetConfig().GetName(), strings.Join(delta, ", ")) + } + } +} + +// ConfigDelta returns one "field: baseline vs experiment" string per field in +// a and b that differs and changes what the run measured — everything that +// stamps onto [RunConfig] except the run's own name. An empty result means +// the two configs are semantically comparable. Callers (e.g. +// [PrintComparison]) use this to flag, not reject, comparisons between +// differently configured runs. +func ConfigDelta(a, b *RunConfig) []string { + var delta []string + add := func(field string, av, bv any) { + if av != bv { + delta = append(delta, fmt.Sprintf("%s: %v vs %v", field, av, bv)) + } + } + add("num_nodes", a.GetNumNodes(), b.GetNumNodes()) + add("mode", a.GetMode(), b.GetMode()) + add("duration", time.Duration(a.GetDuration()), time.Duration(b.GetDuration())) + add("workers", a.GetWorkers(), b.GetWorkers()) + add("payload", a.GetPayload(), b.GetPayload()) + add("rate", a.GetRate(), b.GetRate()) + add("interval", time.Duration(a.GetInterval()), time.Duration(b.GetInterval())) + add("measurement_mode", a.GetMeasurementMode(), b.GetMeasurementMode()) + add("stats_mode", a.GetStatsMode(), b.GetStatsMode()) + add("stream_mode", a.GetStreamMode(), b.GetStreamMode()) + add("quorum_size", a.GetQuorumSize(), b.GetQuorumSize()) + add("max_async", a.GetMaxAsync(), b.GetMaxAsync()) + add("rate_step", a.GetRateStep(), b.GetRateStep()) + add("rate_step_max", a.GetRateStepMax(), b.GetRateStepMax()) + add("call_timeout", time.Duration(a.GetCallTimeout()), time.Duration(b.GetCallTimeout())) + add("send_buffer", a.GetSendBuffer(), b.GetSendBuffer()) + add("recv_buffer", a.GetRecvBuffer(), b.GetRecvBuffer()) + return delta +} + +func formatDuration(d time.Duration) string { + switch { + case d < time.Microsecond: + return fmt.Sprintf("%.1f ns", float64(d.Nanoseconds())) + case d < time.Millisecond: + return fmt.Sprintf("%.1f µs", float64(d.Nanoseconds())/1e3) + case d < time.Second: + return fmt.Sprintf("%.1f ms", float64(d.Nanoseconds())/1e6) + default: + return fmt.Sprintf("%.1f s", d.Seconds()) + } +} diff --git a/benchkit/report_test.go b/benchkit/report_test.go new file mode 100644 index 00000000..9316c486 --- /dev/null +++ b/benchkit/report_test.go @@ -0,0 +1,301 @@ +package benchkit + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "google.golang.org/protobuf/proto" +) + +// TestWriteLoadReport verifies binary round-trip fidelity. +func TestWriteLoadReport(t *testing.T) { + want := Report_builder{ + Label: "run-1", + Results: []*Result{ + Result_builder{ + Config: RunConfig_builder{Name: "QuorumCall"}.Build(), + Throughput: 12345.6, + TotalOps: 5000, + Latencies: []int64{100, 200, 300, 400, 500}, + }.Build(), + Result_builder{ + Config: RunConfig_builder{Name: "Multicast"}.Build(), + Throughput: 999.0, + TotalOps: 100, + Latencies: []int64{50, 75}, + }.Build(), + }, + }.Build() + + path := filepath.Join(t.TempDir(), "results.binpb") + if err := WriteReport(want, path); err != nil { + t.Fatalf("WriteReport: %v", err) + } + got, err := LoadReport(path) + if err != nil { + t.Fatalf("LoadReport: %v", err) + } + + if got.GetLabel() != want.GetLabel() { + t.Errorf("label = %q, want %q", got.GetLabel(), want.GetLabel()) + } + if len(got.GetResults()) != len(want.GetResults()) { + t.Fatalf("results count = %d, want %d", len(got.GetResults()), len(want.GetResults())) + } + for i, w := range want.GetResults() { + g := got.GetResults()[i] + if g.GetConfig().GetName() != w.GetConfig().GetName() { + t.Errorf("[%d] name = %q, want %q", i, g.GetConfig().GetName(), w.GetConfig().GetName()) + } + if g.GetThroughput() != w.GetThroughput() { + t.Errorf("[%d] throughput = %v, want %v", i, g.GetThroughput(), w.GetThroughput()) + } + if len(g.GetLatencies()) != len(w.GetLatencies()) { + t.Errorf("[%d] latencies count = %d, want %d", i, len(g.GetLatencies()), len(w.GetLatencies())) + continue + } + for j, lat := range w.GetLatencies() { + if g.GetLatencies()[j] != lat { + t.Errorf("[%d] latencies[%d] = %d, want %d", i, j, g.GetLatencies()[j], lat) + } + } + } +} + +// TestConfigDelta verifies that [ConfigDelta] reports every semantic field +// that differs between two configs and reports nothing when they match, so +// [PrintComparison] can flag an incompatible comparison instead of silently +// treating it as apples-to-apples. +func TestConfigDelta(t *testing.T) { + base := RunConfig_builder{ + Name: "QuorumCall", NumNodes: 4, Mode: "local", Duration: int64(time.Second), + Workers: 2, Payload: 16, Rate: 100, Interval: int64(50 * time.Millisecond), + QuorumSize: 3, MaxAsync: 500, RateStep: 50, RateStepMax: 200, + CallTimeout: int64(20 * time.Millisecond), StatsMode: StatsMode_EXACT, StreamMode: "dual", + }.Build() + + t.Run("IdenticalConfigsHaveNoDelta", func(t *testing.T) { + other := RunConfig_builder{ + Name: "QuorumCall", NumNodes: 4, Mode: "local", Duration: int64(time.Second), + Workers: 2, Payload: 16, Rate: 100, Interval: int64(50 * time.Millisecond), + QuorumSize: 3, MaxAsync: 500, RateStep: 50, RateStepMax: 200, + CallTimeout: int64(20 * time.Millisecond), StatsMode: StatsMode_EXACT, StreamMode: "dual", + }.Build() + if delta := ConfigDelta(base, other); len(delta) != 0 { + t.Errorf("ConfigDelta(identical configs) = %v, want empty", delta) + } + }) + + t.Run("NameDeltaIsIgnored", func(t *testing.T) { + other := RunConfig_builder{ + Name: "Multicast", NumNodes: 4, Mode: "local", Duration: int64(time.Second), + Workers: 2, Payload: 16, Rate: 100, Interval: int64(50 * time.Millisecond), + QuorumSize: 3, MaxAsync: 500, RateStep: 50, RateStepMax: 200, + CallTimeout: int64(20 * time.Millisecond), StatsMode: StatsMode_EXACT, StreamMode: "dual", + }.Build() + if delta := ConfigDelta(base, other); len(delta) != 0 { + t.Errorf("ConfigDelta(only name differs) = %v, want empty (name is the comparison key, not a semantic field)", delta) + } + }) + + tests := []struct { + name string + mutate func(*RunConfig) + wantHit string + }{ + {"QuorumSize", func(c *RunConfig) { c.SetQuorumSize(4) }, "quorum_size"}, + {"MaxAsync", func(c *RunConfig) { c.SetMaxAsync(1000) }, "max_async"}, + {"RateStep", func(c *RunConfig) { c.SetRateStep(100) }, "rate_step"}, + {"RateStepMax", func(c *RunConfig) { c.SetRateStepMax(400) }, "rate_step_max"}, + {"CallTimeout", func(c *RunConfig) { c.SetCallTimeout(int64(time.Second)) }, "call_timeout"}, + {"StreamMode", func(c *RunConfig) { c.SetStreamMode("dedup") }, "stream_mode"}, + {"StatsMode", func(c *RunConfig) { c.SetStatsMode(StatsMode_HDR) }, "stats_mode"}, + {"NumNodes", func(c *RunConfig) { c.SetNumNodes(8) }, "num_nodes"}, + {"SendBuffer", func(c *RunConfig) { c.SetSendBuffer(4096) }, "send_buffer"}, + {"RecvBuffer", func(c *RunConfig) { c.SetRecvBuffer(4096) }, "recv_buffer"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + other := proto.Clone(base).(*RunConfig) + tt.mutate(other) + delta := ConfigDelta(base, other) + found := false + for _, d := range delta { + if strings.HasPrefix(d, tt.wantHit+":") { + found = true + } + } + if !found { + t.Errorf("ConfigDelta after mutating %s = %v, want an entry prefixed %q", tt.name, delta, tt.wantHit+":") + } + }) + } +} + +// TestPrintComparisonFlagsConfigMismatch verifies that PrintComparison warns +// about a semantic config mismatch (e.g. differing quorum size) between +// matched baseline and experiment results, instead of comparing them as if +// they were run under the same settings. +func TestPrintComparisonFlagsConfigMismatch(t *testing.T) { + baseline := Report_builder{ + Label: "baseline", + Results: []*Result{Result_builder{ + Config: RunConfig_builder{Name: "QuorumCall", QuorumSize: 2}.Build(), + Throughput: 100, + }.Build()}, + }.Build() + experiment := Report_builder{ + Label: "experiment", + Results: []*Result{Result_builder{ + Config: RunConfig_builder{Name: "QuorumCall", QuorumSize: 4}.Build(), + Throughput: 120, + }.Build()}, + }.Build() + + var buf bytes.Buffer + PrintComparison(baseline, experiment, &buf) + out := buf.String() + if !strings.Contains(out, "warning") || !strings.Contains(out, "quorum_size") { + t.Errorf("PrintComparison output missing quorum_size mismatch warning; got:\n%s", out) + } +} + +// TestLoadReportRejectsNonBinary verifies that LoadReport returns an error for +// a file that does not carry the binary magic header. +func TestLoadReportRejectsNonBinary(t *testing.T) { + path := filepath.Join(t.TempDir(), "not-binary.json") + if err := os.WriteFile(path, []byte(`{"label":"x","results":[]}`), 0o644); err != nil { + t.Fatal(err) + } + if _, err := LoadReport(path); err == nil { + t.Error("LoadReport(non-binary file) = nil error, want error") + } +} + +// TestDecodeReport verifies that [DecodeReport] reads back what [WriteReport] +// wrote, and rejects data that does not carry the binary magic header, so a +// caller that reads the file itself gets the same guarantees as [LoadReport]. +func TestDecodeReport(t *testing.T) { + path := filepath.Join(t.TempDir(), "results.binpb") + if err := WriteLabeledReport(nil, "baseline", path); err != nil { + t.Fatalf("WriteLabeledReport: %v", err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + + got, err := DecodeReport(data) + if err != nil { + t.Fatalf("DecodeReport: %v", err) + } + if got.GetLabel() != "baseline" { + t.Errorf("label = %q, want %q", got.GetLabel(), "baseline") + } + + for _, tt := range []struct { + name string + data []byte + }{ + {"empty", nil}, + {"short", data[:len(binaryMagic)-1]}, + {"wrong magic", append([]byte("BKRSv9\n\x00"), data[len(binaryMagic):]...)}, + {"payload without magic", data[len(binaryMagic):]}, + } { + t.Run(tt.name, func(t *testing.T) { + if _, err := DecodeReport(tt.data); err == nil { + t.Error("DecodeReport = nil error, want error") + } + }) + } +} + +// TestWriteLabeledReport verifies that [WriteLabeledReport] wraps results in +// a [Report] carrying the given label and that [LoadReport] reads it back +// unchanged. +func TestWriteLabeledReport(t *testing.T) { + results := []*Result{Result_builder{ + Config: RunConfig_builder{Name: "QuorumCall"}.Build(), + Throughput: 42, + }.Build()} + + path := filepath.Join(t.TempDir(), "results.binpb") + if err := WriteLabeledReport(results, "baseline", path); err != nil { + t.Fatalf("WriteLabeledReport: %v", err) + } + got, err := LoadReport(path) + if err != nil { + t.Fatalf("LoadReport: %v", err) + } + if got.GetLabel() != "baseline" { + t.Errorf("label = %q, want %q", got.GetLabel(), "baseline") + } + if len(got.GetResults()) != 1 || got.GetResults()[0].GetConfig().GetName() != "QuorumCall" { + t.Errorf("results = %v, want one QuorumCall result", got.GetResults()) + } +} + +// TestCompareWithBaseline verifies that [CompareWithBaseline] loads the +// baseline file, wraps results under label, and writes the same output +// [PrintComparison] would. +func TestCompareWithBaseline(t *testing.T) { + baseline := Report_builder{ + Label: "baseline", + Results: []*Result{Result_builder{ + Config: RunConfig_builder{Name: "QuorumCall"}.Build(), + Throughput: 100, + }.Build()}, + }.Build() + path := filepath.Join(t.TempDir(), "baseline.binpb") + if err := WriteReport(baseline, path); err != nil { + t.Fatalf("WriteReport: %v", err) + } + + experimentResults := []*Result{Result_builder{ + Config: RunConfig_builder{Name: "QuorumCall"}.Build(), + Throughput: 120, + }.Build()} + + var got bytes.Buffer + if err := CompareWithBaseline(path, "experiment", experimentResults, &got); err != nil { + t.Fatalf("CompareWithBaseline: %v", err) + } + + var want bytes.Buffer + experiment := Report_builder{Label: "experiment", Results: experimentResults}.Build() + PrintComparison(baseline, experiment, &want) + + if got.String() != want.String() { + t.Errorf("CompareWithBaseline output = %q, want %q", got.String(), want.String()) + } + + if err := CompareWithBaseline(filepath.Join(t.TempDir(), "missing.binpb"), "experiment", experimentResults, &got); err == nil { + t.Error("CompareWithBaseline(missing file) = nil error, want error") + } +} + +// TestListBenches verifies that ListBenches renders one aligned line per +// bench naming its description, and nothing for an empty list. +func TestListBenches(t *testing.T) { + var buf bytes.Buffer + ListBenches(&buf, []Bench{ + {Name: "QuorumCall", Description: "quorum call workload"}, + {Name: "Multicast", Description: "multicast workload"}, + }) + out := buf.String() + for _, want := range []string{"QuorumCall:", "quorum call workload", "Multicast:", "multicast workload"} { + if !strings.Contains(out, want) { + t.Errorf("ListBenches output missing %q; got:\n%s", want, out) + } + } + + var empty bytes.Buffer + ListBenches(&empty, nil) + if empty.Len() != 0 { + t.Errorf("ListBenches(nil) wrote %q, want empty output", empty.String()) + } +} diff --git a/benchkit/runconfig.go b/benchkit/runconfig.go new file mode 100644 index 00000000..4fa12796 --- /dev/null +++ b/benchkit/runconfig.go @@ -0,0 +1,68 @@ +package benchkit + +import "cmp" + +// Dimensions identifies one benchmark configuration across the sweep +// pipeline. Zero buffer sizes select the benchmark binary's defaults. +type Dimensions struct { + Benchmark string `json:"benchmark"` + Nodes int `json:"nodes"` + Workers int `json:"workers"` + Payload int `json:"payload"` + Rate int `json:"rate"` + SendBuffer int `json:"send_buffer"` + RecvBuffer int `json:"recv_buffer"` + StreamMode string `json:"stream_mode"` +} + +// Dimensions returns the sweep dimensions recorded in cfg. +func (cfg *RunConfig) Dimensions() Dimensions { + if cfg == nil { + return Dimensions{} + } + return Dimensions{ + Benchmark: cfg.GetName(), + Nodes: int(cfg.GetNumNodes()), + Workers: int(cfg.GetWorkers()), + Payload: int(cfg.GetPayload()), + Rate: int(cfg.GetRate()), + SendBuffer: int(cfg.GetSendBuffer()), + RecvBuffer: int(cfg.GetRecvBuffer()), + StreamMode: cfg.GetStreamMode(), + } +} + +// DimensionsWithFallback returns the dimensions recorded in cfg, filling +// zero-valued fields from fallback for results written before those fields +// were recorded. +func (cfg *RunConfig) DimensionsWithFallback(fallback Dimensions) Dimensions { + dims := cfg.Dimensions() + dims.Benchmark = cmp.Or(dims.Benchmark, fallback.Benchmark) + dims.Nodes = cmp.Or(dims.Nodes, fallback.Nodes) + dims.Workers = cmp.Or(dims.Workers, fallback.Workers) + dims.Payload = cmp.Or(dims.Payload, fallback.Payload) + dims.Rate = cmp.Or(dims.Rate, fallback.Rate) + dims.SendBuffer = cmp.Or(dims.SendBuffer, fallback.SendBuffer) + dims.RecvBuffer = cmp.Or(dims.RecvBuffer, fallback.RecvBuffer) + dims.StreamMode = cmp.Or(dims.StreamMode, fallback.StreamMode) + return dims +} + +// ApplyDimensions records d in cfg. +func (cfg *RunConfig) ApplyDimensions(d Dimensions) { + cfg.SetName(d.Benchmark) + cfg.SetNumNodes(int32(d.Nodes)) + cfg.SetWorkers(int32(d.Workers)) + cfg.SetPayload(int32(d.Payload)) + cfg.SetRate(int64(d.Rate)) + cfg.SetSendBuffer(int32(d.SendBuffer)) + cfg.SetRecvBuffer(int32(d.RecvBuffer)) + cfg.SetStreamMode(d.StreamMode) +} + +// NewRunConfig returns a run configuration containing d. +func NewRunConfig(d Dimensions) *RunConfig { + cfg := RunConfig_builder{}.Build() + cfg.ApplyDimensions(d) + return cfg +} diff --git a/benchkit/runconfig_test.go b/benchkit/runconfig_test.go new file mode 100644 index 00000000..a707f9cb --- /dev/null +++ b/benchkit/runconfig_test.go @@ -0,0 +1,61 @@ +package benchkit + +import "testing" + +func TestRunConfigDimensionsRoundTrip(t *testing.T) { + want := Dimensions{ + Benchmark: "SymmetricQuorumCall", + Nodes: 9, + Workers: 8, + Payload: 1024, + Rate: 5000, + SendBuffer: 256, + RecvBuffer: 16, + StreamMode: "dedup", + } + cfg := NewRunConfig(want) + if got := cfg.Dimensions(); got != want { + t.Fatalf("Dimensions() = %+v, want %+v", got, want) + } + replacement := Dimensions{Benchmark: "Broadcast", Nodes: 3, Workers: 2} + cfg.ApplyDimensions(replacement) + if got := cfg.Dimensions(); got != replacement { + t.Fatalf("Dimensions() after ApplyDimensions = %+v, want %+v", got, replacement) + } + + var nilConfig *RunConfig + if got := nilConfig.Dimensions(); got != (Dimensions{}) { + t.Fatalf("nil Dimensions() = %+v, want zero value", got) + } +} + +func TestRunConfigDimensionsWithFallback(t *testing.T) { + cfg := NewRunConfig(Dimensions{ + Benchmark: "Q", + Nodes: 3, + Rate: 100, + }) + fallback := Dimensions{ + Benchmark: "fallback", + Nodes: 9, + Workers: 8, + Payload: 1024, + Rate: 5000, + SendBuffer: 256, + RecvBuffer: 16, + StreamMode: "dual", + } + want := Dimensions{ + Benchmark: "Q", + Nodes: 3, + Workers: 8, + Payload: 1024, + Rate: 100, + SendBuffer: 256, + RecvBuffer: 16, + StreamMode: "dual", + } + if got := cfg.DimensionsWithFallback(fallback); got != want { + t.Fatalf("DimensionsWithFallback() = %+v, want %+v", got, want) + } +} diff --git a/benchkit/stats.go b/benchkit/stats.go new file mode 100644 index 00000000..cbb71260 --- /dev/null +++ b/benchkit/stats.go @@ -0,0 +1,453 @@ +package benchkit + +import ( + "fmt" + "iter" + "maps" + "math" + "runtime" + "slices" + "strings" + "sync" + "time" +) + +// Row returns the result's display cells in column order: Name, Throughput, +// Latency, Std.dev, p50, p95, p99, B/op, allocs/op. PrintResults (table.go) +// consumes these cells directly, rather than parsing Format's joined string, +// so adding, removing, or reordering columns here cannot silently break table +// rendering. +func (r *Result) Row() []string { + mean, stddev := r.LatencyMeanAndStdDev() + row := []string{ + r.GetConfig().GetName(), + fmt.Sprintf("%.1f ops/sec", r.GetThroughput()), + formatDuration(mean), + formatDuration(stddev), + } + if pcts := r.Percentiles(0.5, 0.95, 0.99); pcts != nil { + row = append(row, formatDuration(pcts[0]), formatDuration(pcts[1]), formatDuration(pcts[2])) + } else { + row = append(row, "n/a", "n/a", "n/a") + } + return append(row, + fmt.Sprintf("%d B/op", r.GetMemPerOp()), + fmt.Sprintf("%d allocs/op", r.GetAllocsPerOp()), + ) +} + +// Format returns a tab formatted string representation of the result (see +// Row for column order). Always emits nine tab-separated columns, each +// followed by a trailing tab, so tabwriter aligns correctly. +func (r *Result) Format() string { + return strings.Join(r.Row(), "\t") + "\t" +} + +// LatencyMeanAndStdDev returns the mean and standard deviation of the recorded +// latencies. In exact mode this is the sample standard deviation over the raw +// samples; StdDev is zero when fewer than two samples have been recorded. In +// HDR mode, where no raw samples are retained, both are the population mean and +// standard deviation computed from the persisted histogram's weighted +// (value, count) pairs, matching [Histogram.Mean] and [Histogram.StdDev]'s +// HdrHistogram-mirroring convention. +func (r *Result) LatencyMeanAndStdDev() (mean, stddev time.Duration) { + m, sd := resultDist(r).MeanAndStdDev() + return time.Duration(m), time.Duration(sd) +} + +// Percentiles returns the requested quantile values as time.Duration. +// Quantiles are in [0, 1]; e.g. Percentiles(0.5, 0.95) yields p50 and p95. +// In HDR mode the quantiles come from the persisted histogram, accurate to +// its significant figures. Returns nil when no samples have been recorded. +func (r *Result) Percentiles(quantiles ...float64) []time.Duration { + qs := resultDist(r).Quantiles(quantiles...) + if qs == nil { + return nil + } + out := make([]time.Duration, len(qs)) + for i, q := range qs { + out[i] = time.Duration(q) + } + return out +} + +// totalCount sums the counts of a LatencyHistogram. +func totalCount(h *LatencyHistogram) uint64 { + var n uint64 + // h.pairs() + for _, c := range h.GetCount() { + n += c + } + return n +} + +// p50 returns the median of h as a time.Duration, via the Result percentile path. +func p50(h *LatencyHistogram) time.Duration { + pcts := Result_builder{Histogram: h}.Build().Percentiles(0.5) + if pcts == nil { + return -1 + } + return pcts[0] +} + +// pairs yields the histogram's stored (value, count) pairs in recorded order, +// stopping if a malformed message carries fewer counts than values. The +// sequence is re-iterable, so weightedMeanStdDev may range over it twice. +func (h *LatencyHistogram) pairs() iter.Seq2[int64, uint64] { + return func(yield func(int64, uint64) bool) { + values, counts := h.GetValue(), h.GetCount() + for i, v := range values { + if i >= len(counts) { + return + } + if !yield(v, counts[i]) { + return + } + } + } +} + +// weightedMeanStdDev returns the mean and population standard deviation over +// the weighted (value, count) pairs, in the values' units; both are zero when +// the pairs are empty. It ranges over pairs twice, so the sequence must be +// re-iterable (Histogram.buckets and LatencyHistogram.pairs both are). +func weightedMeanStdDev(pairs iter.Seq2[int64, uint64]) (mean, stddev float64) { + var total uint64 + var sum float64 + for v, c := range pairs { + total += c + sum += float64(v) * float64(c) + } + if total == 0 { + return 0, 0 + } + mean = sum / float64(total) + var sqSum float64 + for v, c := range pairs { + d := float64(v) - mean + sqSum += d * d * float64(c) + } + return mean, math.Sqrt(sqSum / float64(total)) +} + +// quantileRank returns the 1-based cumulative-count rank on which the +// q-quantile falls over total samples, with q clamped to [0, 1]. The rank is +// at least 1, so any non-empty distribution yields a value. +func quantileRank(total uint64, q float64) uint64 { + return max(uint64(min(max(q, 0), 1)*float64(total)+0.5), 1) +} + +// ServerMemPerOp yields each server's memory (bytes) and allocations per +// operation, in server order. +func (r *Result) ServerMemPerOp() iter.Seq2[uint64, uint64] { + return func(yield func(memPerOp, allocsPerOp uint64) bool) { + for _, memStat := range r.GetServerStats() { + mem, allocs := memStat.perOp(r.GetTotalOps()) + if !yield(mem, allocs) { + return + } + } + } +} + +// perOp returns the memory (bytes) and allocations per operation, or zero when +// totalOps is zero. +func (m *MemoryStat) perOp(totalOps uint64) (memPerOp, allocsPerOp uint64) { + if totalOps == 0 { + return 0, 0 + } + return m.GetMemory() / totalOps, m.GetAllocs() / totalOps +} + +// Stats records the raw data of a benchmark. Each AddLatency call forwards +// the sample to the configured SampleStore and updates the per-interval +// Welford accumulator. All derived statistics (mean, stddev, percentiles) +// are exposed through the *Result returned by GetResult so there is a +// single source of truth. +// +// The default zero value (&Stats{}) uses StatsMode_EXACT (exact sample +// storage). Use NewStats to select a different store mode. +type Stats struct { + mut sync.Mutex + startTime time.Time + endTime time.Time + startMs runtime.MemStats + endMs runtime.MemStats + + mode StatsMode // backing store mode for the aggregate and per-sender stores + store SampleStore // aggregate store; nil → lazily initialized as StatsMode_EXACT + bySender map[uint32]SampleStore // per-sender one-way latency stores keyed by sender node ID + + // Per-interval Welford accumulator used by the Ticker. Always O(1). + iMean float64 + iM2 float64 // sum of squared deviations + iCount uint64 // samples in the current interval + opCount uint64 // total ops recorded since the last Clear + iOpStart uint64 // opCount at the start of the current interval +} + +// NewStats creates a Stats with the given sample storage mode, applied to both +// the aggregate store and the per-sender stores. The zero value &Stats{} uses +// StatsMode_EXACT without calling NewStats. +func NewStats(mode StatsMode) *Stats { + return &Stats{mode: mode, store: newSampleStore(mode)} +} + +// sampleStore returns the aggregate SampleStore, lazily initializing it as +// StatsMode_EXACT if it has not been set. Must be called with s.mut held. +func (s *Stats) sampleStore() SampleStore { + if s.store == nil { + s.store = newSampleStore(StatsMode_EXACT) + } + return s.store +} + +// Start records the start time and memory stats. +func (s *Stats) Start() { + s.mut.Lock() + defer s.mut.Unlock() + + runtime.ReadMemStats(&s.startMs) + s.startTime = time.Now() +} + +// End records the end time and memory stats. +func (s *Stats) End() { + s.mut.Lock() + defer s.mut.Unlock() + + s.endTime = time.Now() + runtime.ReadMemStats(&s.endMs) +} + +// AddLatency records a latency measurement. It forwards the sample to the +// aggregate SampleStore and updates the per-interval Welford accumulator. +func (s *Stats) AddLatency(l time.Duration) { + ns := l.Nanoseconds() + s.mut.Lock() + s.sampleStore().Add(ns) + s.intervalUpdate(ns) + s.mut.Unlock() +} + +// intervalUpdate advances the per-interval Welford accumulator and increments +// the total op counter. Must be called with s.mut held. +func (s *Stats) intervalUpdate(ns int64) { + s.opCount++ + s.iCount++ + delta := float64(ns) - s.iMean + s.iMean += delta / float64(s.iCount) + delta2 := float64(ns) - s.iMean + s.iM2 += delta * delta2 +} + +// AddOp increments the per-interval operation counter without recording a +// latency sample. Use this in server-measured benchmarks where the client +// only sends messages and latency is collected server-side via Stats.AddLatency. +func (s *Stats) AddOp() { + s.mut.Lock() + s.opCount++ + s.mut.Unlock() +} + +// Ops returns the total number of operations recorded since the last Clear, +// regardless of how they were recorded (AddLatency, AddOp, or +// AddLatencyBySender). Server-measured runners use it to derive client-side +// per-op statistics from the client's own send count. +func (s *Stats) Ops() uint64 { + s.mut.Lock() + defer s.mut.Unlock() + return s.opCount +} + +// TickInterval atomically snapshots the per-interval Welford accumulator and +// resets it for the next interval. It returns the mean latency in nanoseconds, +// the sample standard deviation in nanoseconds, the number of latency samples +// in the interval, and the total op delta (total ops recorded via AddOp or +// AddLatency since the last call to TickInterval or Clear). mean, stddev, and +// count are zero if no latency samples were recorded in the interval, but +// opDelta reflects op-only intervals (server-measured runs record ops via +// AddOp without a latency sample). Called by the Ticker goroutine on each +// tick. +func (s *Stats) TickInterval() (mean, stddev float64, count, opDelta uint64) { + s.mut.Lock() + mean = s.iMean + if s.iCount > 1 { + stddev = math.Sqrt(s.iM2 / float64(s.iCount-1)) + } + count = s.iCount + opDelta = s.opCount - s.iOpStart + // Reset interval state for the next tick. + s.iMean = 0 + s.iM2 = 0 + s.iCount = 0 + s.iOpStart = s.opCount + s.mut.Unlock() + return +} + +// AddLatencyBySender records a one-way latency sample keyed by sender node ID. +// Used by symmetric benchmarks where a node receives from multiple senders and +// each sender's samples must later be corrected by that sender's clock offset +// (see [Stats.GetResultCorrected]). It also updates the per-interval Welford +// accumulator so the [Ticker] reflects server-measured latency. +func (s *Stats) AddLatencyBySender(id uint32, l time.Duration) { + ns := l.Nanoseconds() + s.mut.Lock() + if s.bySender == nil { + s.bySender = make(map[uint32]SampleStore) + } + st := s.bySender[id] + if st == nil { + st = newSampleStore(s.mode) + s.bySender[id] = st + } + st.Add(ns) + s.intervalUpdate(ns) + s.mut.Unlock() +} + +// GetResult computes and returns the result of the benchmark from samples +// recorded via AddLatency. In StatsMode_EXACT the full latency distribution is +// available via Result.Latencies; in StatsMode_HDR Latencies is nil and the +// distribution is carried by Result.Histogram instead. +func (s *Stats) GetResult() *Result { + s.mut.Lock() + defer s.mut.Unlock() + + store := s.sampleStore() + r := &Result{} + n := store.Count() + r.SetTotalOps(n) + r.SetTotalTime(int64(s.endTime.Sub(s.startTime))) + if n > 0 { + r.SetThroughput(float64(n) / time.Duration(r.GetTotalTime()).Seconds()) + r.SetAllocsPerOp((s.endMs.Mallocs - s.startMs.Mallocs) / n) + r.SetMemPerOp((s.endMs.TotalAlloc - s.startMs.TotalAlloc) / n) + if samples := store.Samples(); samples != nil { + r.SetLatencies(slices.Clone(samples)) + } else if hs, ok := store.(*hdrStore); ok { + r.SetHistogram(hs.h.snapshot()) + } + } + return r +} + +// GetResultCorrected computes and returns the result from the per-sender +// latency stores recorded via AddLatencyBySender, adding each sender's clock +// offset (peer clock minus this node's clock, in nanoseconds) to every sample +// so that cross-machine clock skew is removed. A sender absent from offsets is +// treated as zero offset (no correction). Senders are visited in sorted node-ID +// order so the result is deterministic. +// +// In StatsMode_HDR the offset is a per-sender additive constant applied to each +// sender's histogram bucket values; the shifted per-sender histograms are then +// re-quantized onto one canonical histogram (Result.Histogram, Latencies nil), +// bounding memory the same way the aggregate HDR store does. +func (s *Stats) GetResultCorrected(offsets map[uint32]int64) *Result { + s.mut.Lock() + defer s.mut.Unlock() + + if s.mode == StatsMode_HDR { + agg := newHDRHistogram() + for _, id := range slices.Sorted(maps.Keys(s.bySender)) { + if hs, ok := s.bySender[id].(*hdrStore); ok { + agg.recordPairs(hs.h.buckets(), offsets[id]) + } + } + return s.resultFromHistogram(agg) + } + + var corrected []int64 + for _, id := range slices.Sorted(maps.Keys(s.bySender)) { + off := offsets[id] + for _, l := range s.bySender[id].Samples() { + corrected = append(corrected, l+off) + } + } + return s.resultFromSamples(corrected) +} + +// resultFromSamples builds a Result from the given samples and the timing and +// memory window recorded by Start and End. The caller must hold s.mut. +func (s *Stats) resultFromSamples(samples []int64) *Result { + r := s.resultWindow(uint64(len(samples))) + if len(samples) > 0 { + r.SetLatencies(samples) + } + return r +} + +// resultFromHistogram builds a Result carrying agg as its latency distribution, +// with the timing and memory window recorded by Start and End. Total ops is the +// histogram's sample count. The caller must hold s.mut. +func (s *Stats) resultFromHistogram(agg *Histogram) *Result { + r := s.resultWindow(agg.TotalCount()) + if agg.TotalCount() > 0 { + r.SetHistogram(agg.snapshot()) + } + return r +} + +// resultWindow builds a Result with the total ops, throughput, and per-op memory +// stats derived from n and the timing and memory window recorded by Start and +// End, leaving the latency distribution for the caller to attach. Per-op stats +// are set only when n > 0. The caller must hold s.mut. +func (s *Stats) resultWindow(n uint64) *Result { + r := &Result{} + r.SetTotalOps(n) + r.SetTotalTime(int64(s.endTime.Sub(s.startTime))) + if n > 0 { + r.SetThroughput(float64(n) / time.Duration(r.GetTotalTime()).Seconds()) + r.SetAllocsPerOp((s.endMs.Mallocs - s.startMs.Mallocs) / n) + r.SetMemPerOp((s.endMs.TotalAlloc - s.startMs.TotalAlloc) / n) + } + return r +} + +// MemDelta returns the raw malloc count and total bytes allocated between +// the last Start and End calls. Used by the Stop handler to compute +// per-op memory stats for benchmarks where the server has no latency +// samples of its own (e.g. QuorumCall, where the client measures latency). +func (s *Stats) MemDelta() (mallocs, totalAlloc uint64) { + s.mut.Lock() + defer s.mut.Unlock() + return s.endMs.Mallocs - s.startMs.Mallocs, + s.endMs.TotalAlloc - s.startMs.TotalAlloc +} + +// Clear zeroes out all stats, including the aggregate store and the +// per-interval Welford accumulator, keeping the current store mode. +func (s *Stats) Clear() { + s.mut.Lock() + s.resetLocked(s.mode) + s.mut.Unlock() +} + +// Reset zeroes out all stats and (re)configures the aggregate and per-sender +// stores to mode, so one Stats can back consecutive runs, including runs that +// select a different StatsMode. It is the mode-aware form of Clear. +func (s *Stats) Reset(mode StatsMode) { + s.mut.Lock() + s.resetLocked(mode) + s.mut.Unlock() +} + +// resetLocked zeroes out all stats, sets the store mode, and rebuilds the +// aggregate store; per-sender stores are dropped and lazily rebuilt in the new +// mode on the next AddLatencyBySender. The caller must hold s.mut. +func (s *Stats) resetLocked(mode StatsMode) { + s.startTime = time.Time{} + s.endTime = time.Time{} + s.startMs = runtime.MemStats{} + s.endMs = runtime.MemStats{} + s.mode = mode + s.store = newSampleStore(mode) + clear(s.bySender) + // Reset per-interval Welford accumulator and op counters. + s.iMean = 0 + s.iM2 = 0 + s.iCount = 0 + s.opCount = 0 + s.iOpStart = 0 +} diff --git a/benchkit/stats_test.go b/benchkit/stats_test.go new file mode 100644 index 00000000..4ebb37c5 --- /dev/null +++ b/benchkit/stats_test.go @@ -0,0 +1,417 @@ +package benchkit + +import ( + "math" + "slices" + "strings" + "testing" + "time" +) + +func TestResultPercentilesAndLatencies(t *testing.T) { + s := &Stats{} + for i := 1; i <= 100; i++ { + s.AddLatency(time.Duration(i) * time.Nanosecond) + } + r := s.GetResult() + + // Hyndman-Fan R7: p50 of 1..100 is 50.5; p95 is 95.05; p99 is 99.01. + // time.Duration truncates to integer nanoseconds, so the expectations + // below are the floor of those values. + got := r.Percentiles(0.5, 0.95, 0.99) + want := []time.Duration{50, 95, 99} + for i, g := range got { + if g != want[i] { + t.Errorf("Percentiles[%d] = %v, want %v", i, g, want[i]) + } + } + + latencies := r.GetLatencies() + if len(latencies) != 100 { + t.Errorf("Latencies length = %d, want 100", len(latencies)) + } + if latencies[0] != 1 || latencies[99] != 100 { + t.Errorf("Latencies[0]=%v, Latencies[99]=%v; want 1 and 100", latencies[0], latencies[99]) + } +} + +// TestStatsOps verifies that Ops counts every recorded operation regardless of +// how it was recorded, and that Clear resets the counter. ServerMeasured uses +// it to derive client-side per-op memory stats from the client's own send +// count rather than the aggregated server op count. +func TestStatsOps(t *testing.T) { + s := &Stats{} + s.AddLatency(time.Nanosecond) + s.AddOp() + s.AddOp() + s.AddLatencyBySender(1, time.Nanosecond) + if got := s.Ops(); got != 4 { + t.Errorf("Ops() = %d, want 4", got) + } + s.Clear() + if got := s.Ops(); got != 0 { + t.Errorf("Ops() after Clear = %d, want 0", got) + } +} + +func TestStatsClearResetsSamples(t *testing.T) { + s := &Stats{} + s.AddLatency(5 * time.Nanosecond) + s.AddLatency(7 * time.Nanosecond) + s.Clear() + r := s.GetResult() + if got := r.GetLatencies(); len(got) != 0 { + t.Errorf("Latencies after Clear = %v, want empty", got) + } + if got := r.Percentiles(0.5); got != nil { + t.Errorf("Percentiles after Clear = %v, want nil", got) + } +} + +func TestStatsGetResultMeanAndStdDev(t *testing.T) { + s := &Stats{} + // Latencies: 10, 20, 30 ns + // Sample mean = 20 ns; sample variance = 200/2 = 100; sample stddev = 10 ns. + s.Start() + for _, v := range []int{10, 20, 30} { + s.AddLatency(time.Duration(v) * time.Nanosecond) + } + s.End() + + r := s.GetResult() + if got := r.GetTotalOps(); got != 3 { + t.Errorf("TotalOps = %d, want 3", got) + } + if gotMean, gotSD := r.LatencyMeanAndStdDev(); gotMean != 20*time.Nanosecond || gotSD != 10*time.Nanosecond { + t.Errorf("LatencyMeanAndStdDev = (%v, %v), want (20ns, 10ns)", gotMean, gotSD) + } + if got := r.GetLatencies(); len(got) != 3 { + t.Errorf("Latencies length = %d, want 3", len(got)) + } +} + +func TestResultFormat(t *testing.T) { + r := Result_builder{ + Config: RunConfig_builder{Name: "TestBench"}.Build(), + Throughput: 1234.56, + Latencies: []int64{time.Millisecond.Nanoseconds(), 2 * time.Millisecond.Nanoseconds()}, + MemPerOp: 42, + AllocsPerOp: 7, + }.Build() + got := r.Format() + + // Format must contain the benchmark name and all stat columns. + for _, want := range []string{"TestBench", "ops/sec", "ms", "B/op", "allocs/op"} { + if !strings.Contains(got, want) { + t.Errorf("Format() missing %q in output: %s", want, got) + } + } + for _, want := range []string{"1234.6 ops/sec", "1.5 ms"} { + if !strings.Contains(got, want) { + t.Errorf("Format() missing one-decimal value %q in output: %s", want, got) + } + } +} + +// TestResultRow verifies that [Result.Row] returns exactly the nine +// documented columns in order, and that [Result.Format] is derived from Row +// (tab-joined with a trailing tab) rather than an independently formatted +// string. +func TestResultRow(t *testing.T) { + r := Result_builder{ + Config: RunConfig_builder{Name: "TestBench"}.Build(), + Throughput: 1234.56, + Latencies: []int64{time.Millisecond.Nanoseconds(), 2 * time.Millisecond.Nanoseconds()}, + MemPerOp: 42, + AllocsPerOp: 7, + }.Build() + + row := r.Row() + if len(row) != 9 { + t.Fatalf("len(Row()) = %d, want 9", len(row)) + } + if row[0] != "TestBench" { + t.Errorf("Row()[0] = %q, want %q", row[0], "TestBench") + } + if row[1] != "1234.6 ops/sec" { + t.Errorf("Row()[1] = %q, want %q", row[1], "1234.6 ops/sec") + } + if row[7] != "42 B/op" || row[8] != "7 allocs/op" { + t.Errorf("Row()[7:9] = %v, want [42 B/op, 7 allocs/op]", row[7:9]) + } + + if want := strings.Join(row, "\t") + "\t"; r.Format() != want { + t.Errorf("Format() = %q, want %q (Row tab-joined with a trailing tab)", r.Format(), want) + } +} + +// TestResultRowNoLatencySamples verifies that Row falls back to "n/a" for the +// percentile columns when no latency samples are recorded, matching Format's +// prior behavior. +func TestResultRowNoLatencySamples(t *testing.T) { + r := Result_builder{Config: RunConfig_builder{Name: "Empty"}.Build()}.Build() + row := r.Row() + for i, want := range []string{"n/a", "n/a", "n/a"} { + if got := row[4+i]; got != want { + t.Errorf("Row()[%d] = %q, want %q", 4+i, got, want) + } + } +} + +func TestResultLatencyMethodsEdgeCases(t *testing.T) { + empty := &Result{} + if gotMean, gotSD := empty.LatencyMeanAndStdDev(); gotMean != 0 || gotSD != 0 { + t.Errorf("LatencyMeanAndStdDev on empty = (%v, %v), want (0, 0)", gotMean, gotSD) + } + + single := Result_builder{Latencies: []int64{42}}.Build() + if gotMean, gotSD := single.LatencyMeanAndStdDev(); gotMean != 42*time.Nanosecond || gotSD != 0 { + t.Errorf("LatencyMeanAndStdDev on single sample = (%v, %v), want (42ns, 0)", gotMean, gotSD) + } +} + +// TestResultPercentilesMalformedHistogram verifies that Percentiles weights +// only the aligned (value, count) pairs when a LatencyHistogram carries more +// counts than values (e.g. from a truncated or corrupt result file), so every +// requested quantile resolves to a recorded value instead of a fabricated +// zero-nanosecond reading. +func TestResultPercentilesMalformedHistogram(t *testing.T) { + r := Result_builder{ + Histogram: LatencyHistogram_builder{ + Value: []int64{10, 20}, + Count: []uint64{1, 2, 3}, // one more count than values + }.Build(), + }.Build() + // The unmatched third count is dropped, leaving total=3 over the pairs + // (10, 1) and (20, 2); p50's rank (2) and p99's rank (3) both land in the + // second pair. + got := r.Percentiles(0.5, 0.99) + want := []time.Duration{20, 20} + if !slices.Equal(got, want) { + t.Errorf("Percentiles(malformed histogram) = %v, want %v", got, want) + } +} + +func TestSymmetricMulticastCorrectedResult(t *testing.T) { + // Two senders: node 1 with offset +100 (its clock is 100ns ahead of ours, + // so its raw samples read 100ns low and need +100), node 2 with offset -50. + // Loopback (node 3) has offset 0 and is left unchanged. + s := &Stats{} + s.Start() + s.AddLatencyBySender(1, 200*time.Nanosecond) + s.AddLatencyBySender(1, 300*time.Nanosecond) + s.AddLatencyBySender(2, 500*time.Nanosecond) + s.AddLatencyBySender(3, 40*time.Nanosecond) + s.End() + + offsets := map[uint32]int64{1: 100, 2: -50, 3: 0} + r := s.GetResultCorrected(offsets) + + // Buckets are visited in sorted node-ID order: 1, 1, 2, 3. + want := []int64{300, 400, 450, 40} + if got := r.GetLatencies(); !slices.Equal(got, want) { + t.Errorf("corrected latencies = %v, want %v", got, want) + } + if got := r.GetTotalOps(); got != 4 { + t.Errorf("TotalOps = %d, want 4", got) + } +} + +func TestSymmetricMulticastCorrectedMissingOffset(t *testing.T) { + // A sender absent from the offsets map is treated as zero offset. + s := &Stats{} + s.Start() + s.AddLatencyBySender(7, 123*time.Nanosecond) + s.End() + + r := s.GetResultCorrected(map[uint32]int64{}) + if got, want := r.GetLatencies(), []int64{123}; !slices.Equal(got, want) { + t.Errorf("corrected latencies = %v, want %v", got, want) + } +} + +// TestSymmetricMulticastCorrectedResultHDR verifies that in StatsMode_HDR the +// per-sender correction shifts each sender's histogram by that sender's clock +// offset and re-quantizes the shifted per-sender histograms onto one bounded +// histogram: Latencies is nil, the count and distribution match the exact path. +func TestSymmetricMulticastCorrectedResultHDR(t *testing.T) { + // Same senders and offsets as the exact test; corrected samples are + // {300, 400} (sender 1), {450} (sender 2), {40} (sender 3, loopback). + s := NewStats(StatsMode_HDR) + s.Start() + s.AddLatencyBySender(1, 200*time.Nanosecond) + s.AddLatencyBySender(1, 300*time.Nanosecond) + s.AddLatencyBySender(2, 500*time.Nanosecond) + s.AddLatencyBySender(3, 40*time.Nanosecond) + s.End() + + r := s.GetResultCorrected(map[uint32]int64{1: 100, 2: -50, 3: 0}) + if got := r.GetLatencies(); got != nil { + t.Errorf("Latencies in HDR mode = %v, want nil", got) + } + if got := r.GetTotalOps(); got != 4 { + t.Errorf("TotalOps = %d, want 4", got) + } + h := r.GetHistogram() + if h == nil { + t.Fatal("Histogram in HDR mode = nil, want non-nil") + } + var total uint64 + for _, c := range h.GetCount() { + total += c + } + if total != 4 { + t.Errorf("histogram counts sum = %d, want 4", total) + } + // The corrected distribution is {40, 300, 400, 450}ns; its mean is 297.5ns, + // reproduced within HDR precision (3 sigfigs resolves these values finely). + if mean, _ := r.LatencyMeanAndStdDev(); math.Abs(float64(mean)-297.5) > 5 { + t.Errorf("LatencyMeanAndStdDev mean = %v, want ≈297.5ns", mean) + } +} + +// TestStatsResetSwitchesMode verifies that Reset reconfigures the aggregate and +// per-sender stores to the requested mode, so one Stats can back consecutive +// runs with different StatsMode values. +func TestStatsResetSwitchesMode(t *testing.T) { + s := NewStats(StatsMode_EXACT) + + s.Reset(StatsMode_HDR) + s.Start() + s.AddLatency(3 * time.Microsecond) + s.AddLatencyBySender(1, 3*time.Microsecond) + s.End() + if got := s.GetResult().GetLatencies(); got != nil { + t.Errorf("aggregate Latencies after Reset(HDR) = %v, want nil", got) + } + if got := s.GetResultCorrected(nil).GetLatencies(); got != nil { + t.Errorf("per-sender Latencies after Reset(HDR) = %v, want nil", got) + } + if s.GetResult().GetHistogram() == nil { + t.Error("aggregate Histogram after Reset(HDR) = nil, want non-nil") + } + + s.Reset(StatsMode_EXACT) + s.Start() + s.AddLatency(7 * time.Microsecond) + s.End() + if got := s.GetResult().GetLatencies(); len(got) != 1 { + t.Errorf("aggregate Latencies after Reset(EXACT) = %v, want one sample", got) + } + if s.GetResult().GetHistogram() != nil { + t.Error("aggregate Histogram after Reset(EXACT) != nil, want nil") + } +} + +func TestStatsClearResetsBySender(t *testing.T) { + s := &Stats{} + s.AddLatencyBySender(1, 5*time.Nanosecond) + s.AddLatencyBySender(2, 7*time.Nanosecond) + s.Clear() + if got := s.GetResultCorrected(map[uint32]int64{1: 1, 2: 1}).GetLatencies(); len(got) != 0 { + t.Errorf("bySender latencies after Clear = %v, want empty", got) + } +} + +func TestStatsHDRModeResult(t *testing.T) { + // HDR mode counts ops and exposes them via TotalOps; raw samples are not + // retained (Latencies is nil) and the distribution is carried by the + // Histogram field instead, from which percentiles and mean/stddev are + // derived within the histogram's precision. + s := NewStats(StatsMode_HDR) + s.Start() + for i := range 10 { + s.AddLatency(time.Duration(i+1) * time.Microsecond) + } + s.End() + r := s.GetResult() + if got := r.GetTotalOps(); got != 10 { + t.Errorf("TotalOps = %d, want 10", got) + } + if got := r.GetLatencies(); got != nil { + t.Errorf("Latencies in HDR mode = %v, want nil", got) + } + if got := r.GetThroughput(); got == 0 { + t.Errorf("Throughput in HDR mode = 0, want non-zero") + } + h := r.GetHistogram() + if h == nil { + t.Fatal("Histogram in HDR mode = nil, want non-nil") + } + var total uint64 + for _, c := range h.GetCount() { + total += c + } + if total != 10 { + t.Errorf("histogram counts sum = %d, want 10", total) + } + // p50 of 1µs..10µs is 5µs; histogram precision is 3 sigfigs. + if pcts := r.Percentiles(0.5); pcts == nil || math.Abs(float64(pcts[0]-5*time.Microsecond)) > 50 { + t.Errorf("Percentiles(0.5) = %v, want ≈5µs", pcts) + } + // Mean of 1µs..10µs is 5.5µs. + if mean, _ := r.LatencyMeanAndStdDev(); math.Abs(float64(mean-5500*time.Nanosecond)) > 50 { + t.Errorf("LatencyMeanAndStdDev mean = %v, want ≈5.5µs", mean) + } +} + +func TestStatsTickInterval(t *testing.T) { + // TickInterval returns Welford stats and op delta for samples added since + // the last tick, then resets for the next interval. + s := &Stats{} + s.AddLatency(100 * time.Nanosecond) + s.AddLatency(200 * time.Nanosecond) + mean, stddev, count, opDelta := s.TickInterval() + + if count != 2 { + t.Errorf("count = %d, want 2", count) + } + if opDelta != 2 { + t.Errorf("opDelta = %d, want 2", opDelta) + } + wantMean := 150.0 + if math.Abs(mean-wantMean) > 0.001 { + t.Errorf("mean = %v, want %v", mean, wantMean) + } + // Sample stddev of [100, 200]: sqrt(((100-150)² + (200-150)²) / 1) = 70.71... + wantSD := math.Sqrt(5000.0) + if math.Abs(stddev-wantSD) > 0.001 { + t.Errorf("stddev = %v, want %v", stddev, wantSD) + } + + // After TickInterval, adding more samples starts a fresh interval. + s.AddLatency(50 * time.Nanosecond) + mean2, _, count2, opDelta2 := s.TickInterval() + if count2 != 1 { + t.Errorf("second tick count = %d, want 1", count2) + } + if opDelta2 != 1 { + t.Errorf("second tick opDelta = %d, want 1", opDelta2) + } + if math.Abs(mean2-50.0) > 0.001 { + t.Errorf("second tick mean = %v, want 50", mean2) + } +} + +func TestStatsTickIntervalEmpty(t *testing.T) { + // TickInterval with no samples in the interval returns all zeros. + s := &Stats{} + mean, stddev, count, opDelta := s.TickInterval() + if mean != 0 || stddev != 0 || count != 0 || opDelta != 0 { + t.Errorf("TickInterval on empty = (%v, %v, %v, %v), want all zeros", + mean, stddev, count, opDelta) + } +} + +func TestStatsClearResetsIntervalState(t *testing.T) { + // After Clear, TickInterval should see no ops from before the clear. + s := &Stats{} + s.AddLatency(100 * time.Nanosecond) + s.AddLatency(200 * time.Nanosecond) + s.Clear() + _, _, count, opDelta := s.TickInterval() + if count != 0 || opDelta != 0 { + t.Errorf("TickInterval after Clear = (count=%d, opDelta=%d), want (0, 0)", + count, opDelta) + } +} diff --git a/benchkit/store.go b/benchkit/store.go new file mode 100644 index 00000000..abaea98f --- /dev/null +++ b/benchkit/store.go @@ -0,0 +1,110 @@ +package benchkit + +// SampleStore accumulates latency samples for one benchmark run. +// Implementations are not thread-safe; callers must serialize access. +// +// The store provides the raw material for the [Result]: the sample count +// and, when retained, the raw samples. All derived statistics (mean, stddev, +// percentiles) are computed by the Result layer — from Samples() in exact +// mode, and from the persisted LatencyHistogram in HDR mode (see +// [Result.Percentiles] and [Summarize]). +type SampleStore interface { + // Add records one latency sample in nanoseconds. + Add(ns int64) + // Count returns the total number of samples recorded. + Count() uint64 + // Samples returns the raw samples in nanoseconds. Returns nil in HDR + // mode, where raw samples are not retained. + Samples() []int64 + // Reset discards all recorded samples, making the store ready for reuse. + Reset() +} + +// newSampleStore returns a SampleStore for the given mode. +func newSampleStore(mode StatsMode) SampleStore { + switch mode { + case StatsMode_HDR: + return newHDRStore() + default: + return &exactStore{} + } +} + +// exactStore retains every latency sample in a slice. +type exactStore struct { + samples []int64 +} + +func (e *exactStore) Add(ns int64) { e.samples = append(e.samples, ns) } +func (e *exactStore) Count() uint64 { return uint64(len(e.samples)) } +func (e *exactStore) Samples() []int64 { return e.samples } +func (e *exactStore) Reset() { e.samples = e.samples[:0] } + +// The StatsMode_HDR histogram layout: nanosecond resolution floor, a one +// minute ceiling (samples above it are clamped; a per-op latency near the +// ceiling would already have hit the benchSlack timeout), and three +// significant figures (~0.1% relative error, ~216 KiB of buckets). +const ( + hdrLowest = 1 + hdrHighest = int64(60_000_000_000) + hdrSigfigs = 3 +) + +// newHDRHistogram returns a Histogram with benchkit's standard HDR layout: the +// canonical bucket set shared by every StatsMode_HDR result, so a merged or +// clock-offset-corrected histogram keeps the same bounded memory and precision +// as one recorded directly. +func newHDRHistogram() *Histogram { + return NewHistogram(hdrLowest, hdrHighest, hdrSigfigs) +} + +// hdrStore backs StatsMode_HDR: it retains no raw samples, only a log-linear +// [Histogram] in constant memory. The histogram is persisted on the Result as +// a LatencyHistogram (see [Stats.GetResult]), from which consumers compute +// approximate percentiles, mean, and stddev. +type hdrStore struct { + h *Histogram +} + +func newHDRStore() *hdrStore { + return &hdrStore{h: newHDRHistogram()} +} + +// Add records ns, clamped into the trackable range: a sample must never be +// dropped, since the op count feeds throughput. +func (s *hdrStore) Add(ns int64) { + _ = s.h.RecordValue(min(max(ns, 0), hdrHighest)) +} + +func (s *hdrStore) Count() uint64 { return s.h.TotalCount() } +func (s *hdrStore) Samples() []int64 { return nil } +func (s *hdrStore) Reset() { s.h.Reset() } + +// offsetHistogram returns src re-quantized onto the canonical HDR layout with +// delta added to every value, for clock-offset correction of a server-measured +// histogram (delta is the negated per-peer offset, mirroring [CorrectLatencies] +// on the raw-sample path). Returns nil when src is nil or empty. +func offsetHistogram(src *LatencyHistogram, delta int64) *LatencyHistogram { + if src == nil { + return nil + } + h := newHDRHistogram() + h.recordPairs(src.pairs(), delta) + return h.snapshot() +} + +// mergeHistograms combines hists onto one canonical HDR histogram, summing their +// (value, count) pairs. It is the HDR counterpart to concatenating raw latency +// slices (see [AggregateServerResults]): re-quantizing onto the shared layout +// keeps the merged histogram at constant bucket count no matter how many +// per-sender or per-server histograms are combined. Returns nil when no input +// carries samples. +func mergeHistograms(hists ...*LatencyHistogram) *LatencyHistogram { + h := newHDRHistogram() + for _, src := range hists { + if src != nil { + h.recordPairs(src.pairs(), 0) + } + } + return h.snapshot() +} diff --git a/benchkit/store_test.go b/benchkit/store_test.go new file mode 100644 index 00000000..6374a3fb --- /dev/null +++ b/benchkit/store_test.go @@ -0,0 +1,69 @@ +package benchkit + +import ( + "math" + "testing" + "time" +) + +// hist builds a LatencyHistogram from the given nanosecond samples on the +// canonical HDR layout, so tests construct histogram inputs the same way a +// server-measured StatsMode_HDR run does. With no samples it returns nil. +func hist(samples ...int64) *LatencyHistogram { + h := newHDRHistogram() + for _, v := range samples { + _ = h.RecordValue(v) + } + return h.snapshot() +} + +// TestOffsetHistogram verifies that offsetHistogram shifts every bucket value by +// the delta (a per-server clock offset), preserves the sample count, clamps a +// shift that would drive values below zero rather than dropping samples, and +// returns nil for a nil or empty input. +func TestOffsetHistogram(t *testing.T) { + if got := offsetHistogram(nil, 100); got != nil { + t.Errorf("offsetHistogram(nil) = %v, want nil", got) + } + if got := offsetHistogram(hist(), 100); got != nil { + t.Errorf("offsetHistogram(empty) = %v, want nil", got) + } + + src := hist(10_000, 10_000, 10_000, 10_000) // 4 samples at 10µs + + shifted := offsetHistogram(src, 5_000) // +5µs + if got := totalCount(shifted); got != 4 { + t.Fatalf("count after shift = %d, want 4", got) + } + if got := p50(shifted); math.Abs(float64(got-15*time.Microsecond)) > 50 { + t.Errorf("p50 after +5µs shift = %v, want ≈15µs", got) + } + + // A negative shift larger than the samples clamps to >= 0 and never drops a + // sample, whose count feeds throughput and the distribution. + clamped := offsetHistogram(src, -20_000) + if got := totalCount(clamped); got != 4 { + t.Errorf("count after clamping shift = %d, want 4", got) + } +} + +// TestMergeHistograms verifies that mergeHistograms sums the sample counts of +// its inputs onto one canonical histogram, ignores nil and empty inputs, and +// returns nil when nothing carries samples. +func TestMergeHistograms(t *testing.T) { + a := hist(1_000, 1_000, 1_000) // 3 samples at 1µs + b := hist(1_000, 2_000) // 1µs, 2µs + + merged := mergeHistograms(a, nil, b, hist()) // nil and empty inputs ignored + if got := totalCount(merged); got != 5 { + t.Fatalf("merged count = %d, want 5", got) + } + // Median of {1,1,1,1,2}µs is 1µs. + if got := p50(merged); math.Abs(float64(got-1*time.Microsecond)) > 50 { + t.Errorf("merged p50 = %v, want ≈1µs", got) + } + + if got := mergeHistograms(nil, hist()); got != nil { + t.Errorf("mergeHistograms(all empty) = %v, want nil", got) + } +} diff --git a/benchkit/summary.go b/benchkit/summary.go new file mode 100644 index 00000000..57d81492 --- /dev/null +++ b/benchkit/summary.go @@ -0,0 +1,114 @@ +package benchkit + +import ( + "time" + + "golang.org/x/exp/stats" +) + +// Summary is one Result reduced over a trimmed read-time window. The benchmark +// binary records the whole run; presentation tools call Summarize to exclude +// the startup transient without re-running anything (see doc/benchkit.html §10). +// The validity flags state which fields carry meaningful data, so callers never +// infer validity from zero values. +type Summary struct { + // Throughput is the ops/s over the kept window: recomputed from the kept + // ThroughputInterval events, or the stored whole-run value when the Result + // carries no events. Always valid, and additive across nodes. + Throughput float64 + + // CV is the coefficient of variation (σ/μ) of the kept per-interval + // throughputs; meaningful iff CVValid (at least two kept intervals). + CV float64 + CVValid bool + + // Latencies holds the kept raw per-op samples in nanoseconds; meaningful + // iff LatencyValid (false for HDR runs, which retain no raw samples). + // The trim cut is applied only for client-measured exact runs, where one + // op yields one in-order sample; server-measured runs keep the whole-run + // samples (their order does not map to the op count). + Latencies []int64 + LatencyValid bool + + // Histogram is the whole-run latency distribution for HDR runs, which + // retain no raw samples; nil otherwise. The trim does not apply to it — + // the histogram has no time dimension — so statistics derived from it + // describe the whole run. + Histogram *LatencyHistogram +} + +// Summarize derives one Result's throughput, throughput coefficient of +// variation, and latency samples over the window that excludes the first trim +// of the run. When the Result carries an interval event stream, throughput is +// recomputed over the intervals at or after trim and the CV is the σ/μ of +// those per-interval throughputs. The latency slice is cut at the sample index +// implied by the dropped intervals' cumulative op counts only for +// client-measured exact runs, where one op yields one in-order sample; +// server-measured samples arrive out of band and clock-corrected, so they keep +// whole-run percentiles. When there are no events, the stored whole-run +// throughput and latencies are used and the CV is invalid. HDR runs carry no +// raw samples; their whole-run distribution passes through as Histogram. +func Summarize(r *Result, trim time.Duration) Summary { + cfg := r.GetConfig() + exact := cfg.GetStatsMode() == StatsMode_EXACT + clientMeasured := cfg.GetMeasurementMode() == MeasurementMode_CLIENT_MEASURED + latencies := r.GetLatencies() + events := r.GetEvents() + if len(events) == 0 { + return Summary{Throughput: r.GetThroughput(), Latencies: latencies, + LatencyValid: exact, Histogram: r.GetHistogram()} + } + + trimNs := trim.Nanoseconds() + var keptOps uint64 + var keptDurNs int64 + var cutOps uint64 // ops in dropped intervals -> latency sample-index cut + var tputs []float64 + for _, ev := range events { + tp := ev.GetThroughput() + if tp == nil { + continue + } + if ev.GetOffset() < trimNs { + cutOps += tp.GetOps() + continue + } + keptOps += tp.GetOps() + keptDurNs += tp.GetDuration() + if tp.GetDuration() > 0 { + tputs = append(tputs, float64(tp.GetOps())/(float64(tp.GetDuration())/1e9)) + } + } + + throughput := r.GetThroughput() + if keptDurNs > 0 { + throughput = float64(keptOps) / (float64(keptDurNs) / 1e9) + } + // The index-map cut is exact only when each op produced one in-order sample. + // Compare in uint64: converting cutOps to int first can wrap negative on a + // 32-bit int, which would pass the bound and then panic on the slice. + if clientMeasured && exact && cutOps > 0 && cutOps <= uint64(len(latencies)) { + latencies = latencies[cutOps:] + } + return Summary{ + Throughput: throughput, + CV: coeffVar(tputs), + CVValid: len(tputs) >= 2, + Latencies: latencies, + LatencyValid: exact, + Histogram: r.GetHistogram(), + } +} + +// coeffVar returns the coefficient of variation (sample stddev / mean) of xs, +// or 0 when fewer than two samples exist or the mean is zero. +func coeffVar(xs []float64) float64 { + if len(xs) < 2 { + return 0 + } + mean, stddev := stats.MeanAndStdDev(xs) + if mean == 0 { + return 0 + } + return stddev / mean +} diff --git a/benchkit/summary_test.go b/benchkit/summary_test.go new file mode 100644 index 00000000..560078f5 --- /dev/null +++ b/benchkit/summary_test.go @@ -0,0 +1,155 @@ +package benchkit + +import ( + "math" + "testing" + "time" +) + +const intervalNs = int64(time.Second) + +func tputEvent(offsetNs int64, ops uint64) *Event { + return Event_builder{ + Offset: offsetNs, + Throughput: ThroughputInterval_builder{Ops: ops, Duration: intervalNs}.Build(), + }.Build() +} + +// TestSummarizeTrim verifies that Summarize trims the startup transient from a +// Result's interval event stream: throughput is recomputed over the kept +// intervals and the CV is the variation of their per-interval throughputs. The +// latency slice is cut at the dropped intervals' cumulative op count only for +// client-measured exact runs; server-measured runs keep whole-run percentiles +// and HDR runs expose no raw samples. +func TestSummarizeTrim(t *testing.T) { + events := []*Event{ + tputEvent(0, 5), // dropped by trim=1s (5 ops -> sample cut when applicable) + tputEvent(1*intervalNs, 10), + tputEvent(2*intervalNs, 20), + tputEvent(3*intervalNs, 30), + } + const wantTput = float64(20) // (10+20+30) ops / 3 s + const wantCV = 0.5 // mean 20, sample stddev 10 + + tests := []struct { + name string + mode MeasurementMode + stats StatsMode + wantLatLen int + wantLatValid bool + }{ + {"ClientExactCuts", MeasurementMode_CLIENT_MEASURED, StatsMode_EXACT, 60, true}, + {"ServerKeepsAll", MeasurementMode_SERVER_MEASURED, StatsMode_EXACT, 65, true}, + {"HDRNoSamples", MeasurementMode_CLIENT_MEASURED, StatsMode_HDR, 65, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := Result_builder{ + Config: RunConfig_builder{ + Name: "Q", + MeasurementMode: tt.mode, + StatsMode: tt.stats, + }.Build(), + Throughput: 999, // stored whole-run value; must be overridden by the trim + Latencies: make([]int64, 65), + Events: events, + }.Build() + + s := Summarize(r, time.Second) + if s.Throughput != wantTput { + t.Errorf("Throughput = %v, want %v", s.Throughput, wantTput) + } + if !s.CVValid || math.Abs(s.CV-wantCV) > 1e-9 { + t.Errorf("CV = %v (valid %v), want %v (valid)", s.CV, s.CVValid, wantCV) + } + if len(s.Latencies) != tt.wantLatLen { + t.Errorf("Latencies len = %d, want %d", len(s.Latencies), tt.wantLatLen) + } + if s.LatencyValid != tt.wantLatValid { + t.Errorf("LatencyValid = %v, want %v", s.LatencyValid, tt.wantLatValid) + } + }) + } +} + +// TestSummarizeNoEvents verifies that a Result without an event stream falls +// back to the stored whole-run throughput and latencies, with the CV invalid. +func TestSummarizeNoEvents(t *testing.T) { + r := Result_builder{ + Config: RunConfig_builder{Name: "Q"}.Build(), + Throughput: 123, + Latencies: []int64{10, 20, 30}, + }.Build() + + s := Summarize(r, time.Second) + if s.Throughput != 123 { + t.Errorf("Throughput = %v, want 123", s.Throughput) + } + if s.CVValid { + t.Error("CVValid = true, want false without events") + } + if !s.LatencyValid || len(s.Latencies) != 3 { + t.Errorf("Latencies = %v (valid %v), want 3 kept samples", s.Latencies, s.LatencyValid) + } +} + +// TestSummarizeSteadyThroughputCV verifies that a perfectly steady run +// (identical per-interval throughput) reports a valid CV of zero rather than +// being marked invalid. +func TestSummarizeSteadyThroughputCV(t *testing.T) { + r := Result_builder{ + Config: RunConfig_builder{Name: "Q"}.Build(), + Events: []*Event{tputEvent(0, 10), tputEvent(1*intervalNs, 10), tputEvent(2*intervalNs, 10)}, + }.Build() + + s := Summarize(r, 0) + if !s.CVValid { + t.Error("CVValid = false, want true for steady throughput") + } + if s.CV != 0 { + t.Errorf("CV = %v, want 0", s.CV) + } +} + +// TestSummarizeHDRHistogram verifies that an HDR Result's whole-run histogram +// passes through Summarize untrimmed (the histogram has no time dimension), +// and that an exact Result carries no histogram. +func TestSummarizeHDRHistogram(t *testing.T) { + hist := LatencyHistogram_builder{ + Value: []int64{100, 200}, + Count: []uint64{5, 15}, + }.Build() + hdr := Result_builder{ + Config: RunConfig_builder{ + Name: "Q", + StatsMode: StatsMode_HDR, + }.Build(), + Histogram: hist, + Events: []*Event{tputEvent(0, 5), tputEvent(1*intervalNs, 15)}, + }.Build() + + s := Summarize(hdr, time.Second) + if s.LatencyValid { + t.Error("LatencyValid = true, want false for HDR") + } + if s.Histogram == nil { + t.Fatal("Histogram = nil, want pass-through") + } + // Weighted p50 over (100×5, 200×15): the 10th of 20 samples is 200. + dist := s.Dist() + if pcts := dist.Quantiles(0.5); pcts == nil || pcts[0] != 200 { + t.Errorf("histogram p50 = %v, want 200ns", pcts) + } + // Weighted mean: (100·5 + 200·15) / 20 = 175. + if mean, _ := dist.MeanAndStdDev(); mean != 175 { + t.Errorf("histogram mean = %v, want 175", mean) + } + + exact := Result_builder{ + Config: RunConfig_builder{Name: "Q"}.Build(), + Latencies: []int64{1, 2, 3}, + }.Build() + if s := Summarize(exact, time.Second); s.Histogram != nil { + t.Errorf("Histogram = %v, want nil for exact run", s.Histogram) + } +} diff --git a/benchkit/table.go b/benchkit/table.go new file mode 100644 index 00000000..8fb31618 --- /dev/null +++ b/benchkit/table.go @@ -0,0 +1,111 @@ +package benchkit + +import ( + "fmt" + "io" + "unicode/utf8" +) + +// PrintResults renders the standard result table for one node's results to w: +// one row per benchmark with the Result.Row columns (throughput, latency +// mean/stddev/percentiles, per-op memory). When nodeLabel is non-empty, it is +// prepended as a Node column and the table is followed by a blank line, so +// interleaved multi-node output stays attributable. In remote runs the +// per-server memory stats are folded into the B/op and allocs/op columns +// unless serverStats is set, which instead appends separate per-server +// columns. +func PrintResults(w io.Writer, results []*Result, opts Options, serverStats bool, nodeLabel string) { + headers := make([]string, 0, 9) + if nodeLabel != "" { + headers = append(headers, "Node") + } + headers = append(headers, "Benchmark", "Throughput", "Latency", "Std.dev", "p50", "p95", "p99") + if !serverStats || !opts.Remote { + headers = append(headers, "B/op", "allocs/op") + } else { + headers = append(headers, "Client B/op", "Client allocs/op") + for i := 1; i <= opts.NumNodes; i++ { + headers = append(headers, fmt.Sprintf("Server %d B/op", i), fmt.Sprintf("Server %d allocs/op", i)) + } + } + + rows := make([][]string, 0, len(results)) + for _, r := range results { + row := make([]string, 0, len(headers)) + if nodeLabel != "" { + row = append(row, nodeLabel) + } + row = append(row, r.Row()...) + if !serverStats && opts.Remote { + // Add each server's per-op memory into the B/op and allocs/op columns + // (Row's last two cells). Update the row strings only, never r: + // callers print before persisting r, so mutating it here would write + // these display-only combined totals to the output file. + memPerOp, allocsPerOp := r.GetMemPerOp(), r.GetAllocsPerOp() + for mem, allocs := range r.ServerMemPerOp() { + memPerOp += mem + allocsPerOp += allocs + } + row[len(row)-2] = fmt.Sprintf("%d B/op", memPerOp) + row[len(row)-1] = fmt.Sprintf("%d allocs/op", allocsPerOp) + } + if serverStats && opts.Remote { + for mem, allocs := range r.ServerMemPerOp() { + row = append(row, + fmt.Sprintf("%d B/op", mem), + fmt.Sprintf("%d allocs/op", allocs), + ) + } + } + rows = append(rows, row) + } + leftAligned := 1 + if nodeLabel != "" { + leftAligned = 2 + } + printTable(w, headers, rows, leftAligned) + if nodeLabel != "" { + fmt.Fprintln(w) + } +} + +// printTable renders headers and rows with columns padded to their widest +// cell; the first leftAligned columns are left-aligned (names, labels) and the +// rest right-aligned (metrics), so decimal points line up. +func printTable(w io.Writer, headers []string, rows [][]string, leftAligned int) { + widths := make([]int, len(headers)) + for i, h := range headers { + widths[i] = utf8.RuneCountInString(h) + } + for _, row := range rows { + for i, cell := range row { + if i >= len(widths) { + break + } + widths[i] = max(widths[i], utf8.RuneCountInString(cell)) + } + } + + printRow := func(row []string) { + for i := range headers { + if i > 0 { + fmt.Fprint(w, " ") + } + cell := "" + if i < len(row) { + cell = row[i] + } + if i < leftAligned { + fmt.Fprintf(w, "%-*s", widths[i], cell) + continue + } + fmt.Fprintf(w, "%*s", widths[i], cell) + } + fmt.Fprintln(w) + } + + printRow(headers) + for _, row := range rows { + printRow(row) + } +} diff --git a/benchkit/table_test.go b/benchkit/table_test.go new file mode 100644 index 00000000..f4875f0f --- /dev/null +++ b/benchkit/table_test.go @@ -0,0 +1,91 @@ +package benchkit + +import ( + "bytes" + "strings" + "testing" +) + +func TestPrintResultsAlignsMetricColumns(t *testing.T) { + results := []*Result{ + Result_builder{ + Config: RunConfig_builder{Name: "SymmetricMulticast"}.Build(), + Throughput: 68972.16, + Latencies: []int64{584300, 584300}, + MemPerOp: 0, + AllocsPerOp: 0, + }.Build(), + Result_builder{ + Config: RunConfig_builder{Name: "SymmetricQuorumCall"}.Build(), + Throughput: 7910.27, + Latencies: []int64{2600000, 2600000}, + MemPerOp: 7353, + AllocsPerOp: 155, + }.Build(), + } + + var buf bytes.Buffer + PrintResults(&buf, results, Options{}, false, "bb3:9000") + got := buf.String() + + for _, want := range []string{"68972.2 ops/sec", "7910.3 ops/sec", "584.3 µs", "2.6 ms"} { + if !strings.Contains(got, want) { + t.Errorf("PrintResults() missing %q in output:\n%s", want, got) + } + } + if !strings.HasSuffix(got, "\n\n") { + t.Errorf("PrintResults() should leave a blank line after node results, got:\n%q", got) + } + + lines := strings.Split(strings.TrimSuffix(got, "\n\n"), "\n") + if len(lines) != 3 { + t.Fatalf("got %d output lines, want 3:\n%s", len(lines), got) + } + latencyStart := strings.Index(lines[0], "Latency") + if latencyStart < 0 { + t.Fatalf("missing Latency header in:\n%s", got) + } + firstDot := strings.Index(lines[1][latencyStart:], ".") + secondDot := strings.Index(lines[2][latencyStart:], ".") + if firstDot < 0 || secondDot < 0 || firstDot != secondDot { + t.Fatalf("latency decimal points not aligned:\n%s\n%s", lines[1], lines[2]) + } +} + +// TestPrintResultsDoesNotMutateResult guards against the rendering bug where +// folding per-server memory into the combined B/op and allocs/op columns +// mutated the *Result that the caller later serializes. +func TestPrintResultsDoesNotMutateResult(t *testing.T) { + r := Result_builder{ + Config: RunConfig_builder{Name: "SymmetricMulticast"}.Build(), + Throughput: 68972.16, + Latencies: []int64{584300, 584300}, + TotalOps: 100, + MemPerOp: 1000, + AllocsPerOp: 10, + ServerStats: []*MemoryStat{ + MemoryStat_builder{Memory: 200_000, Allocs: 2_000}.Build(), + MemoryStat_builder{Memory: 300_000, Allocs: 3_000}.Build(), + }, + }.Build() + + var buf bytes.Buffer + // Remote run without -server-stats triggers the per-server folding path. + PrintResults(&buf, []*Result{r}, Options{Remote: true}, false, "") + got := buf.String() + + // The rendered row must show the combined per-op memory: + // 1000 + 200000/100 + 300000/100 = 6000 B/op; 10 + 2000/100 + 3000/100 = 60 allocs/op. + for _, want := range []string{"6000 B/op", "60 allocs/op"} { + if !strings.Contains(got, want) { + t.Errorf("PrintResults() missing folded value %q in output:\n%s", want, got) + } + } + // The persisted Result must be untouched. + if got, want := r.GetMemPerOp(), uint64(1000); got != want { + t.Errorf("MemPerOp mutated: got %d, want %d", got, want) + } + if got, want := r.GetAllocsPerOp(), uint64(10); got != want { + t.Errorf("AllocsPerOp mutated: got %d, want %d", got, want) + } +} diff --git a/benchkit/ticker.go b/benchkit/ticker.go new file mode 100644 index 00000000..bfe78c4a --- /dev/null +++ b/benchkit/ticker.go @@ -0,0 +1,160 @@ +package benchkit + +import ( + "math" + "sync" + "time" +) + +// Ticker drives the per-interval event stream. On each tick it calls +// Stats.TickInterval to snapshot the per-interval Welford accumulator, emits +// ThroughputInterval and LatencyInterval events to its event buffer, and +// accumulates per-interval throughput samples for a final coefficient-of- +// variation (CV) computation. +// +// The Ticker owns its event buffer: NewTicker allocates one when interval > 0, +// and Events returns the buffered events for attachment to the Result. When the +// configured interval is zero, the buffer is nil, no background goroutine is +// started, and all emission is a no-op; Stop still returns 0 and Events nil. +type Ticker struct { + interval time.Duration + stats *Stats + buffer *eventBuffer + + done chan struct{} + wg sync.WaitGroup + + // Welford accumulators for per-interval throughput (ops/s) samples. + mu sync.Mutex + tpMean float64 + tpM2 float64 + tpCount uint64 +} + +// NewTicker returns a Ticker that samples stats every interval. stats must not +// be nil. When interval > 0 the Ticker allocates an EventBuffer and starts a +// background goroutine on Start; when interval == 0 no events are collected and +// Events returns nil. +func NewTicker(interval time.Duration, stats *Stats) *Ticker { + var buf *eventBuffer + if interval > 0 { + buf = newEventBuffer() + } + return &Ticker{ + interval: interval, + stats: stats, + buffer: buf, + done: make(chan struct{}), + } +} + +// Start emits a START phase marker (carrying the initial target rate) and +// starts the background ticker goroutine if interval > 0. +func (t *Ticker) Start(rate int64) { + t.buffer.emitPhase(time.Now(), PhaseMarker_START, rate) + if t.interval > 0 { + t.wg.Add(1) + go t.run() + } +} + +// RateStep emits a RATE_STEP phase marker with the new target rate. Used by T4 +// (rate ramping) to annotate each step in the event log. +func (t *Ticker) RateStep(rate int64) { + t.buffer.emitPhase(time.Now(), PhaseMarker_RATE_STEP, rate) +} + +// Stop signals the background goroutine to exit, waits for it to finish, emits +// a STOP phase marker, and returns the coefficient of variation +// (stddev/mean) of the per-interval throughput samples. Returns 0 when fewer +// than two ticks occurred or mean throughput is zero. +func (t *Ticker) Stop() float64 { + if t.interval > 0 { + close(t.done) + t.wg.Wait() + } + t.buffer.emitPhase(time.Now(), PhaseMarker_STOP, 0) + return t.cv() +} + +// Events returns the events buffered during the run, in emission order, for +// attachment to the Result via Result.SetEvents. It returns nil when the Ticker +// was created with interval == 0 (event collection disabled). +func (t *Ticker) Events() []*Event { + return t.buffer.Events() +} + +// run is the background ticker goroutine. It fires every t.interval, reads +// the per-interval counters from Stats, and emits the corresponding events. +// On shutdown it flushes the partial interval since the last tick, so the +// summed interval ops match the total recorded ops. +func (t *Ticker) run() { + defer t.wg.Done() + tk := time.NewTicker(t.interval) + defer tk.Stop() + prev := time.Now() + for { + select { + case now := <-tk.C: + dur := now.Sub(prev) + prev = now + mean, stddev, count, opDelta := t.stats.TickInterval() + if dur > 0 { + tp := float64(opDelta) / dur.Seconds() + t.updateCV(tp) + } + t.buffer.emitThroughput(now, opDelta, dur) + if count > 0 { + t.buffer.emitLatency(now, mean, stddev, count) + } + case <-t.done: + t.flushFinal(prev) + return + } + } +} + +// flushFinal emits the partial interval between the last tick and Stop so that +// trailing ops are not lost from the event stream. An empty tail (no ops and no +// samples) emits nothing. Note that this interval can be much shorter than the +// configured tick interval; consumers must use the recorded duration when +// deriving per-interval throughput. It runs in the ticker goroutine before Stop +// emits the STOP marker, so STOP stays the last event. +func (t *Ticker) flushFinal(prev time.Time) { + now := time.Now() + mean, stddev, count, opDelta := t.stats.TickInterval() + if opDelta == 0 && count == 0 { + return + } + dur := now.Sub(prev) + if dur > 0 { + t.updateCV(float64(opDelta) / dur.Seconds()) + } + t.buffer.emitThroughput(now, opDelta, dur) + if count > 0 { + t.buffer.emitLatency(now, mean, stddev, count) + } +} + +// updateCV updates the Welford accumulators with one throughput sample. +func (t *Ticker) updateCV(tp float64) { + t.mu.Lock() + t.tpCount++ + delta := tp - t.tpMean + t.tpMean += delta / float64(t.tpCount) + delta2 := tp - t.tpMean + t.tpM2 += delta * delta2 + t.mu.Unlock() +} + +// cv returns the coefficient of variation of the throughput samples. +// Returns 0 when fewer than two samples exist or mean is zero. +func (t *Ticker) cv() float64 { + t.mu.Lock() + defer t.mu.Unlock() + if t.tpCount < 2 || t.tpMean == 0 { + return 0 + } + stddev := math.Sqrt(t.tpM2 / float64(t.tpCount-1)) + return stddev / t.tpMean +} diff --git a/benchkit/ticker_test.go b/benchkit/ticker_test.go new file mode 100644 index 00000000..fb6c473a --- /dev/null +++ b/benchkit/ticker_test.go @@ -0,0 +1,150 @@ +package benchkit + +import ( + "testing" + "time" +) + +func TestTickerNilBufferSafe(t *testing.T) { + s := NewStats(StatsMode_EXACT) + tk := NewTicker(0, s) // interval=0: no buffer, no goroutine + tk.Start(100) + tk.RateStep(200) + cv := tk.Stop() + if cv != 0 { + t.Errorf("cv = %v, want 0 (no ticks)", cv) + } + if tk.Events() != nil { + t.Error("Events() should be nil when interval == 0") + } +} + +func TestTickerCVComputation(t *testing.T) { + s := NewStats(StatsMode_EXACT) + tk := NewTicker(0, s) // interval=0: no background goroutine + // Inject synthetic throughput samples directly to test CV arithmetic. + tk.updateCV(100) + tk.updateCV(200) + tk.updateCV(150) + cv := tk.cv() + // mean=150, stddev(sample)=50, cv=50/150≈0.333 + const wantCV = 50.0 / 150.0 + const eps = 1e-9 + diff := cv - wantCV + if diff < -eps || diff > eps { + t.Errorf("cv = %v, want %v", cv, wantCV) + } +} + +func TestTickerCVZeroWhenFewTicks(t *testing.T) { + s := NewStats(StatsMode_EXACT) + tk := NewTicker(0, s) + tk.updateCV(100) // only one sample + if cv := tk.cv(); cv != 0 { + t.Errorf("cv = %v, want 0 with single sample", cv) + } +} + +// TestTickerFlushesFinalPartialInterval verifies that ops recorded between the +// last tick and Stop are emitted as a final partial interval rather than lost: +// the summed interval ops must equal the recorded ops, and STOP must remain the +// last event. +func TestTickerFlushesFinalPartialInterval(t *testing.T) { + s := NewStats(StatsMode_EXACT) + tk := NewTicker(time.Hour, s) // no tick fires during the test + tk.Start(0) + for range 5 { + s.AddLatency(time.Millisecond) + } + tk.Stop() + + events := tk.Events() + var ops uint64 + var latencyIntervals int + for _, ev := range events { + if tp := ev.GetThroughput(); tp != nil { + ops += tp.GetOps() + } + if ev.GetLatency() != nil { + latencyIntervals++ + } + } + if ops != 5 { + t.Errorf("summed interval ops = %d, want 5", ops) + } + if latencyIntervals != 1 { + t.Errorf("latency intervals = %d, want 1", latencyIntervals) + } + last := events[len(events)-1] + if ph := last.GetPhase(); ph == nil || ph.GetPhase() != PhaseMarker_STOP { + t.Errorf("last event is not STOP: %v", last) + } +} + +// TestTickerSkipsEmptyFinalInterval verifies that an empty tail (no ops, no +// samples) does not emit a trailing zero interval. +func TestTickerSkipsEmptyFinalInterval(t *testing.T) { + s := NewStats(StatsMode_EXACT) + tk := NewTicker(time.Hour, s) + tk.Start(0) + tk.Stop() + for _, ev := range tk.Events() { + if ev.GetThroughput() != nil || ev.GetLatency() != nil { + t.Errorf("unexpected interval event in empty run: %v", ev) + } + } +} + +func TestTickerWithIntervalEmitsEvents(t *testing.T) { + // Use a short interval to verify the Ticker emits events into its buffer. + s := NewStats(StatsMode_EXACT) + tk := NewTicker(50*time.Millisecond, s) + + tk.Start(0) + // Add some ops so the throughput interval is non-zero. + for range 100 { + s.AddOp() + } + time.Sleep(120 * time.Millisecond) // allow at least two ticks + tk.Stop() + + events := tk.Events() + // Should have at least: START + ≥2 throughput + STOP + if len(events) < 4 { + t.Errorf("len(events) = %d, want ≥4", len(events)) + } + // First event must be the start marker. + first := events[0] + if ph := first.GetPhase(); ph == nil || ph.GetPhase() != PhaseMarker_START { + t.Errorf("events[0] is not the start marker; phase = %v, throughput = %v, latency = %v", + first.GetPhase(), first.GetThroughput(), first.GetLatency()) + } +} + +// TestTickerRateStepConcurrentWithTicksIsRaceFree exercises the rate-ramping +// path (T4): RateStep is called from the caller's goroutine while the +// background ticker goroutine concurrently emits throughput/latency events +// into the same eventBuffer. Run with -race to catch regressions. +func TestTickerRateStepConcurrentWithTicksIsRaceFree(t *testing.T) { + s := NewStats(StatsMode_EXACT) + tk := NewTicker(time.Millisecond, s) + + tk.Start(100) + for step := int64(200); step <= 1000; step += 200 { + s.AddOp() + tk.RateStep(step) + time.Sleep(2 * time.Millisecond) + } + tk.Stop() + + events := tk.Events() + rateSteps := 0 + for _, ev := range events { + if ph := ev.GetPhase(); ph != nil && ph.GetPhase() == PhaseMarker_RATE_STEP { + rateSteps++ + } + } + if rateSteps != 5 { + t.Errorf("rateSteps = %d, want 5", rateSteps) + } +} diff --git a/benchkit/timeseries.go b/benchkit/timeseries.go new file mode 100644 index 00000000..9ed6a43b --- /dev/null +++ b/benchkit/timeseries.go @@ -0,0 +1,346 @@ +package benchkit + +import ( + "encoding/csv" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "time" +) + +// This file renders the time-series event stream (Result.events) to CSV so a +// plotting front end can consume it: WriteTimeSeriesCSVs turns a run's +// per-node event streams into per-benchmark throughput, latency, and +// saturation CSVs. + +// Plotter consumes events from a single benchmark's event stream. Add +// dispatches one event from the named node (the Report label); Render writes +// the accumulated data to w. +type Plotter interface { + Add(node string, e *Event) + Render(w io.Writer) error +} + +// EventReader fans one event stream to every registered Plotter, dropping the +// startup transient (throughput and latency intervals) before trimNs; phase +// markers always pass through so the run's lifecycle structure is preserved. +type EventReader struct { + plotters []Plotter + trimNs int64 +} + +// NewEventReader returns an EventReader that dispatches each event to every +// registered plotter, dropping interval events recorded before trimNs (0 keeps +// the whole run). +func NewEventReader(trimNs int64, plotters ...Plotter) *EventReader { + return &EventReader{plotters: plotters, trimNs: trimNs} +} + +// Read iterates one node's events and dispatches each to every registered +// plotter, tagged with the node identity so multi-node CSV rows stay +// distinguishable. +func (r *EventReader) Read(node string, events []*Event) { + for _, ev := range events { + if r.trimNs > 0 && ev.GetPhase() == nil && ev.GetOffset() < r.trimNs { + continue + } + for _, p := range r.plotters { + p.Add(node, ev) + } + } +} + +// throughputRow is one ThroughputInterval observation with its offset and any +// active phase annotation. +type throughputRow struct { + node string // node identity (Report label) the interval came from + offsetS float64 // seconds since START + ops uint64 // operations in this interval + dur int64 // nanoseconds; actual interval duration + phase string // annotation: "START", "RATE_STEP", etc.; empty otherwise +} + +// ThroughputTimePlotter collects ThroughputInterval and PhaseMarker events and +// renders columns: offset_s, throughput_ops_s, phase, node. The zero value is +// ready to use. +type ThroughputTimePlotter struct { + rows []throughputRow + pendingPhase string +} + +// Add processes one event from node. +func (p *ThroughputTimePlotter) Add(node string, e *Event) { + if ph := e.GetPhase(); ph != nil { + p.pendingPhase = ph.GetPhase().String() + return + } + if tp := e.GetThroughput(); tp != nil { + p.rows = append(p.rows, throughputRow{ + node: node, + offsetS: float64(e.GetOffset()) / 1e9, + ops: tp.GetOps(), + dur: tp.GetDuration(), + phase: p.pendingPhase, + }) + p.pendingPhase = "" + } +} + +// Render writes CSV to w. node and phase come from report labels and event +// data, not fixed enums, so data rows go through encoding/csv: an embedded +// comma, quote, or newline would otherwise corrupt the file. +func (p *ThroughputTimePlotter) Render(w io.Writer) error { + return writeCSV(w, + []string{"offset_s", "throughput_ops_s", "phase", "node"}, + p.rows, func(r throughputRow) []string { + thr := 0.0 + if r.dur > 0 { + thr = float64(r.ops) / (float64(r.dur) / 1e9) + } + return []string{ + strconv.FormatFloat(r.offsetS, 'f', 6, 64), + strconv.FormatFloat(thr, 'f', 3, 64), + r.phase, + r.node, + } + }) +} + +// latencyRow is one LatencyInterval observation. +type latencyRow struct { + node string + offsetS float64 + meanNs float64 + stddevNs float64 + count uint64 + phase string +} + +// LatencyTimePlotter collects LatencyInterval and PhaseMarker events and +// renders columns: offset_s, mean_ns, stddev_ns, count, phase, node. The zero +// value is ready to use. +type LatencyTimePlotter struct { + rows []latencyRow + pendingPhase string +} + +// Add processes one event from node. +func (p *LatencyTimePlotter) Add(node string, e *Event) { + if ph := e.GetPhase(); ph != nil { + p.pendingPhase = ph.GetPhase().String() + return + } + if lat := e.GetLatency(); lat != nil { + p.rows = append(p.rows, latencyRow{ + node: node, + offsetS: float64(e.GetOffset()) / 1e9, + meanNs: lat.GetMean(), + stddevNs: lat.GetStddev(), + count: lat.GetCount(), + phase: p.pendingPhase, + }) + p.pendingPhase = "" + } +} + +// Render writes CSV to w (see [ThroughputTimePlotter.Render]). +func (p *LatencyTimePlotter) Render(w io.Writer) error { + return writeCSV(w, + []string{"offset_s", "mean_ns", "stddev_ns", "count", "phase", "node"}, + p.rows, func(r latencyRow) []string { + return []string{ + strconv.FormatFloat(r.offsetS, 'f', 6, 64), + strconv.FormatFloat(r.meanNs, 'f', 3, 64), + strconv.FormatFloat(r.stddevNs, 'f', 3, 64), + strconv.FormatUint(r.count, 10), + r.phase, + r.node, + } + }) +} + +// rateLevel accumulates throughput and latency samples for one rate-ramp step +// of one node. +type rateLevel struct { + node string + offeredRate int64 + totalOps uint64 + totalDurNs int64 + latencySum float64 + latencyCount uint64 +} + +// SaturationCurvePlotter builds a saturation curve (offered rate vs achieved +// throughput) from PhaseMarker(RATE_STEP) and ThroughputInterval events. A +// single-rate run with no RATE_STEP events is treated as one level. Each node's +// START opens that node's first level, so multi-node input yields one set of +// levels per node. Render columns: offered_rate, throughput_ops_s, +// mean_latency_ns, node. The zero value is ready to use. +type SaturationCurvePlotter struct { + levels []*rateLevel + current *rateLevel + inMeasure bool // true after START +} + +// Add processes one event from node. +func (p *SaturationCurvePlotter) Add(node string, e *Event) { + if ph := e.GetPhase(); ph != nil { + switch ph.GetPhase() { + case PhaseMarker_START: + // Measurement begins at t=0; open the first level at the initial rate. + p.inMeasure = true + p.current = &rateLevel{node: node, offeredRate: ph.GetRate()} + p.levels = append(p.levels, p.current) + case PhaseMarker_RATE_STEP: + // Transition to a new rate level. + p.current = &rateLevel{node: node, offeredRate: ph.GetRate()} + p.levels = append(p.levels, p.current) + } + return + } + if !p.inMeasure || p.current == nil { + return + } + if tp := e.GetThroughput(); tp != nil { + p.current.totalOps += tp.GetOps() + p.current.totalDurNs += tp.GetDuration() + return + } + if lat := e.GetLatency(); lat != nil { + p.current.latencySum += lat.GetMean() * float64(lat.GetCount()) + p.current.latencyCount += lat.GetCount() + } +} + +// Render writes CSV to w (see [ThroughputTimePlotter.Render]). +func (p *SaturationCurvePlotter) Render(w io.Writer) error { + return writeCSV(w, + []string{"offered_rate", "throughput_ops_s", "mean_latency_ns", "node"}, + p.levels, func(lv *rateLevel) []string { + thr := 0.0 + if lv.totalDurNs > 0 { + thr = float64(lv.totalOps) / (float64(lv.totalDurNs) / 1e9) + } + meanLat := 0.0 + if lv.latencyCount > 0 { + meanLat = lv.latencySum / float64(lv.latencyCount) + } + return []string{ + strconv.FormatInt(lv.offeredRate, 10), + strconv.FormatFloat(thr, 'f', 3, 64), + strconv.FormatFloat(meanLat, 'f', 3, 64), + lv.node, + } + }) +} + +// TimeSeriesNode is one node's event stream, labeled by the node it came from +// (the Report label, or whatever identity the caller assigns). +type TimeSeriesNode struct { + Node string + Events []*Event +} + +// TimeSeriesGroup is one benchmark's per-node event streams, the unit the +// time-series renderer draws a figure from. +type TimeSeriesGroup struct { + Benchmark string + Nodes []TimeSeriesNode +} + +// WriteTimeSeriesCSVs renders each group's throughput-over-time, +// latency-over-time, and saturation-curve CSV into outDir as +// _throughput.csv, _latency.csv, and +// _saturation.csv, and returns the benchmarks that had data, in the +// order given, so the caller can plan one figure per name. Multi-node rows stay +// distinguishable via the node column. Interval events before trim are dropped, +// consistent with the read-time trim [Summarize] applies. The output directory +// is created if it does not exist. +// +// A group whose streams hold no throughput or latency interval — a run measured +// with interval reporting off, or one whose every interval fell before trim — is +// skipped entirely: it writes no CSV and is absent from the returned names, so +// no figure is planned for it. Header-only CSVs would instead leave the figure's +// node list empty, which a report cannot render. +func WriteTimeSeriesCSVs(outDir string, groups []TimeSeriesGroup, trim time.Duration) ([]string, error) { + if err := os.MkdirAll(outDir, 0o755); err != nil { + return nil, fmt.Errorf("create output dir: %w", err) + } + var available []string + for _, group := range groups { + tp := &ThroughputTimePlotter{} + lp := &LatencyTimePlotter{} + sc := &SaturationCurvePlotter{} + reader := NewEventReader(trim.Nanoseconds(), tp, lp, sc) + for _, node := range group.Nodes { + reader.Read(node.Node, node.Events) + } + if len(tp.rows) == 0 && len(lp.rows) == 0 { + continue + } + base := csvBaseName(group.Benchmark) + for _, task := range []struct { + plotter Plotter + filename string + }{ + {tp, base + "_throughput.csv"}, + {lp, base + "_latency.csv"}, + {sc, base + "_saturation.csv"}, + } { + if err := renderTimeSeries(task.plotter, filepath.Join(outDir, task.filename)); err != nil { + return nil, err + } + } + available = append(available, group.Benchmark) + } + return available, nil +} + +// csvBaseName reduces a benchmark name to a filename base that stays inside the +// output directory. A benchmark name reaches here from run configuration, so a +// name carrying a path separator would otherwise place the CSV outside outDir or +// in a directory that was never created. Only the final path element is kept, +// and the elements that name a directory rather than a file fall back to a +// fixed stem. The unmodified benchmark name is still what the caller receives +// back, since the report keys its figures by it. +func csvBaseName(benchmark string) string { + base := filepath.Base(filepath.FromSlash(benchmark)) + switch base { + case ".", "..", string(filepath.Separator): + return "benchmark" + } + return base +} + +// renderTimeSeries renders one plotter's output to the file at path. +func renderTimeSeries(p Plotter, path string) error { + f, err := os.Create(path) + if err != nil { + return fmt.Errorf("create %s: %w", path, err) + } + err = p.Render(f) + if cerr := f.Close(); err == nil { + err = cerr + } + if err != nil { + return fmt.Errorf("render %s: %w", filepath.Base(path), err) + } + return nil +} + +// writeCSV writes a header and one record per row to w. +func writeCSV[T any](w io.Writer, header []string, rows []T, fields func(T) []string) error { + cw := csv.NewWriter(w) + if err := cw.Write(header); err != nil { + return err + } + for _, row := range rows { + if err := cw.Write(fields(row)); err != nil { + return err + } + } + cw.Flush() + return cw.Error() +} diff --git a/benchkit/timeseries_test.go b/benchkit/timeseries_test.go new file mode 100644 index 00000000..a0c20c71 --- /dev/null +++ b/benchkit/timeseries_test.go @@ -0,0 +1,290 @@ +package benchkit + +import ( + "bytes" + "encoding/csv" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +// makeTestEvents builds a minimal event stream for testing plotters. +func makeTestEvents(t *testing.T) []*Event { + t.Helper() + b := newEventBuffer() + + start := time.Unix(1, 0) + const sampleInterval = 500 * time.Millisecond + b.emitPhase(start, PhaseMarker_START, 100) + b.emitThroughput(start.Add(500*time.Millisecond), 50, sampleInterval) + b.emitLatency(start.Add(500*time.Millisecond), 200.0, 30.0, 50) + b.emitThroughput(start.Add(1500*time.Millisecond), 100, sampleInterval) + b.emitLatency(start.Add(1500*time.Millisecond), 150.0, 20.0, 100) + b.emitPhase(start.Add(2000*time.Millisecond), PhaseMarker_RATE_STEP, 200) + b.emitThroughput(start.Add(2500*time.Millisecond), 180, sampleInterval) + b.emitPhase(start.Add(3000*time.Millisecond), PhaseMarker_STOP, 0) + + return b.Events() +} + +// render runs one plotter over the test event stream, tagged with node, and +// returns its rendered CSV. +func render(t *testing.T, p Plotter, node string) string { + t.Helper() + NewEventReader(0, p).Read(node, makeTestEvents(t)) + var buf bytes.Buffer + if err := p.Render(&buf); err != nil { + t.Fatalf("Render: %v", err) + } + return buf.String() +} + +// TestPlotterRender verifies that each plotter emits its documented header and +// one row per event it collects, with no leading comment line: the CSVs are +// read by front ends with no comment syntax. +func TestPlotterRender(t *testing.T) { + tests := []struct { + name string + plotter Plotter + header string + wantRows int + }{ + // 3 ThroughputInterval events. + {"throughput", &ThroughputTimePlotter{}, "offset_s,throughput_ops_s,phase,node", 3}, + // 2 LatencyInterval events. + {"latency", &LatencyTimePlotter{}, "offset_s,mean_ns,stddev_ns,count,phase,node", 2}, + // START + 1 RATE_STEP → 2 levels. + {"saturation", &SaturationCurvePlotter{}, "offered_rate,throughput_ops_s,mean_latency_ns,node", 2}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lines := csvLines(render(t, tt.plotter, "bb1:9000")) + if len(lines) == 0 || lines[0] != tt.header { + t.Fatalf("first line = %q, want the header %q", lines, tt.header) + } + if got := len(lines) - 1; got != tt.wantRows { + t.Errorf("rows = %d, want %d", got, tt.wantRows) + } + }) + } +} + +func TestEventReaderFansToAllPlotters(t *testing.T) { + events := makeTestEvents(t) + tp := &ThroughputTimePlotter{} + lp := &LatencyTimePlotter{} + NewEventReader(0, tp, lp).Read("bb1:9000", events) + if len(tp.rows) == 0 { + t.Error("ThroughputTimePlotter received no rows") + } + if len(lp.rows) == 0 { + t.Error("LatencyTimePlotter received no rows") + } +} + +// TestEventReaderTrim verifies that a trim threshold drops interval events +// recorded before the offset while phase markers still pass through. The test +// stream has interval events at 0.5s, 1.5s, and 2.5s; trimming at 1s drops the +// 0.5s pair only. The RATE_STEP marker still reaches the saturation plotter, so +// it keeps both rate levels. +func TestEventReaderTrim(t *testing.T) { + events := makeTestEvents(t) + tp := &ThroughputTimePlotter{} + lp := &LatencyTimePlotter{} + sc := &SaturationCurvePlotter{} + NewEventReader(int64(time.Second), tp, lp, sc).Read("bb1:9000", events) + // 3 throughput intervals total; the one at 0.5s is dropped, leaving 2. + if got := len(tp.rows); got != 2 { + t.Errorf("throughput rows after trim = %d, want 2", got) + } + // 2 latency intervals total; the one at 0.5s is dropped, leaving 1. + if got := len(lp.rows); got != 1 { + t.Errorf("latency rows after trim = %d, want 1", got) + } + // START and RATE_STEP markers pass through, so both rate levels remain. + if got := len(sc.levels); got != 2 { + t.Errorf("saturation levels after trim = %d, want 2", got) + } +} + +// TestPlottersTagRowsWithNode verifies that two nodes' event streams stay +// distinguishable in the rendered CSVs: every row carries its node identity, +// and the saturation plotter keeps one set of rate levels per node instead of +// merging them. +func TestPlottersTagRowsWithNode(t *testing.T) { + events := makeTestEvents(t) + tp := &ThroughputTimePlotter{} + sc := &SaturationCurvePlotter{} + reader := NewEventReader(0, tp, sc) + for _, node := range []string{"bb1:9000", "bb2:9000"} { + reader.Read(node, events) + } + + var buf bytes.Buffer + if err := tp.Render(&buf); err != nil { + t.Fatalf("Render: %v", err) + } + rows := csvLines(buf.String())[1:] // skip header + if got := len(rows); got != 6 { + t.Fatalf("throughput rows = %d, want 6 (3 per node)", got) + } + for i, row := range rows { + wantNode := "bb1:9000" + if i >= 3 { + wantNode = "bb2:9000" + } + if !strings.HasSuffix(row, ","+wantNode) { + t.Errorf("row %d = %q, want node suffix %q", i, row, wantNode) + } + } + + // Each node's START + RATE_STEP opens its own levels: 2 levels per node. + if got := len(sc.levels); got != 4 { + t.Errorf("saturation levels = %d, want 4 (2 per node)", got) + } + for i, lv := range sc.levels { + wantNode := "bb1:9000" + if i >= 2 { + wantNode = "bb2:9000" + } + if lv.node != wantNode { + t.Errorf("level %d node = %q, want %q", i, lv.node, wantNode) + } + } +} + +// TestWriteTimeSeriesCSVs verifies the group-level pipeline: one benchmark's +// per-node event streams are rendered into the three CSVs, each carrying the +// node column, and the benchmark is reported as available. +func TestWriteTimeSeriesCSVs(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "plots") + groups := []TimeSeriesGroup{{ + Benchmark: "QuorumCall", + Nodes: []TimeSeriesNode{{Node: "bb1:9000", Events: makeTestEvents(t)}}, + }} + + available, err := WriteTimeSeriesCSVs(outDir, groups, 0) + if err != nil { + t.Fatalf("WriteTimeSeriesCSVs: %v", err) + } + if len(available) != 1 || available[0] != "QuorumCall" { + t.Errorf("available = %v, want [QuorumCall]", available) + } + for _, name := range []string{ + "QuorumCall_throughput.csv", + "QuorumCall_latency.csv", + "QuorumCall_saturation.csv", + } { + data, err := os.ReadFile(filepath.Join(outDir, name)) + if err != nil { + t.Errorf("missing CSV: %v", err) + continue + } + if !strings.Contains(string(data), ",bb1:9000") { + t.Errorf("%s: no row carries the node identity:\n%s", name, data) + } + } +} + +// TestWriteTimeSeriesCSVsSkipsEmptyGroup verifies that a group whose streams +// hold no interval event writes no CSV and is absent from the returned names, +// so the caller plans no figure for it: a header-only CSV would leave the +// figure's node list empty. +func TestWriteTimeSeriesCSVsSkipsEmptyGroup(t *testing.T) { + outDir := filepath.Join(t.TempDir(), "plots") + groups := []TimeSeriesGroup{ + {Benchmark: "Empty", Nodes: []TimeSeriesNode{{Node: "bb1:9000"}}}, + {Benchmark: "Trimmed", Nodes: []TimeSeriesNode{{Node: "bb1:9000", Events: makeTestEvents(t)}}}, + } + + // A trim past the end of the stream drops every interval of both groups. + available, err := WriteTimeSeriesCSVs(outDir, groups, time.Hour) + if err != nil { + t.Fatalf("WriteTimeSeriesCSVs: %v", err) + } + if available != nil { + t.Errorf("available = %v, want none", available) + } + entries, err := os.ReadDir(outDir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Errorf("wrote %d file(s), want none", len(entries)) + } +} + +// TestPlotterRenderQuotesCommaInNode verifies that a node identity or phase +// containing a comma round-trips through a CSV reader instead of corrupting +// the file: node/phase values come from report labels and event data, which +// are not under this package's control. All three plotters share the same +// encoding/csv-based Render, so ThroughputTimePlotter stands in for the +// others. +func TestPlotterRenderQuotesCommaInNode(t *testing.T) { + b := newEventBuffer() + start := time.Unix(1, 0) + b.emitPhase(start, PhaseMarker_START, 100) + b.emitThroughput(start.Add(500*time.Millisecond), 50, 500*time.Millisecond) + + p := &ThroughputTimePlotter{} + nodeWithComma := "bb1:9000, region=eu" + NewEventReader(0, p).Read(nodeWithComma, b.Events()) + + var buf bytes.Buffer + if err := p.Render(&buf); err != nil { + t.Fatalf("Render: %v", err) + } + + records, err := csv.NewReader(&buf).ReadAll() + if err != nil { + t.Fatalf("csv.ReadAll: %v", err) + } + if len(records) != 2 { // header + one data row + t.Fatalf("got %d CSV records, want 2 (header + 1 row)", len(records)) + } + if got := records[1][3]; got != nodeWithComma { + t.Errorf("node field = %q, want %q (comma must round-trip, not split the row)", got, nodeWithComma) + } +} + +// csvLines returns the non-empty lines of a rendered CSV. +func csvLines(s string) []string { + var out []string + for line := range strings.SplitSeq(s, "\n") { + if line != "" { + out = append(out, line) + } + } + return out +} + +// TestCSVBaseNameStaysInOutputDir verifies that a benchmark name carrying path +// separators cannot steer a CSV out of the output directory. +func TestCSVBaseNameStaysInOutputDir(t *testing.T) { + tests := []struct { + name string + benchmark string + want string + }{ + {"plain", "ReadQC", "ReadQC"}, + {"relative path", "sub/ReadQC", "ReadQC"}, + {"parent escape", "../../etc/passwd", "passwd"}, + {"absolute", "/tmp/ReadQC", "ReadQC"}, + {"dot", ".", "benchmark"}, + {"dotdot", "..", "benchmark"}, + {"separator", "/", "benchmark"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := csvBaseName(tt.benchmark) + if got != tt.want { + t.Errorf("csvBaseName(%q) = %q, want %q", tt.benchmark, got, tt.want) + } + if strings.ContainsRune(got, filepath.Separator) { + t.Errorf("csvBaseName(%q) = %q, still contains a path separator", tt.benchmark, got) + } + }) + } +} diff --git a/go.work b/go.work index 2fec7f39..7bc09ce2 100644 --- a/go.work +++ b/go.work @@ -2,5 +2,6 @@ go 1.26.2 use ( . + ./benchkit ./examples )