From 014f85da947baf3cf61d67a8b7312d7d864f0a1d Mon Sep 17 00:00:00 2001 From: Bill Date: Tue, 17 Feb 2026 11:28:45 -0700 Subject: [PATCH] Step 591: add benchmark comparison harness --- editor/CMakeLists.txt | 9 ++ editor/src/BenchmarkComparisonHarness.h | 109 ++++++++++++++++ editor/tests/step591_test.cpp | 165 ++++++++++++++++++++++++ progress.md | 33 +++++ 4 files changed, 316 insertions(+) create mode 100644 editor/src/BenchmarkComparisonHarness.h create mode 100644 editor/tests/step591_test.cpp diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 4f02e7a..270b5c9 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -4198,4 +4198,13 @@ target_link_libraries(step590_test PRIVATE tree_sitter_javascript tree_sitter_typescript tree_sitter_java tree_sitter_rust tree_sitter_go) +add_executable(step591_test tests/step591_test.cpp) +target_include_directories(step591_test PRIVATE src) +target_link_libraries(step591_test PRIVATE + nlohmann_json::nlohmann_json + unofficial::tree-sitter::tree-sitter + tree_sitter_python tree_sitter_cpp tree_sitter_elisp + tree_sitter_javascript tree_sitter_typescript + tree_sitter_java tree_sitter_rust tree_sitter_go) + # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies) diff --git a/editor/src/BenchmarkComparisonHarness.h b/editor/src/BenchmarkComparisonHarness.h new file mode 100644 index 0000000..133cc6c --- /dev/null +++ b/editor/src/BenchmarkComparisonHarness.h @@ -0,0 +1,109 @@ +#pragma once +// Step 591: Benchmark and Comparison Harness + +#include +#include +#include +#include + +struct BenchmarkSample { + std::string runId; + std::string scenario; + int throughput = 0; + int reliability = 0; + int tokenEfficiency = 0; +}; + +struct BenchmarkAggregate { + std::string scenario; + int avgThroughput = 0; + int avgReliability = 0; + int avgTokenEfficiency = 0; + int sampleCount = 0; +}; + +struct BenchmarkComparison { + std::string scenario; + int throughputDelta = 0; + int reliabilityDelta = 0; + int tokenEfficiencyDelta = 0; +}; + +class BenchmarkComparisonHarness { +public: + bool addSample(const BenchmarkSample& sample, std::string* error) { + if (!error) return false; + error->clear(); + if (sample.runId.empty()) return fail(error, "run_id_missing"); + if (sample.scenario.empty()) return fail(error, "scenario_missing"); + if (sample.throughput < 0 || sample.reliability < 0 || sample.tokenEfficiency < 0) { + return fail(error, "metric_invalid"); + } + if (seenRunIds_.count(sample.runId) != 0) return fail(error, "run_duplicate"); + seenRunIds_[sample.runId] = true; + samples_.push_back(sample); + return true; + } + + std::vector aggregates() const { + struct Sum { int t = 0; int r = 0; int e = 0; int n = 0; }; + std::map sums; + for (const auto& s : samples_) { + auto& x = sums[s.scenario]; + x.t += s.throughput; + x.r += s.reliability; + x.e += s.tokenEfficiency; + ++x.n; + } + std::vector out; + for (const auto& kv : sums) { + BenchmarkAggregate a; + a.scenario = kv.first; + a.sampleCount = kv.second.n; + a.avgThroughput = kv.second.t / kv.second.n; + a.avgReliability = kv.second.r / kv.second.n; + a.avgTokenEfficiency = kv.second.e / kv.second.n; + out.push_back(a); + } + return out; + } + + std::vector compareRuns(const std::string& baselineRunId, + const std::string& candidateRunId) const { + const auto* a = findRun(baselineRunId); + const auto* b = findRun(candidateRunId); + if (!a || !b || a->scenario != b->scenario) return {}; + BenchmarkComparison c; + c.scenario = a->scenario; + c.throughputDelta = b->throughput - a->throughput; + c.reliabilityDelta = b->reliability - a->reliability; + c.tokenEfficiencyDelta = b->tokenEfficiency - a->tokenEfficiency; + return {c}; + } + + std::vector topByThroughput(const std::string& scenario, + std::size_t limit) const { + std::vector out; + for (const auto& s : samples_) if (scenario.empty() || s.scenario == scenario) out.push_back(s); + std::stable_sort(out.begin(), out.end(), [](const BenchmarkSample& a, const BenchmarkSample& b) { + if (a.throughput != b.throughput) return a.throughput > b.throughput; + return a.runId < b.runId; + }); + if (out.size() > limit) out.resize(limit); + return out; + } + +private: + std::vector samples_; + std::map seenRunIds_; + + static bool fail(std::string* error, const char* code) { + *error = code; + return false; + } + + const BenchmarkSample* findRun(const std::string& runId) const { + for (const auto& s : samples_) if (s.runId == runId) return &s; + return nullptr; + } +}; diff --git a/editor/tests/step591_test.cpp b/editor/tests/step591_test.cpp new file mode 100644 index 0000000..ff97d67 --- /dev/null +++ b/editor/tests/step591_test.cpp @@ -0,0 +1,165 @@ +// Step 591: Benchmark and Comparison Harness (12 tests) + +#include "BenchmarkComparisonHarness.h" + +#include + +static int passed = 0, failed = 0; +#define TEST(name) { std::cout << " " << #name << "... "; } +#define PASS() { std::cout << "PASS\n"; ++passed; } +#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; } +#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {} + +static BenchmarkSample sample(const std::string& runId, + const std::string& scenario, + int t, + int r, + int e) { + return {runId, scenario, t, r, e}; +} + +void test_add_sample_success() { + TEST(add_sample_success); + BenchmarkComparisonHarness h; + std::string error; + CHECK(h.addSample(sample("r1", "intake", 100, 95, 80), &error), "add should succeed"); + CHECK(h.aggregates().size() == 1, "aggregate count mismatch"); + PASS(); +} + +void test_add_sample_rejects_missing_run_id() { + TEST(add_sample_rejects_missing_run_id); + BenchmarkComparisonHarness h; + std::string error; + CHECK(!h.addSample(sample("", "intake", 100, 95, 80), &error), "add should fail"); + CHECK(error == "run_id_missing", "wrong error"); + PASS(); +} + +void test_add_sample_rejects_missing_scenario() { + TEST(add_sample_rejects_missing_scenario); + BenchmarkComparisonHarness h; + std::string error; + CHECK(!h.addSample(sample("r1", "", 100, 95, 80), &error), "add should fail"); + CHECK(error == "scenario_missing", "wrong error"); + PASS(); +} + +void test_add_sample_rejects_negative_metric() { + TEST(add_sample_rejects_negative_metric); + BenchmarkComparisonHarness h; + std::string error; + CHECK(!h.addSample(sample("r1", "intake", -1, 95, 80), &error), "add should fail"); + CHECK(error == "metric_invalid", "wrong error"); + PASS(); +} + +void test_add_sample_rejects_duplicate_run_id() { + TEST(add_sample_rejects_duplicate_run_id); + BenchmarkComparisonHarness h; + std::string error; + CHECK(h.addSample(sample("r1", "intake", 100, 95, 80), &error), "first add failed"); + CHECK(!h.addSample(sample("r1", "intake", 101, 96, 81), &error), "duplicate should fail"); + CHECK(error == "run_duplicate", "wrong error"); + PASS(); +} + +void test_aggregates_average_metrics_per_scenario() { + TEST(aggregates_average_metrics_per_scenario); + BenchmarkComparisonHarness h; + std::string error; + CHECK(h.addSample(sample("r1", "intake", 100, 90, 80), &error), "add r1 failed"); + CHECK(h.addSample(sample("r2", "intake", 200, 100, 60), &error), "add r2 failed"); + const auto a = h.aggregates(); + CHECK(a.size() == 1, "one aggregate expected"); + CHECK(a[0].avgThroughput == 150, "throughput avg mismatch"); + CHECK(a[0].avgReliability == 95, "reliability avg mismatch"); + CHECK(a[0].avgTokenEfficiency == 70, "efficiency avg mismatch"); + PASS(); +} + +void test_aggregates_include_multiple_scenarios() { + TEST(aggregates_include_multiple_scenarios); + BenchmarkComparisonHarness h; + std::string error; + CHECK(h.addSample(sample("r1", "intake", 100, 90, 80), &error), "add r1 failed"); + CHECK(h.addSample(sample("r2", "verify", 200, 95, 70), &error), "add r2 failed"); + CHECK(h.aggregates().size() == 2, "two aggregates expected"); + PASS(); +} + +void test_compare_runs_returns_deltas() { + TEST(compare_runs_returns_deltas); + BenchmarkComparisonHarness h; + std::string error; + CHECK(h.addSample(sample("base", "intake", 100, 90, 80), &error), "add base failed"); + CHECK(h.addSample(sample("cand", "intake", 130, 92, 78), &error), "add cand failed"); + const auto c = h.compareRuns("base", "cand"); + CHECK(c.size() == 1, "comparison should exist"); + CHECK(c[0].throughputDelta == 30, "throughput delta mismatch"); + CHECK(c[0].reliabilityDelta == 2, "reliability delta mismatch"); + CHECK(c[0].tokenEfficiencyDelta == -2, "efficiency delta mismatch"); + PASS(); +} + +void test_compare_runs_empty_when_missing_run() { + TEST(compare_runs_empty_when_missing_run); + BenchmarkComparisonHarness h; + std::string error; + CHECK(h.addSample(sample("base", "intake", 100, 90, 80), &error), "add base failed"); + CHECK(h.compareRuns("base", "missing").empty(), "comparison should be empty"); + PASS(); +} + +void test_compare_runs_empty_when_scenario_mismatch() { + TEST(compare_runs_empty_when_scenario_mismatch); + BenchmarkComparisonHarness h; + std::string error; + CHECK(h.addSample(sample("base", "intake", 100, 90, 80), &error), "add base failed"); + CHECK(h.addSample(sample("cand", "verify", 130, 92, 78), &error), "add cand failed"); + CHECK(h.compareRuns("base", "cand").empty(), "comparison should be empty"); + PASS(); +} + +void test_top_by_throughput_sorted_desc() { + TEST(top_by_throughput_sorted_desc); + BenchmarkComparisonHarness h; + std::string error; + CHECK(h.addSample(sample("r1", "intake", 100, 90, 80), &error), "add r1 failed"); + CHECK(h.addSample(sample("r2", "intake", 300, 90, 80), &error), "add r2 failed"); + CHECK(h.addSample(sample("r3", "intake", 200, 90, 80), &error), "add r3 failed"); + const auto top = h.topByThroughput("intake", 2); + CHECK(top.size() == 2, "top count mismatch"); + CHECK(top[0].runId == "r2", "top run mismatch"); + CHECK(top[1].runId == "r3", "second run mismatch"); + PASS(); +} + +void test_top_by_throughput_empty_limit() { + TEST(top_by_throughput_empty_limit); + BenchmarkComparisonHarness h; + std::string error; + CHECK(h.addSample(sample("r1", "intake", 100, 90, 80), &error), "add failed"); + CHECK(h.topByThroughput("intake", 0).empty(), "zero limit should return empty"); + PASS(); +} + +int main() { + std::cout << "Step 591: Benchmark and Comparison Harness\n"; + + test_add_sample_success(); // 1 + test_add_sample_rejects_missing_run_id(); // 2 + test_add_sample_rejects_missing_scenario(); // 3 + test_add_sample_rejects_negative_metric(); // 4 + test_add_sample_rejects_duplicate_run_id(); // 5 + test_aggregates_average_metrics_per_scenario(); // 6 + test_aggregates_include_multiple_scenarios(); // 7 + test_compare_runs_returns_deltas(); // 8 + test_compare_runs_empty_when_missing_run(); // 9 + test_compare_runs_empty_when_scenario_mismatch();// 10 + test_top_by_throughput_sorted_desc(); // 11 + test_top_by_throughput_empty_limit(); // 12 + + std::cout << "\nResults: " << passed << "/" << (passed + failed) << " passed\n"; + return failed == 0 ? 0 : 1; +} diff --git a/progress.md b/progress.md index 308adc0..0db1b12 100644 --- a/progress.md +++ b/progress.md @@ -11193,3 +11193,36 @@ continuity checks across workflow, queue, and open-buffer state. - `editor/src/CrashRecoveryReliabilitySweep.h` within header-size limit (`73` <= `600`) - `editor/tests/step590_test.cpp` within test-file size guidance (`186` lines) - Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md` + +### Step 591: Benchmark and Comparison Harness +**Status:** PASS (12/12 tests) + +Implements internal benchmark harness support for throughput/reliability/token- +efficiency tracking, aggregation, and baseline-vs-candidate comparison. + +**Files added:** +- `editor/src/BenchmarkComparisonHarness.h` - benchmark harness module: + - benchmark sample ingestion with validation/duplicate guards + - per-scenario metric aggregation + - baseline/candidate delta comparison + - throughput-ranked run extraction +- `editor/tests/step591_test.cpp` - 12 tests covering: + - sample ingestion success/failure behavior + - aggregate computation behavior + - comparison delta behavior and mismatch handling + - top-throughput ordering behavior + - zero-limit behavior + +**Files modified:** +- `editor/CMakeLists.txt` - `step591_test` target + +**Verification run:** +- `cmake -S editor -B editor/build-native` - PASS +- `cmake --build editor/build-native --target step591_test step590_test` - PASS +- `./editor/build-native/step591_test` - PASS (12/12) +- `./editor/build-native/step590_test` - PASS (12/12) regression coverage + +**Architecture gate check:** +- `editor/src/BenchmarkComparisonHarness.h` within header-size limit (`109` <= `600`) +- `editor/tests/step591_test.cpp` within test-file size guidance (`165` lines) +- Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md`