bill/whetstone_DSL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Step 591: add benchmark comparison harness

Browse Source
This commit is contained in:
Bill
2026-02-17 11:28:45 -07:00
parent cc1136f698
commit 014f85da94
4 changed files with 316 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
editor/CMakeLists.txt
View File

@@ -4198,4 +4198,13 @@ target_link_libraries(step590_test PRIVATE
tree_sitter_javascript tree_sitter_typescript
tree_sitter_java tree_sitter_rust tree_sitter_go)
add_executable(step591_test tests/step591_test.cpp)
target_include_directories(step591_test PRIVATE src)
target_link_libraries(step591_test PRIVATE
nlohmann_json::nlohmann_json
unofficial::tree-sitter::tree-sitter
tree_sitter_python tree_sitter_cpp tree_sitter_elisp
tree_sitter_javascript tree_sitter_typescript
tree_sitter_java tree_sitter_rust tree_sitter_go)
# Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies)

109
editor/src/BenchmarkComparisonHarness.h Normal file
View File

@@ -0,0 +1,109 @@
#pragma once
// Step 591: Benchmark and Comparison Harness
#include <algorithm>
#include <map>
#include <string>
#include <vector>
struct BenchmarkSample {
std::string runId;
std::string scenario;
int throughput = 0;
int reliability = 0;
int tokenEfficiency = 0;
};
struct BenchmarkAggregate {
std::string scenario;
int avgThroughput = 0;
int avgReliability = 0;
int avgTokenEfficiency = 0;
int sampleCount = 0;
};
struct BenchmarkComparison {
std::string scenario;
int throughputDelta = 0;
int reliabilityDelta = 0;
int tokenEfficiencyDelta = 0;
};
class BenchmarkComparisonHarness {
public:
bool addSample(const BenchmarkSample& sample, std::string* error) {
if (!error) return false;
error->clear();
if (sample.runId.empty()) return fail(error, "run_id_missing");
if (sample.scenario.empty()) return fail(error, "scenario_missing");
if (sample.throughput < 0 || sample.reliability < 0 || sample.tokenEfficiency < 0) {
return fail(error, "metric_invalid");
}
if (seenRunIds_.count(sample.runId) != 0) return fail(error, "run_duplicate");
seenRunIds_[sample.runId] = true;
samples_.push_back(sample);
return true;
}
std::vector<BenchmarkAggregate> aggregates() const {
struct Sum { int t = 0; int r = 0; int e = 0; int n = 0; };
std::map<std::string, Sum> sums;
for (const auto& s : samples_) {
auto& x = sums[s.scenario];
x.t += s.throughput;
x.r += s.reliability;
x.e += s.tokenEfficiency;
++x.n;
}
std::vector<BenchmarkAggregate> out;
for (const auto& kv : sums) {
BenchmarkAggregate a;
a.scenario = kv.first;
a.sampleCount = kv.second.n;
a.avgThroughput = kv.second.t / kv.second.n;
a.avgReliability = kv.second.r / kv.second.n;
a.avgTokenEfficiency = kv.second.e / kv.second.n;
out.push_back(a);
}
return out;
}
std::vector<BenchmarkComparison> compareRuns(const std::string& baselineRunId,
const std::string& candidateRunId) const {
const auto* a = findRun(baselineRunId);
const auto* b = findRun(candidateRunId);
if (!a || !b || a->scenario != b->scenario) return {};
BenchmarkComparison c;
c.scenario = a->scenario;
c.throughputDelta = b->throughput - a->throughput;
c.reliabilityDelta = b->reliability - a->reliability;
c.tokenEfficiencyDelta = b->tokenEfficiency - a->tokenEfficiency;
return {c};
}
std::vector<BenchmarkSample> topByThroughput(const std::string& scenario,
std::size_t limit) const {
std::vector<BenchmarkSample> out;
for (const auto& s : samples_) if (scenario.empty() || s.scenario == scenario) out.push_back(s);
std::stable_sort(out.begin(), out.end(), [](const BenchmarkSample& a, const BenchmarkSample& b) {
if (a.throughput != b.throughput) return a.throughput > b.throughput;
return a.runId < b.runId;
});
if (out.size() > limit) out.resize(limit);
return out;
}
private:
std::vector<BenchmarkSample> samples_;
std::map<std::string, bool> seenRunIds_;
static bool fail(std::string* error, const char* code) {
*error = code;
return false;
}
const BenchmarkSample* findRun(const std::string& runId) const {
for (const auto& s : samples_) if (s.runId == runId) return &s;
return nullptr;
}
};

165
editor/tests/step591_test.cpp Normal file
View File

@@ -0,0 +1,165 @@
// Step 591: Benchmark and Comparison Harness (12 tests)
#include "BenchmarkComparisonHarness.h"
#include <iostream>
static int passed = 0, failed = 0;
#define TEST(name) { std::cout << " " << #name << "... "; }
#define PASS() { std::cout << "PASS\n"; ++passed; }
#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; }
#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {}
static BenchmarkSample sample(const std::string& runId,
const std::string& scenario,
int t,
int r,
int e) {
return {runId, scenario, t, r, e};
}
void test_add_sample_success() {
TEST(add_sample_success);
BenchmarkComparisonHarness h;
std::string error;
CHECK(h.addSample(sample("r1", "intake", 100, 95, 80), &error), "add should succeed");
CHECK(h.aggregates().size() == 1, "aggregate count mismatch");
PASS();
}
void test_add_sample_rejects_missing_run_id() {
TEST(add_sample_rejects_missing_run_id);
BenchmarkComparisonHarness h;
std::string error;
CHECK(!h.addSample(sample("", "intake", 100, 95, 80), &error), "add should fail");
CHECK(error == "run_id_missing", "wrong error");
PASS();
}
void test_add_sample_rejects_missing_scenario() {
TEST(add_sample_rejects_missing_scenario);
BenchmarkComparisonHarness h;
std::string error;
CHECK(!h.addSample(sample("r1", "", 100, 95, 80), &error), "add should fail");
CHECK(error == "scenario_missing", "wrong error");
PASS();
}
void test_add_sample_rejects_negative_metric() {
TEST(add_sample_rejects_negative_metric);
BenchmarkComparisonHarness h;
std::string error;
CHECK(!h.addSample(sample("r1", "intake", -1, 95, 80), &error), "add should fail");
CHECK(error == "metric_invalid", "wrong error");
PASS();
}
void test_add_sample_rejects_duplicate_run_id() {
TEST(add_sample_rejects_duplicate_run_id);
BenchmarkComparisonHarness h;
std::string error;
CHECK(h.addSample(sample("r1", "intake", 100, 95, 80), &error), "first add failed");
CHECK(!h.addSample(sample("r1", "intake", 101, 96, 81), &error), "duplicate should fail");
CHECK(error == "run_duplicate", "wrong error");
PASS();
}
void test_aggregates_average_metrics_per_scenario() {
TEST(aggregates_average_metrics_per_scenario);
BenchmarkComparisonHarness h;
std::string error;
CHECK(h.addSample(sample("r1", "intake", 100, 90, 80), &error), "add r1 failed");
CHECK(h.addSample(sample("r2", "intake", 200, 100, 60), &error), "add r2 failed");
const auto a = h.aggregates();
CHECK(a.size() == 1, "one aggregate expected");
CHECK(a[0].avgThroughput == 150, "throughput avg mismatch");
CHECK(a[0].avgReliability == 95, "reliability avg mismatch");
CHECK(a[0].avgTokenEfficiency == 70, "efficiency avg mismatch");
PASS();
}
void test_aggregates_include_multiple_scenarios() {
TEST(aggregates_include_multiple_scenarios);
BenchmarkComparisonHarness h;
std::string error;
CHECK(h.addSample(sample("r1", "intake", 100, 90, 80), &error), "add r1 failed");
CHECK(h.addSample(sample("r2", "verify", 200, 95, 70), &error), "add r2 failed");
CHECK(h.aggregates().size() == 2, "two aggregates expected");
PASS();
}
void test_compare_runs_returns_deltas() {
TEST(compare_runs_returns_deltas);
BenchmarkComparisonHarness h;
std::string error;
CHECK(h.addSample(sample("base", "intake", 100, 90, 80), &error), "add base failed");
CHECK(h.addSample(sample("cand", "intake", 130, 92, 78), &error), "add cand failed");
const auto c = h.compareRuns("base", "cand");
CHECK(c.size() == 1, "comparison should exist");
CHECK(c[0].throughputDelta == 30, "throughput delta mismatch");
CHECK(c[0].reliabilityDelta == 2, "reliability delta mismatch");
CHECK(c[0].tokenEfficiencyDelta == -2, "efficiency delta mismatch");
PASS();
}
void test_compare_runs_empty_when_missing_run() {
TEST(compare_runs_empty_when_missing_run);
BenchmarkComparisonHarness h;
std::string error;
CHECK(h.addSample(sample("base", "intake", 100, 90, 80), &error), "add base failed");
CHECK(h.compareRuns("base", "missing").empty(), "comparison should be empty");
PASS();
}
void test_compare_runs_empty_when_scenario_mismatch() {
TEST(compare_runs_empty_when_scenario_mismatch);
BenchmarkComparisonHarness h;
std::string error;
CHECK(h.addSample(sample("base", "intake", 100, 90, 80), &error), "add base failed");
CHECK(h.addSample(sample("cand", "verify", 130, 92, 78), &error), "add cand failed");
CHECK(h.compareRuns("base", "cand").empty(), "comparison should be empty");
PASS();
}
void test_top_by_throughput_sorted_desc() {
TEST(top_by_throughput_sorted_desc);
BenchmarkComparisonHarness h;
std::string error;
CHECK(h.addSample(sample("r1", "intake", 100, 90, 80), &error), "add r1 failed");
CHECK(h.addSample(sample("r2", "intake", 300, 90, 80), &error), "add r2 failed");
CHECK(h.addSample(sample("r3", "intake", 200, 90, 80), &error), "add r3 failed");
const auto top = h.topByThroughput("intake", 2);
CHECK(top.size() == 2, "top count mismatch");
CHECK(top[0].runId == "r2", "top run mismatch");
CHECK(top[1].runId == "r3", "second run mismatch");
PASS();
}
void test_top_by_throughput_empty_limit() {
TEST(top_by_throughput_empty_limit);
BenchmarkComparisonHarness h;
std::string error;
CHECK(h.addSample(sample("r1", "intake", 100, 90, 80), &error), "add failed");
CHECK(h.topByThroughput("intake", 0).empty(), "zero limit should return empty");
PASS();
}
int main() {
std::cout << "Step 591: Benchmark and Comparison Harness\n";
test_add_sample_success(); // 1
test_add_sample_rejects_missing_run_id(); // 2
test_add_sample_rejects_missing_scenario(); // 3
test_add_sample_rejects_negative_metric(); // 4
test_add_sample_rejects_duplicate_run_id(); // 5
test_aggregates_average_metrics_per_scenario(); // 6
test_aggregates_include_multiple_scenarios(); // 7
test_compare_runs_returns_deltas(); // 8
test_compare_runs_empty_when_missing_run(); // 9
test_compare_runs_empty_when_scenario_mismatch();// 10
test_top_by_throughput_sorted_desc(); // 11
test_top_by_throughput_empty_limit(); // 12
std::cout << "\nResults: " << passed << "/" << (passed + failed) << " passed\n";
return failed == 0 ? 0 : 1;
}

33
progress.md
View File

@@ -11193,3 +11193,36 @@ continuity checks across workflow, queue, and open-buffer state.
- `editor/src/CrashRecoveryReliabilitySweep.h` within header-size limit (`73` <= `600`)
- `editor/tests/step590_test.cpp` within test-file size guidance (`186` lines)
- Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md`
### Step 591: Benchmark and Comparison Harness
**Status:** PASS (12/12 tests)
Implements internal benchmark harness support for throughput/reliability/token-
efficiency tracking, aggregation, and baseline-vs-candidate comparison.
**Files added:**
- `editor/src/BenchmarkComparisonHarness.h` - benchmark harness module:
- benchmark sample ingestion with validation/duplicate guards
- per-scenario metric aggregation
- baseline/candidate delta comparison
- throughput-ranked run extraction
- `editor/tests/step591_test.cpp` - 12 tests covering:
- sample ingestion success/failure behavior
- aggregate computation behavior
- comparison delta behavior and mismatch handling
- top-throughput ordering behavior
- zero-limit behavior
**Files modified:**
- `editor/CMakeLists.txt` - `step591_test` target
**Verification run:**
- `cmake -S editor -B editor/build-native` - PASS
- `cmake --build editor/build-native --target step591_test step590_test` - PASS
- `./editor/build-native/step591_test` - PASS (12/12)
- `./editor/build-native/step590_test` - PASS (12/12) regression coverage
**Architecture gate check:**
- `editor/src/BenchmarkComparisonHarness.h` within header-size limit (`109` <= `600`)
- `editor/tests/step591_test.cpp` within test-file size guidance (`165` lines)
- Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md`