From 29840c8e3e8acf8c5411f319cd99aff26734b581 Mon Sep 17 00:00:00 2001 From: Bill Date: Tue, 17 Feb 2026 10:22:18 -0700 Subject: [PATCH] Step 552: add cost-vs-quality regression suite --- editor/CMakeLists.txt | 9 ++ editor/src/CostQualityRegressionSuite.h | 60 +++++++++ editor/tests/step552_test.cpp | 167 ++++++++++++++++++++++++ progress.md | 36 +++++ 4 files changed, 272 insertions(+) create mode 100644 editor/src/CostQualityRegressionSuite.h create mode 100644 editor/tests/step552_test.cpp diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 318595c..9d91afc 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -3847,4 +3847,13 @@ target_link_libraries(step551_test PRIVATE tree_sitter_javascript tree_sitter_typescript tree_sitter_java tree_sitter_rust tree_sitter_go) +add_executable(step552_test tests/step552_test.cpp) +target_include_directories(step552_test PRIVATE src) +target_link_libraries(step552_test PRIVATE + nlohmann_json::nlohmann_json + unofficial::tree-sitter::tree-sitter + tree_sitter_python tree_sitter_cpp tree_sitter_elisp + tree_sitter_javascript tree_sitter_typescript + tree_sitter_java tree_sitter_rust tree_sitter_go) + # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies) diff --git a/editor/src/CostQualityRegressionSuite.h b/editor/src/CostQualityRegressionSuite.h new file mode 100644 index 0000000..c55eace --- /dev/null +++ b/editor/src/CostQualityRegressionSuite.h @@ -0,0 +1,60 @@ +#pragma once +// Step 552: Cost vs Quality Regression Suite + +#include +#include + +struct CostQualityRun { + std::string profile; + int tokensUsed = 0; + double qualityScore = 0.0; // 0..1 + double passRate = 0.0; // 0..1 + int safetyIncidents = 0; +}; + +struct CostQualityPolicy { + double minQualityScore = 0.0; + double minPassRate = 0.0; + int maxSafetyIncidents = 0; + int minTokenReduction = 0; +}; + +struct CostQualityRegressionResult { + bool pass = false; + int tokenReduction = 0; + std::vector failures; +}; + +class CostQualityRegressionSuite { +public: + static CostQualityRegressionResult evaluate(const CostQualityRun& baseline, + const CostQualityRun& optimized, + const CostQualityPolicy& policy) { + CostQualityRegressionResult out; + out.tokenReduction = baseline.tokensUsed - optimized.tokensUsed; + + if (out.tokenReduction < policy.minTokenReduction) { + out.failures.push_back("insufficient_token_reduction"); + } + if (optimized.qualityScore < policy.minQualityScore) { + out.failures.push_back("quality_below_floor"); + } + if (optimized.passRate < policy.minPassRate) { + out.failures.push_back("pass_rate_below_floor"); + } + if (optimized.safetyIncidents > policy.maxSafetyIncidents) { + out.failures.push_back("safety_incidents_exceeded"); + } + + // Relative regression guard: avoid hidden quality collapse even if absolute floor passed. + if (optimized.qualityScore + 1e-9 < baseline.qualityScore - 0.08) { + out.failures.push_back("relative_quality_regression"); + } + if (optimized.passRate + 1e-9 < baseline.passRate - 0.08) { + out.failures.push_back("relative_pass_rate_regression"); + } + + out.pass = out.failures.empty(); + return out; + } +}; diff --git a/editor/tests/step552_test.cpp b/editor/tests/step552_test.cpp new file mode 100644 index 0000000..c902a81 --- /dev/null +++ b/editor/tests/step552_test.cpp @@ -0,0 +1,167 @@ +// Step 552: Cost vs Quality Regression Suite (12 tests) + +#include "CostQualityRegressionSuite.h" + +#include + +static int passed = 0, failed = 0; +#define TEST(name) { std::cout << " " << #name << "... "; } +#define PASS() { std::cout << "PASS\n"; ++passed; } +#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; } +#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {} + +static bool hasFailure(const CostQualityRegressionResult& r, const std::string& f) { + for (const auto& x : r.failures) if (x == f) return true; + return false; +} + +static CostQualityPolicy policy() { + CostQualityPolicy p; + p.minQualityScore = 0.80; + p.minPassRate = 0.85; + p.maxSafetyIncidents = 0; + p.minTokenReduction = 20; + return p; +} + +void test_pass_when_reduction_and_quality_targets_met() { + TEST(pass_when_reduction_and_quality_targets_met); + auto r = CostQualityRegressionSuite::evaluate( + {"baseline", 200, 0.90, 0.92, 0}, + {"optimized", 160, 0.86, 0.90, 0}, + policy()); + CHECK(r.pass, "expected pass"); + PASS(); +} + +void test_fails_when_token_reduction_insufficient() { + TEST(fails_when_token_reduction_insufficient); + auto r = CostQualityRegressionSuite::evaluate( + {"baseline", 200, 0.90, 0.92, 0}, + {"optimized", 190, 0.86, 0.90, 0}, + policy()); + CHECK(hasFailure(r, "insufficient_token_reduction"), "token reduction failure expected"); + PASS(); +} + +void test_fails_when_quality_below_floor() { + TEST(fails_when_quality_below_floor); + auto r = CostQualityRegressionSuite::evaluate( + {"baseline", 200, 0.90, 0.92, 0}, + {"optimized", 160, 0.75, 0.90, 0}, + policy()); + CHECK(hasFailure(r, "quality_below_floor"), "quality floor failure expected"); + PASS(); +} + +void test_fails_when_pass_rate_below_floor() { + TEST(fails_when_pass_rate_below_floor); + auto r = CostQualityRegressionSuite::evaluate( + {"baseline", 200, 0.90, 0.92, 0}, + {"optimized", 160, 0.86, 0.80, 0}, + policy()); + CHECK(hasFailure(r, "pass_rate_below_floor"), "pass rate floor failure expected"); + PASS(); +} + +void test_fails_when_safety_incidents_exceed_limit() { + TEST(fails_when_safety_incidents_exceed_limit); + auto r = CostQualityRegressionSuite::evaluate( + {"baseline", 200, 0.90, 0.92, 0}, + {"optimized", 160, 0.86, 0.90, 1}, + policy()); + CHECK(hasFailure(r, "safety_incidents_exceeded"), "safety failure expected"); + PASS(); +} + +void test_relative_quality_regression_detected() { + TEST(relative_quality_regression_detected); + auto r = CostQualityRegressionSuite::evaluate( + {"baseline", 200, 0.95, 0.92, 0}, + {"optimized", 160, 0.84, 0.90, 0}, + policy()); + CHECK(hasFailure(r, "relative_quality_regression"), "relative quality regression expected"); + PASS(); +} + +void test_relative_pass_rate_regression_detected() { + TEST(relative_pass_rate_regression_detected); + auto r = CostQualityRegressionSuite::evaluate( + {"baseline", 200, 0.90, 0.99, 0}, + {"optimized", 160, 0.86, 0.90, 0}, + policy()); + CHECK(hasFailure(r, "relative_pass_rate_regression"), "relative pass rate regression expected"); + PASS(); +} + +void test_multiple_failures_can_be_reported_together() { + TEST(multiple_failures_can_be_reported_together); + auto r = CostQualityRegressionSuite::evaluate( + {"baseline", 200, 0.95, 0.98, 0}, + {"optimized", 195, 0.70, 0.70, 2}, + policy()); + CHECK(r.failures.size() >= 4, "multiple failures expected"); + PASS(); +} + +void test_exact_threshold_values_pass() { + TEST(exact_threshold_values_pass); + auto r = CostQualityRegressionSuite::evaluate( + {"baseline", 200, 0.88, 0.90, 0}, + {"optimized", 180, 0.80, 0.85, 0}, + policy()); + CHECK(r.pass, "exact floor and reduction thresholds should pass"); + PASS(); +} + +void test_zero_safety_incident_limit_enforced() { + TEST(zero_safety_incident_limit_enforced); + auto p = policy(); + p.maxSafetyIncidents = 0; + auto r = CostQualityRegressionSuite::evaluate( + {"baseline", 200, 0.90, 0.92, 0}, + {"optimized", 160, 0.86, 0.90, 0}, + p); + CHECK(r.pass, "zero incidents should satisfy limit"); + PASS(); +} + +void test_token_reduction_is_reported_in_result() { + TEST(token_reduction_is_reported_in_result); + auto r = CostQualityRegressionSuite::evaluate( + {"baseline", 240, 0.90, 0.92, 0}, + {"optimized", 180, 0.86, 0.90, 0}, + policy()); + CHECK(r.tokenReduction == 60, "token reduction should be reported"); + PASS(); +} + +void test_negative_token_reduction_fails_policy() { + TEST(negative_token_reduction_fails_policy); + auto r = CostQualityRegressionSuite::evaluate( + {"baseline", 180, 0.90, 0.92, 0}, + {"optimized", 220, 0.86, 0.90, 0}, + policy()); + CHECK(hasFailure(r, "insufficient_token_reduction"), "negative reduction should fail policy"); + PASS(); +} + +int main() { + std::cout << "Step 552: Cost vs Quality Regression Suite\n"; + + test_pass_when_reduction_and_quality_targets_met(); // 1 + test_fails_when_token_reduction_insufficient(); // 2 + test_fails_when_quality_below_floor(); // 3 + test_fails_when_pass_rate_below_floor(); // 4 + test_fails_when_safety_incidents_exceed_limit(); // 5 + test_relative_quality_regression_detected(); // 6 + test_relative_pass_rate_regression_detected(); // 7 + test_multiple_failures_can_be_reported_together(); // 8 + test_exact_threshold_values_pass(); // 9 + test_zero_safety_incident_limit_enforced(); // 10 + test_token_reduction_is_reported_in_result(); // 11 + test_negative_token_reduction_fails_policy(); // 12 + + std::cout << "\nResults: " << passed << "/" << (passed + failed) << " passed\n"; + return failed == 0 ? 0 : 1; +} diff --git a/progress.md b/progress.md index 630fcd4..a92ff54 100644 --- a/progress.md +++ b/progress.md @@ -9657,3 +9657,39 @@ escalations while preserving hard safety/security/policy protections. - `editor/src/ReviewGatePolicyRefinement.h` within header-size limit (`70` <= `600`) - `editor/tests/step551_test.cpp` within test-file size guidance (`132` lines) - Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md` + +### Step 552: Cost vs Quality Regression Suite +**Status:** PASS (12/12 tests) + +Implements a cost-vs-quality regression suite to enforce that optimization +policy tightening preserves quality, pass rate, and safety while meeting token +reduction targets. + +**Files added:** +- `editor/src/CostQualityRegressionSuite.h` - regression suite module: + - evaluates optimized runs against absolute quality/pass/safety floors + - enforces minimum token reduction targets + - detects relative quality/pass regressions vs baseline + - reports detailed failure codes and token reduction deltas +- `editor/tests/step552_test.cpp` - 12 tests covering: + - passing baseline/optimized scenario + - token/quality/pass/safety floor failures + - relative regression detection behavior + - multi-failure reporting behavior + - threshold boundary behavior + - token reduction reporting behavior + - negative reduction failure behavior + +**Files modified:** +- `editor/CMakeLists.txt` - `step552_test` target + +**Verification run:** +- `cmake -S editor -B editor/build-native` - PASS +- `cmake --build editor/build-native --target step552_test step551_test` - PASS +- `./editor/build-native/step552_test` - PASS (12/12) +- `./editor/build-native/step551_test` - PASS (12/12) regression coverage + +**Architecture gate check:** +- `editor/src/CostQualityRegressionSuite.h` within header-size limit (`60` <= `600`) +- `editor/tests/step552_test.cpp` within test-file size guidance (`167` lines) +- Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md`