diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 2441be2..0377e5f 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -3793,4 +3793,13 @@ target_link_libraries(step545_test PRIVATE tree_sitter_javascript tree_sitter_typescript tree_sitter_java tree_sitter_rust tree_sitter_go) +add_executable(step546_test tests/step546_test.cpp) +target_include_directories(step546_test PRIVATE src) +target_link_libraries(step546_test PRIVATE + nlohmann_json::nlohmann_json + unofficial::tree-sitter::tree-sitter + tree_sitter_python tree_sitter_cpp tree_sitter_elisp + tree_sitter_javascript tree_sitter_typescript + tree_sitter_java tree_sitter_rust tree_sitter_go) + # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies) diff --git a/editor/src/ConstrainedConfidenceCalibrator.h b/editor/src/ConstrainedConfidenceCalibrator.h new file mode 100644 index 0000000..6507cc5 --- /dev/null +++ b/editor/src/ConstrainedConfidenceCalibrator.h @@ -0,0 +1,97 @@ +#pragma once +// Step 546: Confidence Calibration for Constrained Paths + +#include +#include +#include + +struct CalibrationSample { + std::string route; + bool success = false; + bool postApplyFailure = false; +}; + +struct CalibratedConfidence { + double adjustedConfidence = 0.0; + std::string recommendedRoute; + std::map routeScores; +}; + +class ConstrainedConfidenceCalibrator { +public: + static CalibratedConfidence calibrate( + double baseConfidence, + const std::map& historicalSuccesses, + const std::map& historicalAttempts, + const std::map& historicalPostApplyFailures, + const std::string& preferredRoute) { + CalibratedConfidence out; + + out.routeScores["deterministic_template"] = + scoreFor("deterministic_template", baseConfidence, + historicalSuccesses, historicalAttempts, historicalPostApplyFailures); + out.routeScores["constrained_worker"] = + scoreFor("constrained_worker", baseConfidence, + historicalSuccesses, historicalAttempts, historicalPostApplyFailures); + out.routeScores["general_worker"] = + scoreFor("general_worker", baseConfidence, + historicalSuccesses, historicalAttempts, historicalPostApplyFailures); + + // Mild preference bias only if route is not heavily penalized. + if (!preferredRoute.empty()) { + out.routeScores[preferredRoute] += 0.03; + } + for (auto& kv : out.routeScores) { + kv.second = clamp(kv.second); + } + + out.recommendedRoute = bestRoute(out.routeScores); + out.adjustedConfidence = clamp(out.routeScores[out.recommendedRoute]); + return out; + } + +private: + static double scoreFor(const std::string& route, + double base, + const std::map& successes, + const std::map& attempts, + const std::map& postApplyFailures) { + const int s = lookup(successes, route); + const int a = lookup(attempts, route); + const int f = lookup(postApplyFailures, route); + + double successRate = (a > 0) ? (double)s / (double)a : 0.5; + double failurePenalty = (a > 0) ? (double)f / (double)a : 0.0; + + double score = base; + score += 0.35 * (successRate - 0.5); + score -= 0.45 * failurePenalty; + + // Repeated post-apply failures trigger stronger decay. + if (f >= 3) score -= 0.10; + if (f >= 5) score -= 0.10; + + return clamp(score); + } + + static int lookup(const std::map& m, const std::string& key) { + auto it = m.find(key); + return it == m.end() ? 0 : it->second; + } + + static std::string bestRoute(const std::map& scores) { + std::string best = "general_worker"; + double bestScore = -1.0; + for (const auto& kv : scores) { + if (kv.second > bestScore) { + bestScore = kv.second; + best = kv.first; + } + } + return best; + } + + static double clamp(double x) { + return std::max(0.0, std::min(1.0, x)); + } +}; diff --git a/editor/tests/step546_test.cpp b/editor/tests/step546_test.cpp new file mode 100644 index 0000000..d3eb625 --- /dev/null +++ b/editor/tests/step546_test.cpp @@ -0,0 +1,143 @@ +// Step 546: Confidence Calibration for Constrained Paths (12 tests) + +#include "ConstrainedConfidenceCalibrator.h" + +#include + +static int passed = 0, failed = 0; +#define TEST(name) { std::cout << " " << #name << "... "; } +#define PASS() { std::cout << "PASS\n"; ++passed; } +#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; } +#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {} + +void test_no_history_defaults_to_base_confidence_band() { + TEST(no_history_defaults_to_base_confidence_band); + auto r = ConstrainedConfidenceCalibrator::calibrate(0.70, {}, {}, {}, ""); + CHECK(r.adjustedConfidence > 0.65 && r.adjustedConfidence < 0.75, + "default confidence should remain near base"); + PASS(); +} + +void test_high_success_rate_increases_score() { + TEST(high_success_rate_increases_score); + std::map succ{{"constrained_worker", 9}}; + std::map att{{"constrained_worker", 10}}; + auto r = ConstrainedConfidenceCalibrator::calibrate(0.60, succ, att, {}, "constrained_worker"); + CHECK(r.routeScores["constrained_worker"] > 0.70, "high success should boost score"); + PASS(); +} + +void test_low_success_rate_decreases_score() { + TEST(low_success_rate_decreases_score); + std::map succ{{"deterministic_template", 1}}; + std::map att{{"deterministic_template", 10}}; + auto r = ConstrainedConfidenceCalibrator::calibrate(0.70, succ, att, {}, "deterministic_template"); + CHECK(r.routeScores["deterministic_template"] < 0.60, "low success should reduce score"); + PASS(); +} + +void test_post_apply_failures_penalize_route() { + TEST(post_apply_failures_penalize_route); + std::map succ{{"constrained_worker", 8}}; + std::map att{{"constrained_worker", 10}}; + std::map fail{{"constrained_worker", 4}}; + auto r = ConstrainedConfidenceCalibrator::calibrate(0.75, succ, att, fail, "constrained_worker"); + CHECK(r.routeScores["constrained_worker"] < 0.70, "post-apply failures should penalize score"); + PASS(); +} + +void test_repeated_failures_trigger_extra_decay() { + TEST(repeated_failures_trigger_extra_decay); + std::map att{{"constrained_worker", 10}}; + std::map fail3{{"constrained_worker", 3}}; + std::map fail5{{"constrained_worker", 5}}; + auto r3 = ConstrainedConfidenceCalibrator::calibrate(0.8, {}, att, fail3, "constrained_worker"); + auto r5 = ConstrainedConfidenceCalibrator::calibrate(0.8, {}, att, fail5, "constrained_worker"); + CHECK(r5.routeScores["constrained_worker"] < r3.routeScores["constrained_worker"], + "more repeated failures should decay confidence further"); + PASS(); +} + +void test_best_route_selected_by_highest_score() { + TEST(best_route_selected_by_highest_score); + std::map succ{{"deterministic_template", 9}, {"constrained_worker", 4}}; + std::map att{{"deterministic_template", 10}, {"constrained_worker", 10}}; + auto r = ConstrainedConfidenceCalibrator::calibrate(0.65, succ, att, {}, ""); + CHECK(r.recommendedRoute == "deterministic_template", "highest score route should be selected"); + PASS(); +} + +void test_preferred_route_bias_applies() { + TEST(preferred_route_bias_applies); + auto a = ConstrainedConfidenceCalibrator::calibrate(0.70, {}, {}, {}, ""); + auto b = ConstrainedConfidenceCalibrator::calibrate(0.70, {}, {}, {}, "constrained_worker"); + CHECK(b.routeScores["constrained_worker"] > a.routeScores["constrained_worker"], + "preferred route should receive bias"); + PASS(); +} + +void test_scores_are_clamped_to_one() { + TEST(scores_are_clamped_to_one); + std::map succ{{"deterministic_template", 100}}; + std::map att{{"deterministic_template", 100}}; + auto r = ConstrainedConfidenceCalibrator::calibrate(0.99, succ, att, {}, "deterministic_template"); + CHECK(r.routeScores["deterministic_template"] <= 1.0, "score should clamp to one"); + PASS(); +} + +void test_scores_are_clamped_to_zero() { + TEST(scores_are_clamped_to_zero); + std::map att{{"constrained_worker", 10}}; + std::map fail{{"constrained_worker", 10}}; + auto r = ConstrainedConfidenceCalibrator::calibrate(0.05, {}, att, fail, "constrained_worker"); + CHECK(r.routeScores["constrained_worker"] >= 0.0, "score should clamp at zero"); + PASS(); +} + +void test_adjusted_confidence_matches_recommended_route_score() { + TEST(adjusted_confidence_matches_recommended_route_score); + std::map succ{{"general_worker", 8}}; + std::map att{{"general_worker", 10}}; + auto r = ConstrainedConfidenceCalibrator::calibrate(0.60, succ, att, {}, "general_worker"); + CHECK(r.adjustedConfidence == r.routeScores[r.recommendedRoute], + "adjusted confidence should match recommended route score"); + PASS(); +} + +void test_missing_attempts_uses_neutral_prior() { + TEST(missing_attempts_uses_neutral_prior); + std::map succ{{"constrained_worker", 5}}; + auto r = ConstrainedConfidenceCalibrator::calibrate(0.70, succ, {}, {}, "constrained_worker"); + CHECK(r.routeScores["constrained_worker"] > 0.65 && r.routeScores["constrained_worker"] < 0.80, + "missing attempts should stay near base with mild bias only"); + PASS(); +} + +void test_route_scores_include_all_policy_routes() { + TEST(route_scores_include_all_policy_routes); + auto r = ConstrainedConfidenceCalibrator::calibrate(0.70, {}, {}, {}, ""); + CHECK(r.routeScores.count("deterministic_template") == 1, "missing deterministic route score"); + CHECK(r.routeScores.count("constrained_worker") == 1, "missing constrained route score"); + CHECK(r.routeScores.count("general_worker") == 1, "missing general route score"); + PASS(); +} + +int main() { + std::cout << "Step 546: Confidence Calibration for Constrained Paths\n"; + + test_no_history_defaults_to_base_confidence_band(); // 1 + test_high_success_rate_increases_score(); // 2 + test_low_success_rate_decreases_score(); // 3 + test_post_apply_failures_penalize_route(); // 4 + test_repeated_failures_trigger_extra_decay(); // 5 + test_best_route_selected_by_highest_score(); // 6 + test_preferred_route_bias_applies(); // 7 + test_scores_are_clamped_to_one(); // 8 + test_scores_are_clamped_to_zero(); // 9 + test_adjusted_confidence_matches_recommended_route_score(); // 10 + test_missing_attempts_uses_neutral_prior(); // 11 + test_route_scores_include_all_policy_routes(); // 12 + + std::cout << "\nResults: " << passed << "/" << (passed + failed) << " passed\n"; + return failed == 0 ? 0 : 1; +} diff --git a/progress.md b/progress.md index ba4b13a..5d3d09a 100644 --- a/progress.md +++ b/progress.md @@ -9433,3 +9433,40 @@ dependency noise for narrow operations. - `editor/src/ContextBundleMinimizer.h` within header-size limit (`88` <= `600`) - `editor/tests/step545_test.cpp` within test-file size guidance (`169` lines) - Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md` + +### Step 546: Confidence Calibration for Constrained Paths +**Status:** PASS (12/12 tests) + +Implements confidence calibration for constrained routing by combining base +legal-choice confidence with historical success/failure rates and explicit +penalties for repeated post-apply failures. + +**Files added:** +- `editor/src/ConstrainedConfidenceCalibrator.h` - confidence calibration module: + - route scoring for deterministic/constrained/general policy paths + - success-rate boosts and post-apply failure penalties + - extra decay for repeated post-apply failures + - preferred-route bias with post-bias score clamping + - best-route selection and adjusted-confidence emission +- `editor/tests/step546_test.cpp` - 12 tests covering: + - neutral/no-history behavior + - success/failure rate calibration effects + - repeated-failure decay behavior + - preferred-route bias behavior + - clamping behavior at score bounds + - recommended-route and adjusted-confidence consistency + - route score coverage across all policy paths + +**Files modified:** +- `editor/CMakeLists.txt` - `step546_test` target + +**Verification run:** +- `cmake -S editor -B editor/build-native` - PASS +- `cmake --build editor/build-native --target step546_test step545_test` - PASS +- `./editor/build-native/step546_test` - PASS (12/12) +- `./editor/build-native/step545_test` - PASS (12/12) regression coverage + +**Architecture gate check:** +- `editor/src/ConstrainedConfidenceCalibrator.h` within header-size limit (`97` <= `600`) +- `editor/tests/step546_test.cpp` within test-file size guidance (`143` lines) +- Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md`