Step 546: add constrained confidence calibration model

2026-02-17 10:14:46 -07:00
parent a25f02345b
commit 6d4b806849
4 changed files with 286 additions and 0 deletions
--- a/editor/CMakeLists.txt
+++ b/editor/CMakeLists.txt
@@ -3793,4 +3793,13 @@ target_link_libraries(step545_test PRIVATE
    tree_sitter_javascript tree_sitter_typescript
    tree_sitter_java tree_sitter_rust tree_sitter_go)

+add_executable(step546_test tests/step546_test.cpp)
+target_include_directories(step546_test PRIVATE src)
+target_link_libraries(step546_test PRIVATE
+    nlohmann_json::nlohmann_json
+    unofficial::tree-sitter::tree-sitter
+    tree_sitter_python tree_sitter_cpp tree_sitter_elisp
+    tree_sitter_javascript tree_sitter_typescript
+    tree_sitter_java tree_sitter_rust tree_sitter_go)
+
 # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies)
--- a/editor/src/ConstrainedConfidenceCalibrator.h
+++ b/editor/src/ConstrainedConfidenceCalibrator.h
@@ -0,0 +1,97 @@
+#pragma once
+// Step 546: Confidence Calibration for Constrained Paths
+
+#include <algorithm>
+#include <map>
+#include <string>
+
+struct CalibrationSample {
+    std::string route;
+    bool success = false;
+    bool postApplyFailure = false;
+};
+
+struct CalibratedConfidence {
+    double adjustedConfidence = 0.0;
+    std::string recommendedRoute;
+    std::map<std::string, double> routeScores;
+};
+
+class ConstrainedConfidenceCalibrator {
+public:
+    static CalibratedConfidence calibrate(
+        double baseConfidence,
+        const std::map<std::string, int>& historicalSuccesses,
+        const std::map<std::string, int>& historicalAttempts,
+        const std::map<std::string, int>& historicalPostApplyFailures,
+        const std::string& preferredRoute) {
+        CalibratedConfidence out;
+
+        out.routeScores["deterministic_template"] =
+            scoreFor("deterministic_template", baseConfidence,
+                     historicalSuccesses, historicalAttempts, historicalPostApplyFailures);
+        out.routeScores["constrained_worker"] =
+            scoreFor("constrained_worker", baseConfidence,
+                     historicalSuccesses, historicalAttempts, historicalPostApplyFailures);
+        out.routeScores["general_worker"] =
+            scoreFor("general_worker", baseConfidence,
+                     historicalSuccesses, historicalAttempts, historicalPostApplyFailures);
+
+        // Mild preference bias only if route is not heavily penalized.
+        if (!preferredRoute.empty()) {
+            out.routeScores[preferredRoute] += 0.03;
+        }
+        for (auto& kv : out.routeScores) {
+            kv.second = clamp(kv.second);
+        }
+
+        out.recommendedRoute = bestRoute(out.routeScores);
+        out.adjustedConfidence = clamp(out.routeScores[out.recommendedRoute]);
+        return out;
+    }
+
+private:
+    static double scoreFor(const std::string& route,
+                           double base,
+                           const std::map<std::string, int>& successes,
+                           const std::map<std::string, int>& attempts,
+                           const std::map<std::string, int>& postApplyFailures) {
+        const int s = lookup(successes, route);
+        const int a = lookup(attempts, route);
+        const int f = lookup(postApplyFailures, route);
+
+        double successRate = (a > 0) ? (double)s / (double)a : 0.5;
+        double failurePenalty = (a > 0) ? (double)f / (double)a : 0.0;
+
+        double score = base;
+        score += 0.35 * (successRate - 0.5);
+        score -= 0.45 * failurePenalty;
+
+        // Repeated post-apply failures trigger stronger decay.
+        if (f >= 3) score -= 0.10;
+        if (f >= 5) score -= 0.10;
+
+        return clamp(score);
+    }
+
+    static int lookup(const std::map<std::string, int>& m, const std::string& key) {
+        auto it = m.find(key);
+        return it == m.end() ? 0 : it->second;
+    }
+
+    static std::string bestRoute(const std::map<std::string, double>& scores) {
+        std::string best = "general_worker";
+        double bestScore = -1.0;
+        for (const auto& kv : scores) {
+            if (kv.second > bestScore) {
+                bestScore = kv.second;
+                best = kv.first;
+            }
+        }
+        return best;
+    }
+
+    static double clamp(double x) {
+        return std::max(0.0, std::min(1.0, x));
+    }
+};
--- a/editor/tests/step546_test.cpp
+++ b/editor/tests/step546_test.cpp
@@ -0,0 +1,143 @@
+// Step 546: Confidence Calibration for Constrained Paths (12 tests)
+
+#include "ConstrainedConfidenceCalibrator.h"
+
+#include <iostream>
+
+static int passed = 0, failed = 0;
+#define TEST(name) { std::cout << "  " << #name << "... "; }
+#define PASS() { std::cout << "PASS\n"; ++passed; }
+#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; }
+#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {}
+
+void test_no_history_defaults_to_base_confidence_band() {
+    TEST(no_history_defaults_to_base_confidence_band);
+    auto r = ConstrainedConfidenceCalibrator::calibrate(0.70, {}, {}, {}, "");
+    CHECK(r.adjustedConfidence > 0.65 && r.adjustedConfidence < 0.75,
+          "default confidence should remain near base");
+    PASS();
+}
+
+void test_high_success_rate_increases_score() {
+    TEST(high_success_rate_increases_score);
+    std::map<std::string, int> succ{{"constrained_worker", 9}};
+    std::map<std::string, int> att{{"constrained_worker", 10}};
+    auto r = ConstrainedConfidenceCalibrator::calibrate(0.60, succ, att, {}, "constrained_worker");
+    CHECK(r.routeScores["constrained_worker"] > 0.70, "high success should boost score");
+    PASS();
+}
+
+void test_low_success_rate_decreases_score() {
+    TEST(low_success_rate_decreases_score);
+    std::map<std::string, int> succ{{"deterministic_template", 1}};
+    std::map<std::string, int> att{{"deterministic_template", 10}};
+    auto r = ConstrainedConfidenceCalibrator::calibrate(0.70, succ, att, {}, "deterministic_template");
+    CHECK(r.routeScores["deterministic_template"] < 0.60, "low success should reduce score");
+    PASS();
+}
+
+void test_post_apply_failures_penalize_route() {
+    TEST(post_apply_failures_penalize_route);
+    std::map<std::string, int> succ{{"constrained_worker", 8}};
+    std::map<std::string, int> att{{"constrained_worker", 10}};
+    std::map<std::string, int> fail{{"constrained_worker", 4}};
+    auto r = ConstrainedConfidenceCalibrator::calibrate(0.75, succ, att, fail, "constrained_worker");
+    CHECK(r.routeScores["constrained_worker"] < 0.70, "post-apply failures should penalize score");
+    PASS();
+}
+
+void test_repeated_failures_trigger_extra_decay() {
+    TEST(repeated_failures_trigger_extra_decay);
+    std::map<std::string, int> att{{"constrained_worker", 10}};
+    std::map<std::string, int> fail3{{"constrained_worker", 3}};
+    std::map<std::string, int> fail5{{"constrained_worker", 5}};
+    auto r3 = ConstrainedConfidenceCalibrator::calibrate(0.8, {}, att, fail3, "constrained_worker");
+    auto r5 = ConstrainedConfidenceCalibrator::calibrate(0.8, {}, att, fail5, "constrained_worker");
+    CHECK(r5.routeScores["constrained_worker"] < r3.routeScores["constrained_worker"],
+          "more repeated failures should decay confidence further");
+    PASS();
+}
+
+void test_best_route_selected_by_highest_score() {
+    TEST(best_route_selected_by_highest_score);
+    std::map<std::string, int> succ{{"deterministic_template", 9}, {"constrained_worker", 4}};
+    std::map<std::string, int> att{{"deterministic_template", 10}, {"constrained_worker", 10}};
+    auto r = ConstrainedConfidenceCalibrator::calibrate(0.65, succ, att, {}, "");
+    CHECK(r.recommendedRoute == "deterministic_template", "highest score route should be selected");
+    PASS();
+}
+
+void test_preferred_route_bias_applies() {
+    TEST(preferred_route_bias_applies);
+    auto a = ConstrainedConfidenceCalibrator::calibrate(0.70, {}, {}, {}, "");
+    auto b = ConstrainedConfidenceCalibrator::calibrate(0.70, {}, {}, {}, "constrained_worker");
+    CHECK(b.routeScores["constrained_worker"] > a.routeScores["constrained_worker"],
+          "preferred route should receive bias");
+    PASS();
+}
+
+void test_scores_are_clamped_to_one() {
+    TEST(scores_are_clamped_to_one);
+    std::map<std::string, int> succ{{"deterministic_template", 100}};
+    std::map<std::string, int> att{{"deterministic_template", 100}};
+    auto r = ConstrainedConfidenceCalibrator::calibrate(0.99, succ, att, {}, "deterministic_template");
+    CHECK(r.routeScores["deterministic_template"] <= 1.0, "score should clamp to one");
+    PASS();
+}
+
+void test_scores_are_clamped_to_zero() {
+    TEST(scores_are_clamped_to_zero);
+    std::map<std::string, int> att{{"constrained_worker", 10}};
+    std::map<std::string, int> fail{{"constrained_worker", 10}};
+    auto r = ConstrainedConfidenceCalibrator::calibrate(0.05, {}, att, fail, "constrained_worker");
+    CHECK(r.routeScores["constrained_worker"] >= 0.0, "score should clamp at zero");
+    PASS();
+}
+
+void test_adjusted_confidence_matches_recommended_route_score() {
+    TEST(adjusted_confidence_matches_recommended_route_score);
+    std::map<std::string, int> succ{{"general_worker", 8}};
+    std::map<std::string, int> att{{"general_worker", 10}};
+    auto r = ConstrainedConfidenceCalibrator::calibrate(0.60, succ, att, {}, "general_worker");
+    CHECK(r.adjustedConfidence == r.routeScores[r.recommendedRoute],
+          "adjusted confidence should match recommended route score");
+    PASS();
+}
+
+void test_missing_attempts_uses_neutral_prior() {
+    TEST(missing_attempts_uses_neutral_prior);
+    std::map<std::string, int> succ{{"constrained_worker", 5}};
+    auto r = ConstrainedConfidenceCalibrator::calibrate(0.70, succ, {}, {}, "constrained_worker");
+    CHECK(r.routeScores["constrained_worker"] > 0.65 && r.routeScores["constrained_worker"] < 0.80,
+          "missing attempts should stay near base with mild bias only");
+    PASS();
+}
+
+void test_route_scores_include_all_policy_routes() {
+    TEST(route_scores_include_all_policy_routes);
+    auto r = ConstrainedConfidenceCalibrator::calibrate(0.70, {}, {}, {}, "");
+    CHECK(r.routeScores.count("deterministic_template") == 1, "missing deterministic route score");
+    CHECK(r.routeScores.count("constrained_worker") == 1, "missing constrained route score");
+    CHECK(r.routeScores.count("general_worker") == 1, "missing general route score");
+    PASS();
+}
+
+int main() {
+    std::cout << "Step 546: Confidence Calibration for Constrained Paths\n";
+
+    test_no_history_defaults_to_base_confidence_band();      // 1
+    test_high_success_rate_increases_score();                // 2
+    test_low_success_rate_decreases_score();                 // 3
+    test_post_apply_failures_penalize_route();               // 4
+    test_repeated_failures_trigger_extra_decay();            // 5
+    test_best_route_selected_by_highest_score();             // 6
+    test_preferred_route_bias_applies();                     // 7
+    test_scores_are_clamped_to_one();                        // 8
+    test_scores_are_clamped_to_zero();                       // 9
+    test_adjusted_confidence_matches_recommended_route_score(); // 10
+    test_missing_attempts_uses_neutral_prior();              // 11
+    test_route_scores_include_all_policy_routes();           // 12
+
+    std::cout << "\nResults: " << passed << "/" << (passed + failed) << " passed\n";
+    return failed == 0 ? 0 : 1;
+}
--- a/progress.md
+++ b/progress.md
@@ -9433,3 +9433,40 @@ dependency noise for narrow operations.
 - `editor/src/ContextBundleMinimizer.h` within header-size limit (`88` <= `600`)
 - `editor/tests/step545_test.cpp` within test-file size guidance (`169` lines)
 - Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md`
+
+### Step 546: Confidence Calibration for Constrained Paths
+**Status:** PASS (12/12 tests)
+
+Implements confidence calibration for constrained routing by combining base
+legal-choice confidence with historical success/failure rates and explicit
+penalties for repeated post-apply failures.
+
+**Files added:**
+- `editor/src/ConstrainedConfidenceCalibrator.h` - confidence calibration module:
+  - route scoring for deterministic/constrained/general policy paths
+  - success-rate boosts and post-apply failure penalties
+  - extra decay for repeated post-apply failures
+  - preferred-route bias with post-bias score clamping
+  - best-route selection and adjusted-confidence emission
+- `editor/tests/step546_test.cpp` - 12 tests covering:
+  - neutral/no-history behavior
+  - success/failure rate calibration effects
+  - repeated-failure decay behavior
+  - preferred-route bias behavior
+  - clamping behavior at score bounds
+  - recommended-route and adjusted-confidence consistency
+  - route score coverage across all policy paths
+
+**Files modified:**
+- `editor/CMakeLists.txt` - `step546_test` target
+
+**Verification run:**
+- `cmake -S editor -B editor/build-native` - PASS
+- `cmake --build editor/build-native --target step546_test step545_test` - PASS
+- `./editor/build-native/step546_test` - PASS (12/12)
+- `./editor/build-native/step545_test` - PASS (12/12) regression coverage
+
+**Architecture gate check:**
+- `editor/src/ConstrainedConfidenceCalibrator.h` within header-size limit (`97` <= `600`)
+- `editor/tests/step546_test.cpp` within test-file size guidance (`143` lines)
+- Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md`