From 7ab17e653ed62c0b39ff04f884536c7757ca86b7 Mon Sep 17 00:00:00 2001 From: Bill Date: Tue, 17 Feb 2026 11:41:50 -0700 Subject: [PATCH] Step 597: add guardrail drift monitor --- editor/CMakeLists.txt | 9 ++ editor/src/GuardrailDriftMonitor.h | 58 +++++++++++ editor/tests/step597_test.cpp | 156 +++++++++++++++++++++++++++++ progress.md | 31 ++++++ 4 files changed, 254 insertions(+) create mode 100644 editor/src/GuardrailDriftMonitor.h create mode 100644 editor/tests/step597_test.cpp diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 578dba6..6c0dc1e 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -4252,4 +4252,13 @@ target_link_libraries(step596_test PRIVATE tree_sitter_javascript tree_sitter_typescript tree_sitter_java tree_sitter_rust tree_sitter_go) +add_executable(step597_test tests/step597_test.cpp) +target_include_directories(step597_test PRIVATE src) +target_link_libraries(step597_test PRIVATE + nlohmann_json::nlohmann_json + unofficial::tree-sitter::tree-sitter + tree_sitter_python tree_sitter_cpp tree_sitter_elisp + tree_sitter_javascript tree_sitter_typescript + tree_sitter_java tree_sitter_rust tree_sitter_go) + # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies) diff --git a/editor/src/GuardrailDriftMonitor.h b/editor/src/GuardrailDriftMonitor.h new file mode 100644 index 0000000..175cc9a --- /dev/null +++ b/editor/src/GuardrailDriftMonitor.h @@ -0,0 +1,58 @@ +#pragma once +// Step 597: Guardrail Drift Monitor + +#include "PolicyGuardrailCatalog.h" + +#include +#include + +struct DriftObservation { + std::string operation; + std::string observedOutcome; +}; + +struct DriftFinding { + std::string operation; + std::string expectedPolicy; + std::string observedOutcome; +}; + +class GuardrailDriftMonitor { +public: + static std::vector detect(const PolicyGuardrailCatalog& catalog, + const std::vector& observations) { + std::vector findings; + for (const auto& o : observations) { + const auto p = catalog.evaluate(o.operation); + const std::string expected = decisionName(p.decision); + if (expected != o.observedOutcome) { + findings.push_back({o.operation, expected, o.observedOutcome}); + } + } + return findings; + } + + static int driftScore(const std::vector& findings, int totalObservations) { + if (totalObservations <= 0) return 0; + int score = static_cast((100.0 * findings.size()) / totalObservations); + if (score > 100) score = 100; + return score; + } + + static std::vector remediationHints(const std::vector& findings) { + std::vector hints; + for (const auto& f : findings) { + if (f.expectedPolicy == "deny") hints.push_back("enforce_hard_block:" + f.operation); + else if (f.expectedPolicy == "require_review") hints.push_back("restore_review_gate:" + f.operation); + else hints.push_back("align_allow_path:" + f.operation); + } + return hints; + } + +private: + static std::string decisionName(GuardrailDecision d) { + if (d == GuardrailDecision::Allow) return "allow"; + if (d == GuardrailDecision::Deny) return "deny"; + return "require_review"; + } +}; diff --git a/editor/tests/step597_test.cpp b/editor/tests/step597_test.cpp new file mode 100644 index 0000000..782fc3c --- /dev/null +++ b/editor/tests/step597_test.cpp @@ -0,0 +1,156 @@ +// Step 597: Guardrail Drift Monitor (12 tests) + +#include "GuardrailDriftMonitor.h" + +#include + +static int passed = 0, failed = 0; +#define TEST(name) { std::cout << " " << #name << "... "; } +#define PASS() { std::cout << "PASS\n"; ++passed; } +#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; } +#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {} + +static PolicyGuardrailCatalog catalog() { + PolicyGuardrailCatalog c; + std::string error; + c.addRule({"allow-push", "git push", GuardrailDecision::Allow, "safe"}, &error); + c.addRule({"review-deploy", "deploy", GuardrailDecision::RequireReview, "review"}, &error); + c.addRule({"deny-rm", "rm -rf", GuardrailDecision::Deny, "deny"}, &error); + return c; +} + +void test_detect_no_drift_when_outcomes_match() { + TEST(detect_no_drift_when_outcomes_match); + auto c = catalog(); + std::vector obs = { + {"git push origin", "allow"}, + {"deploy prod", "require_review"}, + {"rm -rf /tmp/x", "deny"} + }; + CHECK(GuardrailDriftMonitor::detect(c, obs).empty(), "no drift expected"); + PASS(); +} + +void test_detect_drift_for_allow_mismatch() { + TEST(detect_drift_for_allow_mismatch); + auto c = catalog(); + std::vector obs = { + {"git push origin", "require_review"} + }; + auto f = GuardrailDriftMonitor::detect(c, obs); + CHECK(f.size() == 1, "one drift expected"); + CHECK(f[0].expectedPolicy == "allow", "expected policy mismatch"); + PASS(); +} + +void test_detect_drift_for_review_mismatch() { + TEST(detect_drift_for_review_mismatch); + auto c = catalog(); + std::vector obs = { + {"deploy prod", "allow"} + }; + auto f = GuardrailDriftMonitor::detect(c, obs); + CHECK(f.size() == 1, "one drift expected"); + CHECK(f[0].expectedPolicy == "require_review", "expected policy mismatch"); + PASS(); +} + +void test_detect_drift_for_deny_mismatch() { + TEST(detect_drift_for_deny_mismatch); + auto c = catalog(); + std::vector obs = { + {"rm -rf /tmp/x", "allow"} + }; + auto f = GuardrailDriftMonitor::detect(c, obs); + CHECK(f.size() == 1, "one drift expected"); + CHECK(f[0].expectedPolicy == "deny", "expected policy mismatch"); + PASS(); +} + +void test_default_policy_drift_detected() { + TEST(default_policy_drift_detected); + auto c = catalog(); + std::vector obs = { + {"unknown op", "allow"} + }; + auto f = GuardrailDriftMonitor::detect(c, obs); + CHECK(f.size() == 1, "default drift expected"); + CHECK(f[0].expectedPolicy == "require_review", "default expected policy mismatch"); + PASS(); +} + +void test_drift_score_zero_when_no_observations() { + TEST(drift_score_zero_when_no_observations); + CHECK(GuardrailDriftMonitor::driftScore({}, 0) == 0, "zero observations score should be zero"); + PASS(); +} + +void test_drift_score_percentage() { + TEST(drift_score_percentage); + std::vector f(2); + CHECK(GuardrailDriftMonitor::driftScore(f, 8) == 25, "drift score should be 25"); + PASS(); +} + +void test_drift_score_capped_at_100() { + TEST(drift_score_capped_at_100); + std::vector f(10); + CHECK(GuardrailDriftMonitor::driftScore(f, 5) == 100, "score cap mismatch"); + PASS(); +} + +void test_remediation_hint_for_deny_drift() { + TEST(remediation_hint_for_deny_drift); + std::vector f = {{"rm -rf /tmp/x", "deny", "allow"}}; + auto hints = GuardrailDriftMonitor::remediationHints(f); + CHECK(hints.size() == 1, "hint count mismatch"); + CHECK(hints[0].find("enforce_hard_block:") == 0, "deny hint mismatch"); + PASS(); +} + +void test_remediation_hint_for_review_drift() { + TEST(remediation_hint_for_review_drift); + std::vector f = {{"deploy prod", "require_review", "allow"}}; + auto hints = GuardrailDriftMonitor::remediationHints(f); + CHECK(hints[0].find("restore_review_gate:") == 0, "review hint mismatch"); + PASS(); +} + +void test_remediation_hint_for_allow_drift() { + TEST(remediation_hint_for_allow_drift); + std::vector f = {{"git push origin", "allow", "require_review"}}; + auto hints = GuardrailDriftMonitor::remediationHints(f); + CHECK(hints[0].find("align_allow_path:") == 0, "allow hint mismatch"); + PASS(); +} + +void test_multiple_findings_generate_multiple_hints() { + TEST(multiple_findings_generate_multiple_hints); + std::vector f = { + {"deploy prod", "require_review", "allow"}, + {"rm -rf /tmp/x", "deny", "allow"} + }; + auto hints = GuardrailDriftMonitor::remediationHints(f); + CHECK(hints.size() == 2, "two hints expected"); + PASS(); +} + +int main() { + std::cout << "Step 597: Guardrail Drift Monitor\n"; + + test_detect_no_drift_when_outcomes_match(); // 1 + test_detect_drift_for_allow_mismatch(); // 2 + test_detect_drift_for_review_mismatch(); // 3 + test_detect_drift_for_deny_mismatch(); // 4 + test_default_policy_drift_detected(); // 5 + test_drift_score_zero_when_no_observations(); // 6 + test_drift_score_percentage(); // 7 + test_drift_score_capped_at_100(); // 8 + test_remediation_hint_for_deny_drift(); // 9 + test_remediation_hint_for_review_drift(); // 10 + test_remediation_hint_for_allow_drift(); // 11 + test_multiple_findings_generate_multiple_hints();// 12 + + std::cout << "\nResults: " << passed << "/" << (passed + failed) << " passed\n"; + return failed == 0 ? 0 : 1; +} diff --git a/progress.md b/progress.md index f16a38f..e6acc1e 100644 --- a/progress.md +++ b/progress.md @@ -11440,3 +11440,34 @@ risk tracking across approval flows. - `editor/src/EscalationAuditLedger.h` within header-size limit (`74` <= `600`) - `editor/tests/step596_test.cpp` within test-file size guidance (`166` lines) - Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md` + +### Step 597: Guardrail Drift Monitor +**Status:** PASS (12/12 tests) + +Implements drift monitoring between expected guardrail policy decisions and +observed runtime outcomes, with remediation hint generation. + +**Files added:** +- `editor/src/GuardrailDriftMonitor.h` - drift monitor module: + - expected-vs-observed drift detection + - drift scoring based on observation coverage + - remediation hint generation by expected policy type +- `editor/tests/step597_test.cpp` - 12 tests covering: + - no-drift and per-policy drift detection behavior + - default-policy drift behavior + - drift score behavior (including cap) + - remediation hint behavior for allow/review/deny mismatches + +**Files modified:** +- `editor/CMakeLists.txt` - `step597_test` target + +**Verification run:** +- `cmake -S editor -B editor/build-native` - PASS +- `cmake --build editor/build-native --target step597_test step596_test` - PASS +- `./editor/build-native/step597_test` - PASS (12/12) +- `./editor/build-native/step596_test` - PASS (12/12) regression coverage + +**Architecture gate check:** +- `editor/src/GuardrailDriftMonitor.h` within header-size limit (`58` <= `600`) +- `editor/tests/step597_test.cpp` within test-file size guidance (`156` lines) +- Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md`