bill/whetstone_DSL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Step 597: add guardrail drift monitor

Browse Source
This commit is contained in:
Bill
2026-02-17 11:41:50 -07:00
parent c9749d078a
commit 7ab17e653e
4 changed files with 254 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
editor/CMakeLists.txt
View File

@@ -4252,4 +4252,13 @@ target_link_libraries(step596_test PRIVATE
tree_sitter_javascript tree_sitter_typescript
tree_sitter_java tree_sitter_rust tree_sitter_go)
add_executable(step597_test tests/step597_test.cpp)
target_include_directories(step597_test PRIVATE src)
target_link_libraries(step597_test PRIVATE
nlohmann_json::nlohmann_json
unofficial::tree-sitter::tree-sitter
tree_sitter_python tree_sitter_cpp tree_sitter_elisp
tree_sitter_javascript tree_sitter_typescript
tree_sitter_java tree_sitter_rust tree_sitter_go)
# Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies)

58
editor/src/GuardrailDriftMonitor.h Normal file
View File

@@ -0,0 +1,58 @@
#pragma once
// Step 597: Guardrail Drift Monitor
#include "PolicyGuardrailCatalog.h"
#include <string>
#include <vector>
struct DriftObservation {
std::string operation;
std::string observedOutcome;
};
struct DriftFinding {
std::string operation;
std::string expectedPolicy;
std::string observedOutcome;
};
class GuardrailDriftMonitor {
public:
static std::vector<DriftFinding> detect(const PolicyGuardrailCatalog& catalog,
const std::vector<DriftObservation>& observations) {
std::vector<DriftFinding> findings;
for (const auto& o : observations) {
const auto p = catalog.evaluate(o.operation);
const std::string expected = decisionName(p.decision);
if (expected != o.observedOutcome) {
findings.push_back({o.operation, expected, o.observedOutcome});
}
}
return findings;
}
static int driftScore(const std::vector<DriftFinding>& findings, int totalObservations) {
if (totalObservations <= 0) return 0;
int score = static_cast<int>((100.0 * findings.size()) / totalObservations);
if (score > 100) score = 100;
return score;
}
static std::vector<std::string> remediationHints(const std::vector<DriftFinding>& findings) {
std::vector<std::string> hints;
for (const auto& f : findings) {
if (f.expectedPolicy == "deny") hints.push_back("enforce_hard_block:" + f.operation);
else if (f.expectedPolicy == "require_review") hints.push_back("restore_review_gate:" + f.operation);
else hints.push_back("align_allow_path:" + f.operation);
}
return hints;
}
private:
static std::string decisionName(GuardrailDecision d) {
if (d == GuardrailDecision::Allow) return "allow";
if (d == GuardrailDecision::Deny) return "deny";
return "require_review";
}
};

156
editor/tests/step597_test.cpp Normal file
View File

@@ -0,0 +1,156 @@
// Step 597: Guardrail Drift Monitor (12 tests)
#include "GuardrailDriftMonitor.h"
#include <iostream>
static int passed = 0, failed = 0;
#define TEST(name) { std::cout << " " << #name << "... "; }
#define PASS() { std::cout << "PASS\n"; ++passed; }
#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; }
#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {}
static PolicyGuardrailCatalog catalog() {
PolicyGuardrailCatalog c;
std::string error;
c.addRule({"allow-push", "git push", GuardrailDecision::Allow, "safe"}, &error);
c.addRule({"review-deploy", "deploy", GuardrailDecision::RequireReview, "review"}, &error);
c.addRule({"deny-rm", "rm -rf", GuardrailDecision::Deny, "deny"}, &error);
return c;
}
void test_detect_no_drift_when_outcomes_match() {
TEST(detect_no_drift_when_outcomes_match);
auto c = catalog();
std::vector<DriftObservation> obs = {
{"git push origin", "allow"},
{"deploy prod", "require_review"},
{"rm -rf /tmp/x", "deny"}
};
CHECK(GuardrailDriftMonitor::detect(c, obs).empty(), "no drift expected");
PASS();
}
void test_detect_drift_for_allow_mismatch() {
TEST(detect_drift_for_allow_mismatch);
auto c = catalog();
std::vector<DriftObservation> obs = {
{"git push origin", "require_review"}
};
auto f = GuardrailDriftMonitor::detect(c, obs);
CHECK(f.size() == 1, "one drift expected");
CHECK(f[0].expectedPolicy == "allow", "expected policy mismatch");
PASS();
}
void test_detect_drift_for_review_mismatch() {
TEST(detect_drift_for_review_mismatch);
auto c = catalog();
std::vector<DriftObservation> obs = {
{"deploy prod", "allow"}
};
auto f = GuardrailDriftMonitor::detect(c, obs);
CHECK(f.size() == 1, "one drift expected");
CHECK(f[0].expectedPolicy == "require_review", "expected policy mismatch");
PASS();
}
void test_detect_drift_for_deny_mismatch() {
TEST(detect_drift_for_deny_mismatch);
auto c = catalog();
std::vector<DriftObservation> obs = {
{"rm -rf /tmp/x", "allow"}
};
auto f = GuardrailDriftMonitor::detect(c, obs);
CHECK(f.size() == 1, "one drift expected");
CHECK(f[0].expectedPolicy == "deny", "expected policy mismatch");
PASS();
}
void test_default_policy_drift_detected() {
TEST(default_policy_drift_detected);
auto c = catalog();
std::vector<DriftObservation> obs = {
{"unknown op", "allow"}
};
auto f = GuardrailDriftMonitor::detect(c, obs);
CHECK(f.size() == 1, "default drift expected");
CHECK(f[0].expectedPolicy == "require_review", "default expected policy mismatch");
PASS();
}
void test_drift_score_zero_when_no_observations() {
TEST(drift_score_zero_when_no_observations);
CHECK(GuardrailDriftMonitor::driftScore({}, 0) == 0, "zero observations score should be zero");
PASS();
}
void test_drift_score_percentage() {
TEST(drift_score_percentage);
std::vector<DriftFinding> f(2);
CHECK(GuardrailDriftMonitor::driftScore(f, 8) == 25, "drift score should be 25");
PASS();
}
void test_drift_score_capped_at_100() {
TEST(drift_score_capped_at_100);
std::vector<DriftFinding> f(10);
CHECK(GuardrailDriftMonitor::driftScore(f, 5) == 100, "score cap mismatch");
PASS();
}
void test_remediation_hint_for_deny_drift() {
TEST(remediation_hint_for_deny_drift);
std::vector<DriftFinding> f = {{"rm -rf /tmp/x", "deny", "allow"}};
auto hints = GuardrailDriftMonitor::remediationHints(f);
CHECK(hints.size() == 1, "hint count mismatch");
CHECK(hints[0].find("enforce_hard_block:") == 0, "deny hint mismatch");
PASS();
}
void test_remediation_hint_for_review_drift() {
TEST(remediation_hint_for_review_drift);
std::vector<DriftFinding> f = {{"deploy prod", "require_review", "allow"}};
auto hints = GuardrailDriftMonitor::remediationHints(f);
CHECK(hints[0].find("restore_review_gate:") == 0, "review hint mismatch");
PASS();
}
void test_remediation_hint_for_allow_drift() {
TEST(remediation_hint_for_allow_drift);
std::vector<DriftFinding> f = {{"git push origin", "allow", "require_review"}};
auto hints = GuardrailDriftMonitor::remediationHints(f);
CHECK(hints[0].find("align_allow_path:") == 0, "allow hint mismatch");
PASS();
}
void test_multiple_findings_generate_multiple_hints() {
TEST(multiple_findings_generate_multiple_hints);
std::vector<DriftFinding> f = {
{"deploy prod", "require_review", "allow"},
{"rm -rf /tmp/x", "deny", "allow"}
};
auto hints = GuardrailDriftMonitor::remediationHints(f);
CHECK(hints.size() == 2, "two hints expected");
PASS();
}
int main() {
std::cout << "Step 597: Guardrail Drift Monitor\n";
test_detect_no_drift_when_outcomes_match(); // 1
test_detect_drift_for_allow_mismatch(); // 2
test_detect_drift_for_review_mismatch(); // 3
test_detect_drift_for_deny_mismatch(); // 4
test_default_policy_drift_detected(); // 5
test_drift_score_zero_when_no_observations(); // 6
test_drift_score_percentage(); // 7
test_drift_score_capped_at_100(); // 8
test_remediation_hint_for_deny_drift(); // 9
test_remediation_hint_for_review_drift(); // 10
test_remediation_hint_for_allow_drift(); // 11
test_multiple_findings_generate_multiple_hints();// 12
std::cout << "\nResults: " << passed << "/" << (passed + failed) << " passed\n";
return failed == 0 ? 0 : 1;
}

31
progress.md
View File

@@ -11440,3 +11440,34 @@ risk tracking across approval flows.
- `editor/src/EscalationAuditLedger.h` within header-size limit (`74` <= `600`)
- `editor/tests/step596_test.cpp` within test-file size guidance (`166` lines)
- Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md`
### Step 597: Guardrail Drift Monitor
**Status:** PASS (12/12 tests)
Implements drift monitoring between expected guardrail policy decisions and
observed runtime outcomes, with remediation hint generation.
**Files added:**
- `editor/src/GuardrailDriftMonitor.h` - drift monitor module:
- expected-vs-observed drift detection
- drift scoring based on observation coverage
- remediation hint generation by expected policy type
- `editor/tests/step597_test.cpp` - 12 tests covering:
- no-drift and per-policy drift detection behavior
- default-policy drift behavior
- drift score behavior (including cap)
- remediation hint behavior for allow/review/deny mismatches
**Files modified:**
- `editor/CMakeLists.txt` - `step597_test` target
**Verification run:**
- `cmake -S editor -B editor/build-native` - PASS
- `cmake --build editor/build-native --target step597_test step596_test` - PASS
- `./editor/build-native/step597_test` - PASS (12/12)
- `./editor/build-native/step596_test` - PASS (12/12) regression coverage
**Architecture gate check:**
- `editor/src/GuardrailDriftMonitor.h` within header-size limit (`58` <= `600`)
- `editor/tests/step597_test.cpp` within test-file size guidance (`156` lines)
- Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md`