Add semantic fallback gap audit and budget gate (sprints 236-237)

tools/mcp/analyze_semantic_fallback_gaps.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+import argparse
+import json
+from dataclasses import dataclass, asdict
+from pathlib import Path
+from typing import Dict, List
+
+
+@dataclass
+class FallbackRecord:
+    run_id: str
+    input_file: str
+    timestamp: str
+    mode: str
+    enabled: bool
+    fallback_applied: bool
+    skipped_reason: str
+    native_task_count: int
+    native_semantic_signal_count: int
+    expanded_task_count: int
+    gap_class: str
+    likely_root_cause: str
+    recommended_tools: List[str]
+    recommended_taskitem_constraints: List[str]
+
+
+def load_json(path: Path) -> Dict:
+    with path.open("r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def classify_root_cause(native_task_count: int, native_semantic_signal_count: int) -> str:
+    if native_task_count <= 2 and native_semantic_signal_count == 0:
+        return "native_decomposition_and_semantic_signal_deficit"
+    if native_task_count <= 2:
+        return "native_decomposition_too_shallow"
+    if native_semantic_signal_count == 0:
+        return "semantic_rationale_missing"
+    return "native_decomposition_sufficient"
+
+
+def classify_gap(expansion: Dict) -> str:
+    enabled = bool(expansion.get("enabled", False))
+    fallback_applied = bool(expansion.get("fallback_applied", False))
+    mode = str(expansion.get("mode", ""))
+    if not enabled:
+        return "semantic_expansion_disabled"
+    if fallback_applied:
+        return "semantic_fallback_triggered"
+    if mode == "fallback_only":
+        return "native_first_pass"
+    return "semantic_expansion_not_applied"
+
+
+def recommendations(gap_class: str, root_cause: str) -> Dict[str, List[str]]:
+    if gap_class == "semantic_fallback_triggered":
+        tools = [
+            "whetstone_architect_intake",
+            "whetstone_generate_taskitems",
+            "whetstone_validate_taskitem",
+            "whetstone_queue_ready",
+        ]
+        constraints = [
+            "min_native_task_count>=5",
+            "require_semantic_reason_tokens=true",
+            "require_execution_specificity_score>=85",
+            "require_deterministic_rollback_replay_contract=true",
+        ]
+        if root_cause == "semantic_rationale_missing":
+            tools.append("whetstone_validate_taskitem")
+            constraints.append("require_reason_contains_semantic_risk_contract_capability=true")
+        return {"tools": tools, "constraints": constraints}
+
+    if gap_class == "semantic_expansion_disabled":
+        return {
+            "tools": ["whetstone_generate_taskitems", "whetstone_queue_ready"],
+            "constraints": ["set_WSTONE_SEMANTIC_TASK_EXPANSION=1_for_complex_specs"],
+        }
+
+    return {
+        "tools": ["whetstone_validate_taskitem"],
+        "constraints": ["monitor_semantic_fallback_rate<=0.35"],
+    }
+
+
+def load_records(runs_root: Path, include_glob: str) -> List[FallbackRecord]:
+    records: List[FallbackRecord] = []
+    for summary_path in sorted(runs_root.glob(f"{include_glob}/00_summary.json")):
+        try:
+            summary = load_json(summary_path)
+        except Exception:
+            continue
+        expansion = summary.get("semantic_task_expansion") or {}
+        if not isinstance(expansion, dict):
+            continue
+
+        native_task_count = int(expansion.get("native_task_count", 0) or 0)
+        native_semantic_signal_count = int(expansion.get("native_semantic_signal_count", 0) or 0)
+        gap_class = classify_gap(expansion)
+        root_cause = classify_root_cause(native_task_count, native_semantic_signal_count)
+        rec = recommendations(gap_class, root_cause)
+
+        records.append(
+            FallbackRecord(
+                run_id=summary_path.parent.name,
+                input_file=str(summary.get("input_file", "")),
+                timestamp=str(summary.get("timestamp", "")),
+                mode=str(expansion.get("mode", "")),
+                enabled=bool(expansion.get("enabled", False)),
+                fallback_applied=bool(expansion.get("fallback_applied", False)),
+                skipped_reason=str(expansion.get("skipped_reason", "")),
+                native_task_count=native_task_count,
+                native_semantic_signal_count=native_semantic_signal_count,
+                expanded_task_count=int(expansion.get("expanded_task_count", 0) or 0),
+                gap_class=gap_class,
+                likely_root_cause=root_cause,
+                recommended_tools=rec["tools"],
+                recommended_taskitem_constraints=rec["constraints"],
+            )
+        )
+
+    return records
+
+
+def write_json(path: Path, obj: Dict) -> None:
+    with path.open("w", encoding="utf-8") as f:
+        json.dump(obj, f, indent=2, sort_keys=True)
+        f.write("\n")
+
+
+def write_jsonl(path: Path, rows: List[Dict]) -> None:
+    with path.open("w", encoding="utf-8") as f:
+        for row in rows:
+            f.write(json.dumps(row, sort_keys=True))
+            f.write("\n")
+
+
+def summarize(records: List[FallbackRecord]) -> Dict:
+    total = len(records)
+    fallback_count = sum(1 for r in records if r.fallback_applied)
+    enabled_count = sum(1 for r in records if r.enabled)
+    by_gap: Dict[str, int] = {}
+    by_root: Dict[str, int] = {}
+    tool_frequency: Dict[str, int] = {}
+
+    for r in records:
+        by_gap[r.gap_class] = by_gap.get(r.gap_class, 0) + 1
+        by_root[r.likely_root_cause] = by_root.get(r.likely_root_cause, 0) + 1
+        for t in r.recommended_tools:
+            tool_frequency[t] = tool_frequency.get(t, 0) + 1
+
+    fallback_rate = (fallback_count / enabled_count) if enabled_count else 0.0
+    return {
+        "record_count": total,
+        "enabled_count": enabled_count,
+        "fallback_applied_count": fallback_count,
+        "fallback_rate": round(fallback_rate, 4),
+        "gap_class_counts": by_gap,
+        "root_cause_counts": by_root,
+        "recommended_tool_frequency": tool_frequency,
+        "status": "ok" if total > 0 else "no_data",
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Analyze semantic fallback usage and emit remediation metadata.")
+    parser.add_argument("--runs-root", default="logs/taskitem_runs", help="Run root containing per-run 00_summary.json files")
+    parser.add_argument("--include-glob", default="*", help="Folder glob under runs-root to include")
+    parser.add_argument("--out-dir", required=True, help="Output directory for audit artifacts")
+    args = parser.parse_args()
+
+    runs_root = Path(args.runs_root)
+    out_dir = Path(args.out_dir)
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    records = load_records(runs_root, args.include_glob)
+    rows = [asdict(r) for r in records]
+    write_jsonl(out_dir / "semantic_fallback_records.jsonl", rows)
+
+    summary = summarize(records)
+    write_json(out_dir / "semantic_fallback_summary.json", summary)
+
+    recommendations_payload = {
+        "top_recommendations": sorted(
+            summary.get("recommended_tool_frequency", {}).items(),
+            key=lambda kv: (-int(kv[1]), str(kv[0])),
+        ),
+        "records": rows,
+    }
+    write_json(out_dir / "semantic_fallback_tooling_recommendations.json", recommendations_payload)
+
+    print(
+        json.dumps(
+            {
+                "status": summary.get("status", "no_data"),
+                "record_count": summary.get("record_count", 0),
+                "fallback_rate": summary.get("fallback_rate", 0.0),
+                "out_dir": str(out_dir),
+            },
+            sort_keys=True,
+        )
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/mcp/check_semantic_fallback_budget.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+import argparse
+import json
+from pathlib import Path
+from typing import Dict
+
+
+def load_json(path: Path) -> Dict:
+    with path.open("r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def write_json(path: Path, obj: Dict) -> None:
+    with path.open("w", encoding="utf-8") as f:
+        json.dump(obj, f, indent=2, sort_keys=True)
+        f.write("\n")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Check semantic fallback usage against a configured budget.")
+    parser.add_argument("--summary", required=True, help="Path to semantic_fallback_summary.json")
+    parser.add_argument("--out", required=True, help="Output gate evaluation JSON")
+    parser.add_argument("--max-fallback-rate", type=float, default=0.35, help="Max allowed fallback rate in [0,1]")
+    parser.add_argument("--min-record-count", type=int, default=5, help="Minimum records required before strict gating")
+    args = parser.parse_args()
+
+    summary = load_json(Path(args.summary))
+    fallback_rate = float(summary.get("fallback_rate", 0.0) or 0.0)
+    record_count = int(summary.get("record_count", 0) or 0)
+
+    enforceable = record_count >= args.min_record_count
+    passed = True
+    reason = "insufficient_data"
+    if enforceable:
+        passed = fallback_rate <= args.max_fallback_rate
+        reason = "within_budget" if passed else "fallback_rate_exceeds_budget"
+
+    result = {
+        "passed": bool(passed),
+        "reason": reason,
+        "enforceable": bool(enforceable),
+        "record_count": record_count,
+        "min_record_count": int(args.min_record_count),
+        "fallback_rate": fallback_rate,
+        "max_fallback_rate": float(args.max_fallback_rate),
+    }
+    write_json(Path(args.out), result)
+
+    print(json.dumps(result, sort_keys=True))
+    return 0 if passed else 9
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/mcp/run_semantic_fallback_budget_gate.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+RUNS_ROOT="${WSTONE_RUNS_ROOT:-$ROOT_DIR/logs/taskitem_runs}"
+INCLUDE_GLOB="${WSTONE_SEMANTIC_FALLBACK_INCLUDE_GLOB:-*}"
+OUT_DIR="${1:-$RUNS_ROOT/TEST_ONLY_semantic_fallback_budget_gate_$(date +%Y%m%d_%H%M%S)}"
+MAX_RATE="${WSTONE_SEMANTIC_MAX_FALLBACK_RATE:-0.35}"
+MIN_RECORDS="${WSTONE_SEMANTIC_FALLBACK_MIN_RECORDS:-5}"
+
+mkdir -p "$OUT_DIR"
+
+python3 "$ROOT_DIR/tools/mcp/analyze_semantic_fallback_gaps.py" \
+  --runs-root "$RUNS_ROOT" \
+  --include-glob "$INCLUDE_GLOB" \
+  --out-dir "$OUT_DIR" >/dev/null
+
+if python3 "$ROOT_DIR/tools/mcp/check_semantic_fallback_budget.py" \
+  --summary "$OUT_DIR/semantic_fallback_summary.json" \
+  --out "$OUT_DIR/semantic_fallback_budget_gate.json" \
+  --max-fallback-rate "$MAX_RATE" \
+  --min-record-count "$MIN_RECORDS" >/dev/null; then
+  jq -n \
+    --arg out_dir "$OUT_DIR" \
+    --argjson gate "$(cat "$OUT_DIR/semantic_fallback_budget_gate.json")" \
+    '{status:"pass", out_dir:$out_dir, gate:$gate}'
+else
+  jq -n \
+    --arg out_dir "$OUT_DIR" \
+    --argjson gate "$(cat "$OUT_DIR/semantic_fallback_budget_gate.json")" \
+    '{status:"fail", out_dir:$out_dir, gate:$gate}'
+  exit 9
+fi