whetstone_DSL/tools/mcp/analyze_taskitem_calibration.py

#!/usr/bin/env python3
import argparse
import json
from dataclasses import dataclass, asdict
from pathlib import Path
from typing import Dict, List, Tuple


@dataclass
class CalibrationRecord:
    run_id: str
    sprint: str
    timestamp: str
    strict_execution_contract: bool
    queue_ready: bool
    validation_failing_count: int
    blocker_count: int
    average_execution_specificity_score: float
    average_score: float
    escalate_count: int
    dominant_gap_class: str
    actual_ready: bool


def load_json(path: Path) -> Dict:
    with path.open("r", encoding="utf-8") as f:
        return json.load(f)


def dominant_gap_class(summary: Dict) -> str:
    queue_hints = (summary.get("queue_ready") or {}).get("remediation_routing_hints") or {}
    gap = queue_hints.get("dominant_gap_class", "")
    if isinstance(gap, str) and gap:
        return gap

    gap_counts = (summary.get("validation") or {}).get("gap_class_counts") or {}
    if not isinstance(gap_counts, dict) or not gap_counts:
        return "none"

    best_key = "none"
    best_count = -1
    for key in sorted(gap_counts.keys()):
        val = gap_counts.get(key, 0)
        if isinstance(val, (int, float)) and val > best_count:
            best_key = key
            best_count = int(val)
    return best_key


def load_records(runs_root: Path) -> List[CalibrationRecord]:
    records: List[CalibrationRecord] = []
    for summary_path in sorted(runs_root.glob("*/00_summary.json")):
        try:
            summary = load_json(summary_path)
        except Exception:
            continue

        queue = summary.get("queue_ready") or {}
        validation = summary.get("validation") or {}
        taskitems = summary.get("taskitems") or {}

        queue_ready = bool(queue.get("ready", False))
        failing_count = int(validation.get("failing_count", 0) or 0)
        actual_ready = queue_ready and failing_count == 0

        records.append(
            CalibrationRecord(
                run_id=summary_path.parent.name,
                sprint=str(summary.get("sprint", "")),
                timestamp=str(summary.get("timestamp", "")),
                strict_execution_contract=bool(queue.get("strict_execution_contract", taskitems.get("strict_execution_contract", False))),
                queue_ready=queue_ready,
                validation_failing_count=failing_count,
                blocker_count=int(queue.get("blocker_count", 0) or 0),
                average_execution_specificity_score=float(validation.get("average_execution_specificity_score", 0.0) or 0.0),
                average_score=float(validation.get("average_score", 0.0) or 0.0),
                escalate_count=int(taskitems.get("escalate_count", 0) or 0),
                dominant_gap_class=dominant_gap_class(summary),
                actual_ready=actual_ready,
            )
        )
    return records


def confusion(records: List[CalibrationRecord], threshold: int) -> Dict[str, int]:
    tp = fp = tn = fn = 0
    for r in records:
        pred_ready = r.average_execution_specificity_score >= threshold
        if pred_ready and r.actual_ready:
            tp += 1
        elif pred_ready and not r.actual_ready:
            fp += 1
        elif not pred_ready and not r.actual_ready:
            tn += 1
        else:
            fn += 1
    return {"tp": tp, "fp": fp, "tn": tn, "fn": fn}


def pick_threshold(records: List[CalibrationRecord]) -> Tuple[int, Dict[str, int]]:
    best_t = 60
    best = confusion(records, best_t)

    def rank(c: Dict[str, int]) -> Tuple[int, int, int]:
        # In strict mode calibration, false-ready is more expensive than false-blocked.
        errors = (c["fp"] * 4) + c["fn"]
        balance = abs(c["fp"] - c["fn"])
        correct = c["tp"] + c["tn"]
        return (errors, balance, -correct)

    best_rank = rank(best)
    for t in range(0, 101):
        c = confusion(records, t)
        r = rank(c)
        if r < best_rank or (r == best_rank and t > best_t):
            best_t = t
            best = c
            best_rank = r
    return best_t, best


def summarize_distribution(records: List[CalibrationRecord]) -> Dict:
    bins: Dict[str, Dict[str, int]] = {}
    for floor in range(0, 101, 10):
        bins[f"{floor:02d}-{min(99, floor+9):02d}"] = {"total": 0, "ready": 0, "blocked": 0}

    for r in records:
        score = max(0, min(100, int(r.average_execution_specificity_score)))
        floor = (score // 10) * 10
        key = f"{floor:02d}-{min(99, floor+9):02d}"
        row = bins[key]
        row["total"] += 1
        if r.actual_ready:
            row["ready"] += 1
        else:
            row["blocked"] += 1

    return {
        "record_count": len(records),
        "bins": bins,
    }


def per_gap_thresholds(records: List[CalibrationRecord]) -> Dict[str, Dict]:
    grouped: Dict[str, List[CalibrationRecord]] = {}
    for r in records:
        grouped.setdefault(r.dominant_gap_class, []).append(r)

    out: Dict[str, Dict] = {}
    for gap in sorted(grouped.keys()):
        subset = grouped[gap]
        if len(subset) < 3:
            out[gap] = {
                "record_count": len(subset),
                "recommended_threshold": 60,
                "insufficient_data": True,
            }
            continue
        threshold, c = pick_threshold(subset)
        out[gap] = {
            "record_count": len(subset),
            "recommended_threshold": threshold,
            "confusion": c,
            "insufficient_data": False,
        }
    return out


def misclassifications(records: List[CalibrationRecord], threshold: int) -> List[Dict]:
    rows: List[Dict] = []
    for r in records:
        pred_ready = r.average_execution_specificity_score >= threshold
        if pred_ready == r.actual_ready:
            continue
        row = asdict(r)
        row["pred_ready"] = pred_ready
        row["reason"] = "false_positive" if pred_ready else "false_negative"
        rows.append(row)
    rows.sort(key=lambda x: (x["reason"], x["run_id"]))
    return rows


def write_json(path: Path, obj: Dict) -> None:
    with path.open("w", encoding="utf-8") as f:
        json.dump(obj, f, indent=2, sort_keys=True)
        f.write("\n")


def write_jsonl(path: Path, rows: List[Dict]) -> None:
    with path.open("w", encoding="utf-8") as f:
        for row in rows:
            f.write(json.dumps(row, sort_keys=True))
            f.write("\n")


def main() -> int:
    parser = argparse.ArgumentParser(description="Analyze taskitem calibration from pipeline summaries.")
    parser.add_argument("--runs-root", default="logs/taskitem_runs", help="Directory containing run output folders with 00_summary.json")
    parser.add_argument("--out-dir", required=True, help="Directory for calibration outputs")
    args = parser.parse_args()

    runs_root = Path(args.runs_root)
    out_dir = Path(args.out_dir)
    out_dir.mkdir(parents=True, exist_ok=True)

    records = load_records(runs_root)
    rows = [asdict(r) for r in records]
    write_jsonl(out_dir / "calibration_records.jsonl", rows)

    if not records:
        summary = {
            "record_count": 0,
            "recommended_threshold": 60,
            "confusion": {"tp": 0, "fp": 0, "tn": 0, "fn": 0},
            "per_gap_thresholds": {},
            "misclassification_count": 0,
            "status": "insufficient_data",
        }
        write_json(out_dir / "score_distribution.json", {"record_count": 0, "bins": {}})
        write_json(out_dir / "threshold_recommendations.json", summary)
        write_json(out_dir / "misclassification_report.json", {"records": []})
        print(json.dumps({
            "status": "insufficient_data",
            "out_dir": str(out_dir),
            "record_count": 0,
            "recommended_threshold": 60,
            "misclassification_count": 0,
        }))
        return 0

    strict_records = [r for r in records if r.strict_execution_contract]
    if len(strict_records) >= 10:
        scoped_records = strict_records
        calibration_scope = "strict_execution_contract_only"
    else:
        scoped_records = records
        calibration_scope = "all_runs_fallback"

    threshold, c = pick_threshold(scoped_records)
    if threshold < 60:
        threshold = 60
        c = confusion(scoped_records, threshold)
    dist = summarize_distribution(scoped_records)
    per_gap = per_gap_thresholds(scoped_records)
    misses = misclassifications(scoped_records, threshold)

    write_json(out_dir / "score_distribution.json", dist)
    write_json(out_dir / "threshold_recommendations.json", {
        "record_count": len(scoped_records),
        "calibration_scope": calibration_scope,
        "recommended_threshold": threshold,
        "confusion": c,
        "per_gap_thresholds": per_gap,
        "status": "ok",
    })
    write_json(out_dir / "misclassification_report.json", {
        "record_count": len(scoped_records),
        "calibration_scope": calibration_scope,
        "threshold": threshold,
        "misclassification_count": len(misses),
        "records": misses,
    })

    print(json.dumps({
        "status": "ok",
        "out_dir": str(out_dir),
        "record_count": len(scoped_records),
        "calibration_scope": calibration_scope,
        "recommended_threshold": threshold,
        "misclassification_count": len(misses),
        "confusion": c,
    }))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())