Sprint 003 findings + Sprint 004 pipeline decision audit plan

- split_prereq_op_dataset.py: extract accepted/rejected/schema-drift rows per gate
- gen_prereq_op_rsa_data.py: convert accepted rows to Fabricate TSV format
- 1000-run extracted dataset: 1297 accepted prereq_op rows, 24 schema-drift
- Key finding: needs_validate_intake always True; effective gate is binary
  (needs_resolve_dependencies 77%, needs_architect_review 20%)
- Key finding: run corpus text is template-driven (17 unique inputs) — not
  suitable for text-based training; RSA exists before ideal input is available
- ROADMAP: gate lifecycle framing — goal is choice of backend, not forced
  determinism; RSA operates on imperfect/mixed/incomplete input by design
- training_data_strategy.md: taskitem-as-self-contained-work-unit constraint
- SPRINT-003: updated with 1000-run results, text audit findings, design constraints
- SPRINT-004: pipeline decision audit — manual walkthrough of every decision
  from project description to generated artifact; execute next session

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

tools/gen_prereq_op_rsa_data.py

@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+"""Generate Fabricate-format TSV training data for the two prereq_op binary specialists.
+
+Source: semantic/prereq_op_dataset_1000/accepted.ndjson
+        (extracted from real whetstone_DSL taskitem run artifacts)
+
+Produces four files:
+  prereq_resolve_train.tsv   / prereq_resolve_eval.tsv
+  prereq_architect_train.tsv / prereq_architect_eval.tsv
+
+Each file uses Fabricate TSV format: label<TAB>0<TAB>text
+  label: 0 = no, 1 = yes
+  0:     unused hop field (always 0)
+  text:  composed from task context fields
+
+Text composition (per row):
+  title: <task title>
+  reasons: <reason1>; <reason2>; ...
+  accepts: <acceptance criterion text>; ...
+  constraints: <constraint text>; ...
+
+Usage:
+  python3 tools/gen_prereq_op_rsa_data.py \\
+      --input semantic/prereq_op_dataset_1000/accepted.ndjson \\
+      --output-dir semantic/prereq_op_tsv \\
+      --eval-frac 0.15 \\
+      --seed 42
+"""
+
+import argparse
+import json
+import random
+from pathlib import Path
+
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+
+
+def compose_text(ctx: dict) -> str:
+    parts = []
+
+    title = (ctx.get("title") or "").strip()
+    if title:
+        parts.append(f"title: {title}")
+
+    reasons = ctx.get("reasons") or []
+    if reasons:
+        parts.append("reasons: " + "; ".join(str(r).strip() for r in reasons if r))
+
+    reqs = ctx.get("normalized_requirements") or []
+    accepts = [r["normalizedText"] for r in reqs if r.get("kind") == "acceptance" and r.get("normalizedText")]
+    constraints = [r["normalizedText"] for r in reqs if r.get("kind") == "constraint" and r.get("normalizedText")]
+
+    if accepts:
+        parts.append("accepts: " + "; ".join(accepts))
+    if constraints:
+        parts.append("constraints: " + "; ".join(constraints))
+
+    return " | ".join(parts)
+
+
+def write_tsv(path: Path, rows: list[tuple[int, str]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as fh:
+        for label, text in rows:
+            fh.write(f"{label}\t0\t{text}\n")
+
+
+def label_summary(rows: list[tuple[int, str]]) -> dict:
+    pos = sum(1 for label, _ in rows if label == 1)
+    neg = len(rows) - pos
+    return {"total": len(rows), "pos": pos, "neg": neg, "pos_pct": round(pos / len(rows) * 100, 1)}
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "--input",
+        default="semantic/prereq_op_dataset_1000/accepted.ndjson",
+        help="Accepted prereq_op rows in NDJSON format.",
+    )
+    parser.add_argument(
+        "--output-dir",
+        default="semantic/prereq_op_tsv",
+        help="Directory for output TSV files.",
+    )
+    parser.add_argument(
+        "--eval-frac",
+        type=float,
+        default=0.15,
+        help="Fraction of rows to use for eval (default: 0.15).",
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=42,
+        help="Random seed for train/eval split.",
+    )
+    parser.add_argument(
+        "--max-rows",
+        type=int,
+        default=0,
+        help="Optional cap on input rows (0 = no limit).",
+    )
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    input_path = Path(args.input)
+    if not input_path.is_absolute():
+        input_path = (REPO_ROOT / input_path).resolve()
+    output_dir = Path(args.output_dir)
+    if not output_dir.is_absolute():
+        output_dir = (REPO_ROOT / output_dir).resolve()
+
+    rows: list[dict] = []
+    with input_path.open("r", encoding="utf-8") as fh:
+        for line in fh:
+            line = line.strip()
+            if not line:
+                continue
+            rows.append(json.loads(line))
+            if args.max_rows and len(rows) >= args.max_rows:
+                break
+
+    rng = random.Random(args.seed)
+    rng.shuffle(rows)
+
+    n_eval = max(1, int(len(rows) * args.eval_frac))
+    eval_rows = rows[:n_eval]
+    train_rows = rows[n_eval:]
+
+    def to_binary(split: list[dict], key: str) -> list[tuple[int, str]]:
+        result = []
+        for row in split:
+            label_val = row["decision_label"].get(key, False)
+            text = compose_text(row["decision_context"])
+            if not text:
+                continue
+            result.append((1 if label_val else 0, text))
+        return result
+
+    resolve_train = to_binary(train_rows, "needs_resolve_dependencies")
+    resolve_eval = to_binary(eval_rows, "needs_resolve_dependencies")
+    architect_train = to_binary(train_rows, "needs_architect_review")
+    architect_eval = to_binary(eval_rows, "needs_architect_review")
+
+    write_tsv(output_dir / "prereq_resolve_train.tsv", resolve_train)
+    write_tsv(output_dir / "prereq_resolve_eval.tsv", resolve_eval)
+    write_tsv(output_dir / "prereq_architect_train.tsv", architect_train)
+    write_tsv(output_dir / "prereq_architect_eval.tsv", architect_eval)
+
+    summary = {
+        "input": str(input_path),
+        "output_dir": str(output_dir),
+        "total_rows": len(rows),
+        "train_rows": len(train_rows),
+        "eval_rows": len(eval_rows),
+        "needs_resolve_dependencies": {
+            "train": label_summary(resolve_train),
+            "eval": label_summary(resolve_eval),
+        },
+        "needs_architect_review": {
+            "train": label_summary(architect_train),
+            "eval": label_summary(architect_eval),
+        },
+    }
+    print(json.dumps(summary, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())