#!/usr/bin/env python3 """ gen_from_projects.py Runs all project benchmark specs through the whetstone_mcp pipeline (architect_intake → generate_taskitems) and extracts labeled training examples for each specialist from the real pipeline output. Why this matters: - Training data from sprint plans uses step description text. - Specialists run at inference time on taskitem TITLES + INTENT. - These are similar but not identical formats. Running real specs through the pipeline produces training examples in the exact inference-time format. - The pipeline output is the "correct answer" — workerType from inferWorkerType, prerequisiteOps from inferPrerequisiteOps, verificationType from the contract. Input files: datasets/project_benchmarks/common_projects_100.jsonl (100 projects) datasets/project_benchmarks/challenging_*.jsonl (25 projects) Output TSVs (appended to existing data or written fresh): specialists/data/generated/prereq_op_projects_train.tsv specialists/data/generated/prereq_op_projects_eval.tsv specialists/data/generated/worker_type_projects_train.tsv specialists/data/generated/worker_type_projects_eval.tsv specialists/data/generated/verification_type_projects_train.tsv specialists/data/generated/verification_type_projects_eval.tsv Text format for all specialists: " | <intent[:120]>" Using pipe separator so title and intent are both visible but the model sees them as one continuous text sequence. Usage: python3 specialists/scripts/gen_from_projects.py \\ --whetstone-bin editor/build-native/whetstone_mcp \\ --workspace . """ import argparse import json import random import subprocess import sys from collections import Counter from pathlib import Path # --------------------------------------------------------------------------- # Label maps # --------------------------------------------------------------------------- # Text-based prereq classifier — same tokens as gen_prereq_op_data.py so that # project-spec task text is labeled consistently with the sprint-plan data. # Using text rather than the C++ heuristic's prerequisiteOps because the heuristic # never emits "manual-approval" (circular labels = all "standard"). _PREREQ_REVIEW_TOKENS = { "introduce", "phase", "framework", "architecture", "design", "refactor", "planning", "taxonomy", "grammar", "interface", "api surface", "subsystem", "extension point", "abstraction", "decompos", "scaffold", "polyglot", "cross-language", "cross-project", "multi-language", "orchestrat", "migration plan", "phase plan", "protocol design", "new layer", "foundation", "bootstrap", "initial", "phase 1", "phase 2", "phase 3", "sprint plan", "roadmap", } _PREREQ_APPROVAL_TOKENS = { "release", "deploy", "publish", "migrate", "security", "compliance", "upgrade", "breaking change", "production", "rollout", "finalize", "sign off", "certification", "approval", "cutover", "go-live", "data migration", "schema migration", "breaking", "deprecat", "audit", "regulatory", "gdpr", "pii", "secret", "credential", "vault", "access control", "rbac", "acl", "permission", "fraud", "encryption", } def prereq_ops_to_class(ops: list[str]) -> str: """Legacy: classify by actual ops list (unused — text_to_prereq_class is preferred).""" s = set(ops) has_review = "architect-review" in s has_approval = "manual-approval" in s if has_review and has_approval: return "full_gates" if has_review: return "needs_review" if has_approval: return "needs_approval" return "standard" def text_to_prereq_class(text: str) -> str: """Label prereq class from task title+intent text (same signal the specialist sees).""" t = text.lower() has_review = any(tok in t for tok in _PREREQ_REVIEW_TOKENS) has_approval = any(tok in t for tok in _PREREQ_APPROVAL_TOKENS) if has_review and has_approval: return "full_gates" if has_review: return "needs_review" if has_approval: return "needs_approval" return "standard" PREREQ_LABEL_INDEX = {"standard": 0, "needs_review": 1, "needs_approval": 2, "full_gates": 3} WORKER_LABEL_INDEX = {"implementer": 0, "reviewer": 1, "architect": 2, "qa": 3} VERIF_LABEL_INDEX = {"unit": 0, "integration": 1, "schema": 2, "smoke": 3, "docs": 4} # --------------------------------------------------------------------------- # MCP pipeline helpers # --------------------------------------------------------------------------- def mcp_call(binary: str, workspace: str, tool: str, args: dict) -> dict: payload = json.dumps({ "jsonrpc": "2.0", "id": 1, "method": "tools/call", "params": {"name": tool, "arguments": args} }) try: r = subprocess.run( [binary, "--workspace", workspace, "--language", "cpp"], input=payload, capture_output=True, text=True, timeout=20 ) content = (json.loads(r.stdout) .get("result", {}) .get("content", [{}])[0] .get("text", "")) return json.loads(content) if content else {} except Exception as e: print(f" [WARN] mcp_call {tool}: {e}", file=sys.stderr) return {} def run_pipeline(binary: str, workspace: str, spec: str) -> list[dict]: """Run architect_intake → generate_taskitems for one spec. Return task list.""" intake = mcp_call(binary, workspace, "whetstone_architect_intake", {"markdown": spec}) reqs = intake.get("normalizedRequirements", []) if not reqs: return [] result = mcp_call(binary, workspace, "whetstone_generate_taskitems", { "normalizedRequirements": reqs, "conflicts": intake.get("conflicts", []) }) return result.get("tasks", []) def task_text(task: dict) -> str: """Combine title + intent for specialist input.""" title = task.get("title", "").strip() intent = task.get("intent", "").strip()[:120] if intent and intent != title: return f"{title} | {intent}" return title # --------------------------------------------------------------------------- # Load project specs # --------------------------------------------------------------------------- def load_projects(dataset_dir: Path) -> list[dict]: projects = [] # Common 100 common = dataset_dir / "common_projects_100.jsonl" if common.exists(): with open(common) as f: for line in f: projects.append(json.loads(line)) # All challenging sets for path in sorted(dataset_dir.glob("challenging_*.jsonl")): with open(path) as f: for line in f: p = json.loads(line) if p.get("spec"): projects.append(p) print(f"Loaded {len(projects)} project specs") return projects def spec_markdown(project: dict) -> str: """Wrap a project spec into the sections architect_intake expects.""" return ( f"## Goals\n{project['spec']}\n\n" f"## Constraints\n- Production quality\n- Well-tested\n\n" f"## Acceptance Criteria\n- All core features implemented\n" f"- Tests passing\n" ) # --------------------------------------------------------------------------- # Balance + write TSV # --------------------------------------------------------------------------- def balance_and_split(examples: list[tuple[str, str]], label_index: dict[str, int], label_order: list[str], seed: int = 42, eval_frac: float = 0.15, min_per_class: int = 20) -> tuple[list, list]: """Return (train_rows, eval_rows) as list of (label_str, text) pairs.""" by_class: dict[str, list[str]] = {} for lbl, text in examples: by_class.setdefault(lbl, []).append(text) counts = {lbl: len(items) for lbl, items in by_class.items()} print(f" Raw class distribution: {counts}") if not counts: return [], [] sorted_vals = sorted(counts.values()) median = sorted_vals[len(sorted_vals) // 2] target = max(median, min_per_class) print(f" Balancing to {target} per class") rng = random.Random(seed) balanced: list[tuple[str, str]] = [] for lbl in label_order: items = by_class.get(lbl, []) if not items: print(f" WARNING: no examples for '{lbl}'", file=sys.stderr) continue rng.shuffle(items) chosen = items[:target] while len(chosen) < target: chosen += items balanced.extend((lbl, t) for t in chosen[:target]) rng.shuffle(balanced) n_eval = max(1, int(len(balanced) * eval_frac)) return balanced[n_eval:], balanced[:n_eval] def write_tsv(path: Path, rows: list[tuple[str, str]], label_index: dict[str, int]) -> None: path.parent.mkdir(parents=True, exist_ok=True) with open(path, "w") as f: for lbl, text in rows: idx = label_index[lbl] f.write(f"{idx}\t0\t{text}\n") print(f" Wrote {len(rows)} rows → {path}") # --------------------------------------------------------------------------- # Main # --------------------------------------------------------------------------- def main(): ap = argparse.ArgumentParser() ap.add_argument("--whetstone-bin", default="editor/build-native/whetstone_mcp") ap.add_argument("--workspace", default=".") ap.add_argument("--dataset-dir", default="datasets/project_benchmarks") ap.add_argument("--out-dir", default="specialists/data/generated") ap.add_argument("--eval-frac", type=float, default=0.15) ap.add_argument("--seed", type=int, default=42) ap.add_argument("--dry-run", action="store_true", help="Print extracted examples without writing files") args = ap.parse_args() binary = str(Path(args.workspace) / args.whetstone_bin) workspace = str(Path(args.workspace).resolve()) out_dir = Path(args.out_dir) projects = load_projects(Path(args.workspace) / args.dataset_dir) prereq_examples: list[tuple[str, str]] = [] worker_examples: list[tuple[str, str]] = [] verif_examples: list[tuple[str, str]] = [] for i, project in enumerate(projects): print(f"[{i+1:3}/{len(projects)}] {project.get('project','?')} ({project.get('category','?')})") spec_md = spec_markdown(project) tasks = run_pipeline(binary, workspace, spec_md) if not tasks: print(" -> no tasks generated, skipping") continue for task in tasks: text = task_text(task) if not text or len(text) < 8: continue # prereq_op_selector — use text-based classifier (same signal as sprint-plan # data) to avoid circular heuristic labels (C++ heuristic never emits # manual-approval, so ops-based labeling produces 100% "standard"). prereq_examples.append((text_to_prereq_class(text), text)) # worker_type wt = task.get("workerType", "") if wt in WORKER_LABEL_INDEX: worker_examples.append((wt, text)) # verification_type vt = task.get("executionContract", {}).get("verificationType", "") if vt in VERIF_LABEL_INDEX: verif_examples.append((vt, text)) print(f" -> {len(tasks)} tasks, prereq={len(prereq_examples)} " f"worker={len(worker_examples)} verif={len(verif_examples)}") if args.dry_run: print("\n=== SAMPLE EXAMPLES ===") for name, examples in [("prereq", prereq_examples[:5]), ("worker", worker_examples[:5]), ("verif", verif_examples[:5])]: print(f"\n{name}:") for lbl, text in examples: print(f" [{lbl}] {text[:80]}") return PREREQ_ORDER = ["standard", "needs_review", "needs_approval", "full_gates"] WORKER_ORDER = ["implementer", "reviewer", "architect", "qa"] VERIF_ORDER = ["unit", "integration", "schema", "smoke", "docs"] print("\n=== prereq_op from projects ===") train, eval_ = balance_and_split(prereq_examples, PREREQ_LABEL_INDEX, PREREQ_ORDER, seed=args.seed, eval_frac=args.eval_frac) write_tsv(out_dir / "prereq_op_projects_train.tsv", train, PREREQ_LABEL_INDEX) write_tsv(out_dir / "prereq_op_projects_eval.tsv", eval_, PREREQ_LABEL_INDEX) print("\n=== worker_type from projects ===") train, eval_ = balance_and_split(worker_examples, WORKER_LABEL_INDEX, WORKER_ORDER, seed=args.seed, eval_frac=args.eval_frac) write_tsv(out_dir / "worker_type_projects_train.tsv", train, WORKER_LABEL_INDEX) write_tsv(out_dir / "worker_type_projects_eval.tsv", eval_, WORKER_LABEL_INDEX) print("\n=== verification_type from projects ===") train, eval_ = balance_and_split(verif_examples, VERIF_LABEL_INDEX, VERIF_ORDER, seed=args.seed, eval_frac=args.eval_frac) write_tsv(out_dir / "verification_type_projects_train.tsv", train, VERIF_LABEL_INDEX) write_tsv(out_dir / "verification_type_projects_eval.tsv", eval_, VERIF_LABEL_INDEX) print("\nDone. Combine with sprint-plan TSVs using:") print(" cat specialists/data/generated/prereq_op_train.tsv \\") print(" specialists/data/generated/prereq_op_projects_train.tsv \\") print(" > specialists/data/combined/prereq_op_train.tsv") if __name__ == "__main__": main()