Sprint 003 findings + Sprint 004 pipeline decision audit plan
- split_prereq_op_dataset.py: extract accepted/rejected/schema-drift rows per gate - gen_prereq_op_rsa_data.py: convert accepted rows to Fabricate TSV format - 1000-run extracted dataset: 1297 accepted prereq_op rows, 24 schema-drift - Key finding: needs_validate_intake always True; effective gate is binary (needs_resolve_dependencies 77%, needs_architect_review 20%) - Key finding: run corpus text is template-driven (17 unique inputs) — not suitable for text-based training; RSA exists before ideal input is available - ROADMAP: gate lifecycle framing — goal is choice of backend, not forced determinism; RSA operates on imperfect/mixed/incomplete input by design - training_data_strategy.md: taskitem-as-self-contained-work-unit constraint - SPRINT-003: updated with 1000-run results, text audit findings, design constraints - SPRINT-004: pipeline decision audit — manual walkthrough of every decision from project description to generated artifact; execute next session Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Bill Holcombe
174
tools/gen_prereq_op_rsa_data.py
Normal file
174
tools/gen_prereq_op_rsa_data.py
Normal file
@@ -0,0 +1,174 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate Fabricate-format TSV training data for the two prereq_op binary specialists.
|
||||
|
||||
Source: semantic/prereq_op_dataset_1000/accepted.ndjson
|
||||
(extracted from real whetstone_DSL taskitem run artifacts)
|
||||
|
||||
Produces four files:
|
||||
prereq_resolve_train.tsv / prereq_resolve_eval.tsv
|
||||
prereq_architect_train.tsv / prereq_architect_eval.tsv
|
||||
|
||||
Each file uses Fabricate TSV format: label<TAB>0<TAB>text
|
||||
label: 0 = no, 1 = yes
|
||||
0: unused hop field (always 0)
|
||||
text: composed from task context fields
|
||||
|
||||
Text composition (per row):
|
||||
title: <task title>
|
||||
reasons: <reason1>; <reason2>; ...
|
||||
accepts: <acceptance criterion text>; ...
|
||||
constraints: <constraint text>; ...
|
||||
|
||||
Usage:
|
||||
python3 tools/gen_prereq_op_rsa_data.py \\
|
||||
--input semantic/prereq_op_dataset_1000/accepted.ndjson \\
|
||||
--output-dir semantic/prereq_op_tsv \\
|
||||
--eval-frac 0.15 \\
|
||||
--seed 42
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import random
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def compose_text(ctx: dict) -> str:
|
||||
parts = []
|
||||
|
||||
title = (ctx.get("title") or "").strip()
|
||||
if title:
|
||||
parts.append(f"title: {title}")
|
||||
|
||||
reasons = ctx.get("reasons") or []
|
||||
if reasons:
|
||||
parts.append("reasons: " + "; ".join(str(r).strip() for r in reasons if r))
|
||||
|
||||
reqs = ctx.get("normalized_requirements") or []
|
||||
accepts = [r["normalizedText"] for r in reqs if r.get("kind") == "acceptance" and r.get("normalizedText")]
|
||||
constraints = [r["normalizedText"] for r in reqs if r.get("kind") == "constraint" and r.get("normalizedText")]
|
||||
|
||||
if accepts:
|
||||
parts.append("accepts: " + "; ".join(accepts))
|
||||
if constraints:
|
||||
parts.append("constraints: " + "; ".join(constraints))
|
||||
|
||||
return " | ".join(parts)
|
||||
|
||||
|
||||
def write_tsv(path: Path, rows: list[tuple[int, str]]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as fh:
|
||||
for label, text in rows:
|
||||
fh.write(f"{label}\t0\t{text}\n")
|
||||
|
||||
|
||||
def label_summary(rows: list[tuple[int, str]]) -> dict:
|
||||
pos = sum(1 for label, _ in rows if label == 1)
|
||||
neg = len(rows) - pos
|
||||
return {"total": len(rows), "pos": pos, "neg": neg, "pos_pct": round(pos / len(rows) * 100, 1)}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--input",
|
||||
default="semantic/prereq_op_dataset_1000/accepted.ndjson",
|
||||
help="Accepted prereq_op rows in NDJSON format.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
default="semantic/prereq_op_tsv",
|
||||
help="Directory for output TSV files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--eval-frac",
|
||||
type=float,
|
||||
default=0.15,
|
||||
help="Fraction of rows to use for eval (default: 0.15).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=42,
|
||||
help="Random seed for train/eval split.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-rows",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Optional cap on input rows (0 = no limit).",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
input_path = Path(args.input)
|
||||
if not input_path.is_absolute():
|
||||
input_path = (REPO_ROOT / input_path).resolve()
|
||||
output_dir = Path(args.output_dir)
|
||||
if not output_dir.is_absolute():
|
||||
output_dir = (REPO_ROOT / output_dir).resolve()
|
||||
|
||||
rows: list[dict] = []
|
||||
with input_path.open("r", encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
rows.append(json.loads(line))
|
||||
if args.max_rows and len(rows) >= args.max_rows:
|
||||
break
|
||||
|
||||
rng = random.Random(args.seed)
|
||||
rng.shuffle(rows)
|
||||
|
||||
n_eval = max(1, int(len(rows) * args.eval_frac))
|
||||
eval_rows = rows[:n_eval]
|
||||
train_rows = rows[n_eval:]
|
||||
|
||||
def to_binary(split: list[dict], key: str) -> list[tuple[int, str]]:
|
||||
result = []
|
||||
for row in split:
|
||||
label_val = row["decision_label"].get(key, False)
|
||||
text = compose_text(row["decision_context"])
|
||||
if not text:
|
||||
continue
|
||||
result.append((1 if label_val else 0, text))
|
||||
return result
|
||||
|
||||
resolve_train = to_binary(train_rows, "needs_resolve_dependencies")
|
||||
resolve_eval = to_binary(eval_rows, "needs_resolve_dependencies")
|
||||
architect_train = to_binary(train_rows, "needs_architect_review")
|
||||
architect_eval = to_binary(eval_rows, "needs_architect_review")
|
||||
|
||||
write_tsv(output_dir / "prereq_resolve_train.tsv", resolve_train)
|
||||
write_tsv(output_dir / "prereq_resolve_eval.tsv", resolve_eval)
|
||||
write_tsv(output_dir / "prereq_architect_train.tsv", architect_train)
|
||||
write_tsv(output_dir / "prereq_architect_eval.tsv", architect_eval)
|
||||
|
||||
summary = {
|
||||
"input": str(input_path),
|
||||
"output_dir": str(output_dir),
|
||||
"total_rows": len(rows),
|
||||
"train_rows": len(train_rows),
|
||||
"eval_rows": len(eval_rows),
|
||||
"needs_resolve_dependencies": {
|
||||
"train": label_summary(resolve_train),
|
||||
"eval": label_summary(resolve_eval),
|
||||
},
|
||||
"needs_architect_review": {
|
||||
"train": label_summary(architect_train),
|
||||
"eval": label_summary(architect_eval),
|
||||
},
|
||||
}
|
||||
print(json.dumps(summary, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
117
tools/split_prereq_op_dataset.py
Normal file
117
tools/split_prereq_op_dataset.py
Normal file
@@ -0,0 +1,117 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Split extracted gate rows into prereq_op_selector accepted / rejected / schema-drift buckets.
|
||||
|
||||
Reads an NDJSON file of extracted gate rows (from extract_gate_rows.py) and
|
||||
produces three output files covering only prereq_op_selector rows:
|
||||
|
||||
accepted.ndjson — rows with no rejection flags
|
||||
rejected.ndjson — rows with any rejection flag
|
||||
schema_drift.ndjson — subset of rejected rows where rejection is caused by
|
||||
invalid_prerequisite_values (schema-drift label pollution)
|
||||
|
||||
Also prints a brief summary to stdout.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
SCHEMA_DRIFT_FLAG_PREFIX = "invalid_prerequisite_values"
|
||||
|
||||
|
||||
def is_schema_drift(row: dict) -> bool:
|
||||
return any(f.startswith(SCHEMA_DRIFT_FLAG_PREFIX) for f in row.get("rejection_flags", []))
|
||||
|
||||
|
||||
def write_ndjson(path: Path, rows: list[dict]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as fh:
|
||||
for row in rows:
|
||||
fh.write(json.dumps(row, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument(
|
||||
"--input",
|
||||
default="semantic/extracted_gate_rows_slice.ndjson",
|
||||
help="NDJSON file produced by extract_gate_rows.py.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
default="semantic/prereq_op_dataset",
|
||||
help="Directory where accepted/rejected/schema_drift files are written.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
repo_root = Path(__file__).resolve().parent.parent
|
||||
input_path = Path(args.input)
|
||||
if not input_path.is_absolute():
|
||||
input_path = (repo_root / input_path).resolve()
|
||||
output_dir = Path(args.output_dir)
|
||||
if not output_dir.is_absolute():
|
||||
output_dir = (repo_root / output_dir).resolve()
|
||||
|
||||
accepted: list[dict] = []
|
||||
rejected: list[dict] = []
|
||||
schema_drift: list[dict] = []
|
||||
skipped = 0
|
||||
|
||||
with input_path.open("r", encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
row = json.loads(line)
|
||||
if row.get("gate_id") != "prereq_op_selector":
|
||||
skipped += 1
|
||||
continue
|
||||
flags = row.get("rejection_flags", [])
|
||||
if not flags:
|
||||
accepted.append(row)
|
||||
else:
|
||||
rejected.append(row)
|
||||
if is_schema_drift(row):
|
||||
schema_drift.append(row)
|
||||
|
||||
write_ndjson(output_dir / "accepted.ndjson", accepted)
|
||||
write_ndjson(output_dir / "rejected.ndjson", rejected)
|
||||
write_ndjson(output_dir / "schema_drift.ndjson", schema_drift)
|
||||
|
||||
# Flag breakdown across rejected rows
|
||||
flag_counts: dict[str, int] = {}
|
||||
for row in rejected:
|
||||
for f in row.get("rejection_flags", []):
|
||||
flag_counts[f] = flag_counts.get(f, 0) + 1
|
||||
|
||||
# Label distribution for accepted rows
|
||||
label_counts: dict[str, int] = {
|
||||
"needs_validate_intake": 0,
|
||||
"needs_architect_review": 0,
|
||||
"needs_resolve_dependencies": 0,
|
||||
}
|
||||
for row in accepted:
|
||||
label = row.get("decision_label", {})
|
||||
for key in label_counts:
|
||||
if label.get(key):
|
||||
label_counts[key] += 1
|
||||
|
||||
summary = {
|
||||
"input": str(input_path),
|
||||
"output_dir": str(output_dir),
|
||||
"rows_accepted": len(accepted),
|
||||
"rows_rejected": len(rejected),
|
||||
"rows_schema_drift": len(schema_drift),
|
||||
"rows_skipped_other_gates": skipped,
|
||||
"rejection_flag_breakdown": flag_counts,
|
||||
"accepted_label_distribution": label_counts,
|
||||
}
|
||||
print(json.dumps(summary, indent=2, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user