From b68c95e6f45fc8f28798916420c918fe725c6e29 Mon Sep 17 00:00:00 2001 From: Bill Date: Thu, 26 Feb 2026 14:12:33 -0700 Subject: [PATCH] Add spec hardener and reusable hard-gate wrapper --- NEXT_AGENT_NOTES.md | 2 + docs/progress_log_2026-02-26.md | 7 ++ ...int228_231_execution_tracker_2026-02-26.md | 21 ++++- tools/mcp/run_spec_hardening_gate.sh | 83 +++++++++++++++++++ tools/mcp/spec_planning_hardener.py | 65 +++++++++++++++ 5 files changed, 177 insertions(+), 1 deletion(-) create mode 100755 tools/mcp/run_spec_hardening_gate.sh create mode 100755 tools/mcp/spec_planning_hardener.py diff --git a/NEXT_AGENT_NOTES.md b/NEXT_AGENT_NOTES.md index 89dc6f4..9fd7f5e 100644 --- a/NEXT_AGENT_NOTES.md +++ b/NEXT_AGENT_NOTES.md @@ -16,6 +16,8 @@ - First-class planning tranche (new): - `docs/sprint228_231_execution_tracker_2026-02-26.md` - `tools/mcp/spec_planning_readiness.py` + - `tools/mcp/spec_planning_hardener.py` + - `tools/mcp/run_spec_hardening_gate.sh` ## Dated Handoff diff --git a/docs/progress_log_2026-02-26.md b/docs/progress_log_2026-02-26.md index c0bad3b..568d7fe 100644 --- a/docs/progress_log_2026-02-26.md +++ b/docs/progress_log_2026-02-26.md @@ -60,3 +60,10 @@ - `WSTONE_SPEC_READINESS_HARD_GATE` - `WSTONE_SPEC_READINESS_MIN_SCORE` - Spec readiness is now available in pipeline `00_summary.json` as `planning_readiness`. +- Added auto-hardening utility: + - `tools/mcp/spec_planning_hardener.py` +- Added end-to-end hardening gate wrapper: + - `tools/mcp/run_spec_hardening_gate.sh` +- Demonstrated hard-gate delta on sprint sample: + - original spec blocked (`rc=7`, score `17`) + - hardened spec passed (`rc=0`, score `100`) diff --git a/docs/sprint228_231_execution_tracker_2026-02-26.md b/docs/sprint228_231_execution_tracker_2026-02-26.md index b2750e7..46f2e10 100644 --- a/docs/sprint228_231_execution_tracker_2026-02-26.md +++ b/docs/sprint228_231_execution_tracker_2026-02-26.md @@ -27,11 +27,30 @@ Planned and started: - `logs/taskitem_runs/spec_planning_baseline_20260226/fullstack_spec_readiness.json` - `logs/taskitem_runs/spec_planning_baseline_20260226/deterministic_spec_readiness.json` +- `logs/taskitem_runs/spec_planning_hardening_20260226/sprint228_before.json` +- `logs/taskitem_runs/spec_planning_hardening_20260226/sprint228_after.json` +- `logs/taskitem_runs/spec_planning_hardening_20260226/sprint228_hardened.md` +- `logs/taskitem_runs/spec_hardening_gate_20260226_s228_r2/report.json` Observed baseline: - both sample docs currently return `needs_spec_hardening` - low readiness scores indicate missing first-class sections for constraints/acceptance/environment/security/performance. +Hardening delta (sprint228 plan sample): +- before: `score=17`, verdict `needs_spec_hardening`, missing actions `10` +- after auto-hardening scaffold: `score=100`, verdict `execution_ready`, missing actions `0` + +Pipeline hard-gate proof: +- original spec run (hard gate on): `rc=7` with explicit readiness failure + - `logs/taskitem_runs/sprint228_plan_20260226_141045/00a_spec_readiness.json` +- hardened spec run (hard gate on): `rc=0` and full pipeline completion + - `logs/taskitem_runs/sprint228_hardened_20260226_141045/00_summary.json` + +Reusable flow wrapper: +- `tools/mcp/run_spec_hardening_gate.sh` +- example invocation: + - `tools/mcp/run_spec_hardening_gate.sh sprint228_plan.md logs/taskitem_runs/spec_hardening_gate_20260226_s228_r2` + ## Explicit Completion Signal - Sprint 228: `PARTIAL` (tool implemented, baseline executed, pipeline precheck integrated) @@ -42,5 +61,5 @@ Observed baseline: ## Next Closure Work - Enable hard-gate mode for selected sprint ranges and capture pass/fail deltas. -- Add structured spec hardening pass that applies template scaffolds to candidate specs. +- Add reusable wrapper script for readiness->hardening->hard-gate flow across catalogs. - Re-run benchmarks using hardened specs and measure downstream readiness deltas. diff --git a/tools/mcp/run_spec_hardening_gate.sh b/tools/mcp/run_spec_hardening_gate.sh new file mode 100755 index 0000000..3e76d38 --- /dev/null +++ b/tools/mcp/run_spec_hardening_gate.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +SPEC_INPUT="${1:?usage: run_spec_hardening_gate.sh [out_dir]}" +OUT_DIR="${2:-$ROOT_DIR/logs/taskitem_runs/spec_hardening_gate_$(date +%Y%m%d_%H%M%S)}" +MIN_SCORE="${WSTONE_SPEC_READINESS_MIN_SCORE:-65}" +RUN_PIPELINE="${WSTONE_SPEC_HARDENING_RUN_PIPELINE:-1}" + +if [[ "$SPEC_INPUT" = /* ]]; then + SPEC_PATH="$SPEC_INPUT" +else + SPEC_PATH="$ROOT_DIR/$SPEC_INPUT" +fi + +if [[ ! -f "$SPEC_PATH" ]]; then + echo "error: spec not found: $SPEC_PATH" >&2 + exit 2 +fi + +mkdir -p "$OUT_DIR" + +BEFORE_JSON="$OUT_DIR/before_readiness.json" +HARDENED_MD="$OUT_DIR/hardened_spec.md" +AFTER_JSON="$OUT_DIR/after_readiness.json" +REPORT_JSON="$OUT_DIR/report.json" + +python3 "$ROOT_DIR/tools/mcp/spec_planning_readiness.py" --spec "$SPEC_PATH" --out "$BEFORE_JSON" >/dev/null +python3 "$ROOT_DIR/tools/mcp/spec_planning_hardener.py" --spec "$SPEC_PATH" --readiness "$BEFORE_JSON" --out "$HARDENED_MD" > "$OUT_DIR/hardener_report.json" +python3 "$ROOT_DIR/tools/mcp/spec_planning_readiness.py" --spec "$HARDENED_MD" --out "$AFTER_JSON" >/dev/null + +ORIG_RC=null +HARD_RC=null +ORIG_LOG="" +HARD_LOG="" + +if [[ "$RUN_PIPELINE" == "1" ]]; then + ORIG_LOG="$OUT_DIR/original_pipeline.log" + HARD_LOG="$OUT_DIR/hardened_pipeline.log" + + set +e + WSTONE_RUN_READINESS_SUITE=0 WSTONE_SPEC_READINESS_PRECHECK=1 WSTONE_SPEC_READINESS_HARD_GATE=1 WSTONE_SPEC_READINESS_MIN_SCORE="$MIN_SCORE" \ + "$ROOT_DIR/tools/mcp/run_sprint_taskitem_pipeline.sh" "$SPEC_PATH" >"$ORIG_LOG" 2>&1 + ORIG_RC=$? + + WSTONE_RUN_READINESS_SUITE=0 WSTONE_SPEC_READINESS_PRECHECK=1 WSTONE_SPEC_READINESS_HARD_GATE=1 WSTONE_SPEC_READINESS_MIN_SCORE="$MIN_SCORE" \ + "$ROOT_DIR/tools/mcp/run_sprint_taskitem_pipeline.sh" "$HARDENED_MD" >"$HARD_LOG" 2>&1 + HARD_RC=$? + set -e +fi + +jq -nc \ + --arg spec "$SPEC_PATH" \ + --arg out_dir "$OUT_DIR" \ + --argjson before "$(cat "$BEFORE_JSON")" \ + --argjson after "$(cat "$AFTER_JSON")" \ + --argjson min_score "$MIN_SCORE" \ + --argjson run_pipeline "$RUN_PIPELINE" \ + --argjson original_rc "$ORIG_RC" \ + --argjson hardened_rc "$HARD_RC" \ + --arg original_log "$ORIG_LOG" \ + --arg hardened_log "$HARD_LOG" \ + '{ + spec:$spec, + out_dir:$out_dir, + readiness:{ + before:{score:$before.scores.total,verdict:$before.verdict,missing_count:($before.missing_actions|length)}, + after:{score:$after.scores.total,verdict:$after.verdict,missing_count:($after.missing_actions|length)}, + delta_score:($after.scores.total-$before.scores.total) + }, + hard_gate:{ + min_score:$min_score, + run_pipeline:($run_pipeline==1), + original_rc:$original_rc, + hardened_rc:$hardened_rc, + original_log:$original_log, + hardened_log:$hardened_log + } + }' > "$REPORT_JSON" + +echo "Spec hardening gate complete." +echo "Output dir: $OUT_DIR" +cat "$REPORT_JSON" diff --git a/tools/mcp/spec_planning_hardener.py b/tools/mcp/spec_planning_hardener.py new file mode 100755 index 0000000..bf187ce --- /dev/null +++ b/tools/mcp/spec_planning_hardener.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def load_readiness(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def harden_markdown(original: str, readiness: dict) -> str: + template = (readiness.get("recommended_template") or "").strip() + if not template: + return original + + missing = readiness.get("missing_actions") or [] + missing_lines = "\n".join([f"- [{m.get('type','?')}] `{m.get('key','?')}`: {m.get('action','')}" for m in missing]) + + addendum = ( + "\n\n---\n\n" + "## Planning Hardening Addendum\n\n" + "This section is auto-generated to make the spec execution-ready for Whetstone.\n\n" + "### Detected Gaps\n" + f"{missing_lines if missing_lines else '- none'}\n\n" + "### Recommended Scaffold\n\n" + f"{template}\n" + ) + + if "## Planning Hardening Addendum" in original: + return original + return original.rstrip() + addendum + + +def main() -> None: + ap = argparse.ArgumentParser(description="Generate hardened markdown spec scaffold from planning readiness output.") + ap.add_argument("--spec", required=True, help="Input markdown spec path") + ap.add_argument("--readiness", required=True, help="Readiness JSON path from spec_planning_readiness.py") + ap.add_argument("--out", required=True, help="Output hardened markdown path") + args = ap.parse_args() + + spec_path = Path(args.spec) + readiness_path = Path(args.readiness) + out_path = Path(args.out) + + original = spec_path.read_text(encoding="utf-8") + readiness = load_readiness(readiness_path) + hardened = harden_markdown(original, readiness) + + out_path.parent.mkdir(parents=True, exist_ok=True) + out_path.write_text(hardened, encoding="utf-8") + + report = { + "input_spec": str(spec_path), + "readiness_json": str(readiness_path), + "output_spec": str(out_path), + "missing_count": len(readiness.get("missing_actions") or []), + "added_addendum": hardened != original, + } + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + main()