Add sprint 234 semantic planning coverage gate
This commit is contained in:
@@ -22,11 +22,13 @@
|
||||
- `docs/sprint228_231_execution_tracker_2026-02-26.md`
|
||||
- `docs/sprint232_execution_tracker_2026-02-26.md`
|
||||
- `docs/sprint233_execution_tracker_2026-02-26.md`
|
||||
- `docs/sprint234_execution_tracker_2026-02-26.md`
|
||||
- `tools/mcp/spec_planning_readiness.py`
|
||||
- `tools/mcp/spec_planning_hardener.py`
|
||||
- `tools/mcp/run_spec_hardening_gate.sh`
|
||||
- `tools/mcp/markdown_to_semantic_annotations.py`
|
||||
- `tools/mcp/augment_spec_with_semantic_packet.py`
|
||||
- `tools/mcp/check_semantic_planning_gate.py`
|
||||
- `docs/spec_hardening_batch_report_2026-02-26.md`
|
||||
|
||||
## Dated Handoff
|
||||
|
||||
@@ -97,3 +97,11 @@
|
||||
- A/B run shows measurable decomposition delta (TEST_ONLY artifacts):
|
||||
- validation taskitems `2 -> 8`
|
||||
- avg execution specificity `75.0 -> 84.75`
|
||||
|
||||
## Sprint 234 Added (Same Day)
|
||||
|
||||
- Added semantic quality gate tooling:
|
||||
- `tools/mcp/check_semantic_planning_gate.py`
|
||||
- Integrated optional semantic gate into pipeline with hard-fail behavior.
|
||||
- Smoke-verified gate-enabled run (TEST_ONLY):
|
||||
- `logs/taskitem_runs/TEST_ONLY_sprint234_semantic_gate_smoke_20260226/00_summary.json`
|
||||
|
||||
26
docs/sprint234_execution_tracker_2026-02-26.md
Normal file
26
docs/sprint234_execution_tracker_2026-02-26.md
Normal file
@@ -0,0 +1,26 @@
|
||||
# Sprint 234 Execution Tracker - 2026-02-26
|
||||
|
||||
## Scope
|
||||
- `sprint234_plan.md`
|
||||
|
||||
## Implemented
|
||||
|
||||
- Added gate checker tool:
|
||||
- `tools/mcp/check_semantic_planning_gate.py`
|
||||
|
||||
Pipeline integration in `tools/mcp/run_sprint_taskitem_pipeline.sh`:
|
||||
- `WSTONE_SEMANTIC_COVERAGE_GATE` (default `0`)
|
||||
- `WSTONE_SEMANTIC_MIN_COVERAGE` (default `0.9`)
|
||||
- `WSTONE_SEMANTIC_REQUIRE_CAPS_FOR_COMPLEX` (default `1`)
|
||||
- `WSTONE_SEMANTIC_MIN_COMPLEXITY_SCORE` (default `2`)
|
||||
- emits `00bb_semantic_planning_gate.json` when gate is enabled
|
||||
- summary now includes `semantic_planning_gate`
|
||||
|
||||
## Baseline Smoke
|
||||
|
||||
- gate-enabled smoke run (TEST_ONLY):
|
||||
- `logs/taskitem_runs/TEST_ONLY_sprint234_semantic_gate_smoke_20260226/00_summary.json`
|
||||
|
||||
## Explicit Completion Signal
|
||||
|
||||
- Sprint 234: `DONE` (implemented + integrated + smoke verified)
|
||||
5
editor/src/Sprint234IntegrationSummary.h
Normal file
5
editor/src/Sprint234IntegrationSummary.h
Normal file
@@ -0,0 +1,5 @@
|
||||
#pragma once
|
||||
|
||||
// Sprint 234 integration summary:
|
||||
// - Added semantic planning quality gate and optional hard-fail pipeline enforcement.
|
||||
// - Pipeline summaries now include semantic gate evaluation telemetry.
|
||||
12
sprint234_plan.md
Normal file
12
sprint234_plan.md
Normal file
@@ -0,0 +1,12 @@
|
||||
# Sprint 234 Plan: Semantic Planning Coverage Gate
|
||||
|
||||
## Goal
|
||||
Add enforceable semantic planning quality gates prior to taskitem generation.
|
||||
|
||||
## Steps
|
||||
- Step 2211: Add semantic planning gate checker tool.
|
||||
- Step 2212: Gate on semantic coverage ratio threshold.
|
||||
- Step 2213: Require capability requirements for complex semantic packets.
|
||||
- Step 2214: Integrate optional hard-fail semantic gate into pipeline.
|
||||
- Step 2215: Emit semantic gate telemetry in pipeline summary.
|
||||
- Step 2216: Add `Sprint234IntegrationSummary.h`.
|
||||
68
tools/mcp/check_semantic_planning_gate.py
Executable file
68
tools/mcp/check_semantic_planning_gate.py
Executable file
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
|
||||
def gate(packet: Dict[str, object], min_coverage: float, require_caps_for_complex: bool, min_complexity_score: int) -> Dict[str, object]:
|
||||
coverage_ratio = float((((packet.get("coverage") or {}).get("coverage_ratio")) or 0.0))
|
||||
anns = packet.get("annotations") or []
|
||||
|
||||
cap_count = 0
|
||||
complexity_score = 0
|
||||
for ann in anns:
|
||||
if not isinstance(ann, dict):
|
||||
continue
|
||||
t = ann.get("type")
|
||||
if t == "capability_requirement":
|
||||
cap_count += 1
|
||||
if t == "complexity":
|
||||
fields = ann.get("fields") or {}
|
||||
complexity_score = int(fields.get("score") or 0)
|
||||
|
||||
reasons: List[str] = []
|
||||
if coverage_ratio < min_coverage:
|
||||
reasons.append(f"coverage_below_threshold:{coverage_ratio}<{min_coverage}")
|
||||
|
||||
if require_caps_for_complex and complexity_score >= min_complexity_score and cap_count == 0:
|
||||
reasons.append("complex_spec_missing_capability_requirements")
|
||||
|
||||
passed = len(reasons) == 0
|
||||
return {
|
||||
"passed": passed,
|
||||
"reasons": reasons,
|
||||
"metrics": {
|
||||
"coverage_ratio": coverage_ratio,
|
||||
"capability_requirement_count": cap_count,
|
||||
"complexity_score": complexity_score,
|
||||
"min_coverage": min_coverage,
|
||||
"require_caps_for_complex": require_caps_for_complex,
|
||||
"min_complexity_score": min_complexity_score,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser(description="Check semantic planning packet quality gate.")
|
||||
ap.add_argument("--packet", required=True)
|
||||
ap.add_argument("--out", required=True)
|
||||
ap.add_argument("--min-coverage", type=float, default=0.9)
|
||||
ap.add_argument("--require-caps-for-complex", action="store_true")
|
||||
ap.add_argument("--min-complexity-score", type=int, default=2)
|
||||
args = ap.parse_args()
|
||||
|
||||
packet = json.loads(Path(args.packet).read_text(encoding="utf-8"))
|
||||
res = gate(packet, args.min_coverage, args.require_caps_for_complex, args.min_complexity_score)
|
||||
out = Path(args.out)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(json.dumps(res, indent=2) + "\n", encoding="utf-8")
|
||||
print(json.dumps(res, indent=2))
|
||||
if not res["passed"]:
|
||||
raise SystemExit(3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -28,6 +28,10 @@ SEMANTIC_PLANNING_BRIDGE="${WSTONE_SEMANTIC_PLANNING_BRIDGE:-1}"
|
||||
SEMANTIC_INTAKE_AUGMENT="${WSTONE_SEMANTIC_INTAKE_AUGMENT:-1}"
|
||||
SEMANTIC_REQUIREMENT_INJECTION="${WSTONE_SEMANTIC_REQUIREMENT_INJECTION:-1}"
|
||||
SEMANTIC_TASK_EXPANSION="${WSTONE_SEMANTIC_TASK_EXPANSION:-1}"
|
||||
SEMANTIC_COVERAGE_GATE="${WSTONE_SEMANTIC_COVERAGE_GATE:-0}"
|
||||
SEMANTIC_MIN_COVERAGE="${WSTONE_SEMANTIC_MIN_COVERAGE:-0.9}"
|
||||
SEMANTIC_REQUIRE_CAPS_FOR_COMPLEX="${WSTONE_SEMANTIC_REQUIRE_CAPS_FOR_COMPLEX:-1}"
|
||||
SEMANTIC_MIN_COMPLEXITY_SCORE="${WSTONE_SEMANTIC_MIN_COMPLEXITY_SCORE:-2}"
|
||||
CAPABILITY_SIGNALS_JSON="${WSTONE_CAPABILITY_SIGNALS_JSON:-}"
|
||||
if [[ -z "$CAPABILITY_SIGNALS_JSON" ]]; then
|
||||
CAPABILITY_SIGNALS_JSON='{}'
|
||||
@@ -78,11 +82,31 @@ SEMANTIC_PLANNING_JSON='{}'
|
||||
SEMANTIC_AUGMENT_JSON='{}'
|
||||
SEMANTIC_REQUIREMENT_INJECTION_JSON='{}'
|
||||
SEMANTIC_TASK_EXPANSION_JSON='{}'
|
||||
SEMANTIC_GATE_JSON='{}'
|
||||
if [[ "$SEMANTIC_PLANNING_BRIDGE" == "1" ]]; then
|
||||
python3 "$ROOT_DIR/tools/mcp/markdown_to_semantic_annotations.py" \
|
||||
--spec "$INPUT_FILE" \
|
||||
--out "$OUT_DIR/00b_semantic_planning_annotations.json" >/dev/null
|
||||
SEMANTIC_PLANNING_JSON="$(cat "$OUT_DIR/00b_semantic_planning_annotations.json")"
|
||||
if [[ "$SEMANTIC_COVERAGE_GATE" == "1" ]]; then
|
||||
gate_flags=()
|
||||
if [[ "$SEMANTIC_REQUIRE_CAPS_FOR_COMPLEX" == "1" ]]; then
|
||||
gate_flags+=(--require-caps-for-complex)
|
||||
fi
|
||||
if python3 "$ROOT_DIR/tools/mcp/check_semantic_planning_gate.py" \
|
||||
--packet "$OUT_DIR/00b_semantic_planning_annotations.json" \
|
||||
--out "$OUT_DIR/00bb_semantic_planning_gate.json" \
|
||||
--min-coverage "$SEMANTIC_MIN_COVERAGE" \
|
||||
--min-complexity-score "$SEMANTIC_MIN_COMPLEXITY_SCORE" \
|
||||
"${gate_flags[@]}" >/dev/null; then
|
||||
SEMANTIC_GATE_JSON="$(cat "$OUT_DIR/00bb_semantic_planning_gate.json")"
|
||||
else
|
||||
SEMANTIC_GATE_JSON="$(cat "$OUT_DIR/00bb_semantic_planning_gate.json")"
|
||||
echo "error: semantic planning gate failed" >&2
|
||||
echo "error: see $OUT_DIR/00bb_semantic_planning_gate.json" >&2
|
||||
exit 8
|
||||
fi
|
||||
fi
|
||||
if [[ "$SEMANTIC_INTAKE_AUGMENT" == "1" ]]; then
|
||||
python3 "$ROOT_DIR/tools/mcp/augment_spec_with_semantic_packet.py" \
|
||||
--spec "$INPUT_FILE" \
|
||||
@@ -324,6 +348,7 @@ SUMMARY_JSON="$(jq -nc \
|
||||
--arg workspace "$WORKSPACE" \
|
||||
--argjson planning_readiness "$SPEC_READINESS_JSON" \
|
||||
--argjson semantic_planning "$SEMANTIC_PLANNING_JSON" \
|
||||
--argjson semantic_gate "$SEMANTIC_GATE_JSON" \
|
||||
--argjson semantic_intake_augment "$SEMANTIC_AUGMENT_JSON" \
|
||||
--argjson semantic_requirement_injection "$SEMANTIC_REQUIREMENT_INJECTION_JSON" \
|
||||
--argjson semantic_task_expansion "$SEMANTIC_TASK_EXPANSION_JSON" \
|
||||
@@ -340,6 +365,7 @@ SUMMARY_JSON="$(jq -nc \
|
||||
output_dir: $outDir,
|
||||
planning_readiness: $planning_readiness,
|
||||
semantic_planning_annotations: $semantic_planning,
|
||||
semantic_planning_gate: $semantic_gate,
|
||||
semantic_intake_augment: $semantic_intake_augment,
|
||||
semantic_requirement_injection: $semantic_requirement_injection,
|
||||
semantic_task_expansion: $semantic_task_expansion,
|
||||
|
||||
Reference in New Issue
Block a user