From 55876d3540ef33576d87262e57be0d54880f25a4 Mon Sep 17 00:00:00 2001 From: Bill Date: Thu, 26 Feb 2026 14:08:12 -0700 Subject: [PATCH] Start first-class planning tranche with spec readiness scoring --- NEXT_AGENT_NOTES.md | 3 + docs/constructive_editing_runtime_plan.md | 150 ++++++++++++++++ docs/spec_planning_baseline_2026-02-26.md | 26 +++ ...int228_231_execution_tracker_2026-02-26.md | 40 +++++ editor/src/Sprint228IntegrationSummary.h | 4 + editor/src/Sprint229IntegrationSummary.h | 4 + editor/src/Sprint230IntegrationSummary.h | 4 + editor/src/Sprint231IntegrationSummary.h | 4 + sprint228_plan.md | 11 ++ sprint229_plan.md | 11 ++ sprint230_plan.md | 11 ++ sprint231_plan.md | 11 ++ tools/mcp/spec_planning_readiness.py | 165 ++++++++++++++++++ 13 files changed, 444 insertions(+) create mode 100644 docs/constructive_editing_runtime_plan.md create mode 100644 docs/spec_planning_baseline_2026-02-26.md create mode 100644 docs/sprint228_231_execution_tracker_2026-02-26.md create mode 100644 editor/src/Sprint228IntegrationSummary.h create mode 100644 editor/src/Sprint229IntegrationSummary.h create mode 100644 editor/src/Sprint230IntegrationSummary.h create mode 100644 editor/src/Sprint231IntegrationSummary.h create mode 100644 sprint228_plan.md create mode 100644 sprint229_plan.md create mode 100644 sprint230_plan.md create mode 100644 sprint231_plan.md create mode 100755 tools/mcp/spec_planning_readiness.py diff --git a/NEXT_AGENT_NOTES.md b/NEXT_AGENT_NOTES.md index 23cb6d1..89dc6f4 100644 --- a/NEXT_AGENT_NOTES.md +++ b/NEXT_AGENT_NOTES.md @@ -13,6 +13,9 @@ - Latest parity closure evidence: - `logs/taskitem_runs/challenging_subset_prod_20260226_r7/summary.json` - `logs/taskitem_runs/challenging_fullstack_multifile_20260226_r7/summary.json` +- First-class planning tranche (new): + - `docs/sprint228_231_execution_tracker_2026-02-26.md` + - `tools/mcp/spec_planning_readiness.py` ## Dated Handoff diff --git a/docs/constructive_editing_runtime_plan.md b/docs/constructive_editing_runtime_plan.md new file mode 100644 index 0000000..e6e0b89 --- /dev/null +++ b/docs/constructive_editing_runtime_plan.md @@ -0,0 +1,150 @@ +# Constructive Editing Runtime Plan (Low-Res) + +## Objective + +Move from MCP contract-level/runtime packet behavior to real constructive +editing and code generation execution for representative languages: + +- `cpp` +- `rust` +- `python` +- `typescript` +- `javascript` +- `go` +- `java` +- `elisp` + +## Current Baseline + +- Sprint 146-155 MCP handlers are now runtimeized and stateful. +- Representative language runtime exists and is wired to tool handlers. +- Handler-level step tests (`1703`-`1795` tool surface set) are passing. +- Taskitem pipeline logging is active and appending LoRA capture records. + +## Low-Res Plan + +1. Runtime to Engine Binding +- Replace synthetic stage outcomes with calls into existing adapter/sync/ + regeneration/merge modules. +- Keep current response envelope stable while adding real engine evidence. + +2. File Mutation and Replay Integrity +- Add real file-level mutation application paths for constructive step/loop. +- Persist before/after snapshots and replay metadata per operation. + +3. Toolchain Execution Integration +- Wire provider routing to real build/test command execution paths. +- Normalize diagnostics from execution output into canonical payloads. + +4. Multi-Language Qualification +- Run representative-language deterministic replay and diff checks. +- Gate language tier transitions using real replay/rollback evidence. + +5. GA Hardening +- Add end-to-end constructive smoke scenarios per language family. +- Finalize rollout controls and promote only evidence-backed tiers. + +6. First-Class Planning Readiness +- Add pre-execution spec readiness scoring and gap detection. +- Synthesize missing constraints/acceptance/environment scaffolds before taskitem generation. +- Gate taskitem execution on minimum planning-readiness thresholds. + +## Implementation Map + +### Phase A: Bind Constructive Step to Real Adapter Operations + +Primary files: +- `editor/src/mcp/RegisterSprint152Tools.h` +- `editor/src/graduation/RepresentativeLanguageRuntime.h` +- `editor/src/CppConstructiveEditAdapter.h` +- `editor/src/RustGoConstructiveEditAdapter.h` +- `editor/src/PythonTypeScriptConstructiveEditAdapter.h` +- `editor/src/AdapterOperationUtil.h` + +Deliverables: +- `whetstone_run_constructive_step` uses language-specific adapter execution. +- `whetstone_get_constructive_status` returns real adapter diagnostics. +- `whetstone_run_constructive_loop` records per-stage real outcomes. + +Validation: +- Existing step tests: `1763`, `1764`, `1765`. +- New integration test family: constructive step with real adapter outputs. + +### Phase B: Real Sync/Regenerate/Merge Path + +Primary files: +- `editor/src/mcp/RegisterSprint142Tools.h` +- `editor/src/mcp/RegisterSprint143Tools.h` +- `editor/src/mcp/RegisterSprint144Tools.h` +- `editor/src/TextASTSync.h` +- `editor/src/graduation/CppTextDeltaParserBridgeModel.h` +- `editor/src/graduation/ConflictRegionDetectorModel.h` +- `editor/src/graduation/MergePolicyEngineModel.h` + +Deliverables: +- Sync tools call real text->AST machinery for supported languages. +- Regeneration tools call real AST->text generator path. +- Merge tools emit real conflict regions and policy outcomes. + +Validation: +- Existing step tests: `1663`-`1665`, `1673`-`1675`, `1683`-`1685`. +- New cross-language sync/regenerate/merge integration tests. + +### Phase C: Real Toolchain Provider and Diagnostic Integration + +Primary files: +- `editor/src/mcp/RegisterSprint151Tools.h` +- `editor/src/BuildSystem.h` +- `editor/src/Diagnostics.h` +- `editor/src/StructuredDiagnostics.h` +- `editor/src/LspOps.h` +- `editor/src/graduation/DiagnosticNormalizationCanonicalModel.h` + +Deliverables: +- `whetstone_list_toolchain_providers` reflects runtime-detected providers. +- `whetstone_probe_toolchain_provider` executes probe checks. +- `whetstone_normalize_diagnostics` ingests raw provider/LSP output. + +Validation: +- Existing step tests: `1753`, `1754`, `1755`. +- New tests with representative toolchain output fixtures. + +### Phase D: Replay/Transaction/GA Evidence from Real Runs + +Primary files: +- `editor/src/mcp/RegisterSprint153Tools.h` +- `editor/src/mcp/RegisterSprint154Tools.h` +- `editor/src/mcp/RegisterSprint155Tools.h` +- `editor/src/graduation/ConstructiveTransactionSchema.h` +- `editor/src/graduation/ConstructiveReleaseCertificationPacketModel.h` +- `editor/src/graduation/MCPReplayDiagnosticsArtifact.h` + +Deliverables: +- Replay suite compares real run traces. +- Transaction resume/rollback replays actual state transitions. +- GA gate evaluates real determinism and recovery evidence. + +Validation: +- Existing step tests: `1773`-`1775`, `1783`-`1785`, `1793`-`1795`. +- New end-to-end replay and rollback scenario tests. + +## Taskitem Execution Policy for This Plan + +- For each implementation phase, generate taskitems using: + - `tools/mcp/run_sprint_taskitem_pipeline.sh ` +- Export each run: + - `tools/mcp/export_taskitem_run_for_lora.sh ` +- Keep `training_data/lora/taskitem_pipeline_runs.jsonl` append-only. + +## Immediate Next Slice + +Start Phase A: +- Bind `whetstone_run_constructive_step` to concrete language adapters. +- Keep fallback path for unsupported operations. +- Add targeted integration tests for `cpp`, `rust`, `python`, `typescript`. + +Parallel planning tranche (active): +- `sprint228_plan.md` to `sprint231_plan.md` +- baseline artifacts: + - `logs/taskitem_runs/spec_planning_baseline_20260226/fullstack_spec_readiness.json` + - `logs/taskitem_runs/spec_planning_baseline_20260226/deterministic_spec_readiness.json` diff --git a/docs/spec_planning_baseline_2026-02-26.md b/docs/spec_planning_baseline_2026-02-26.md new file mode 100644 index 0000000..8dfee45 --- /dev/null +++ b/docs/spec_planning_baseline_2026-02-26.md @@ -0,0 +1,26 @@ +# Spec Planning Baseline - 2026-02-26 + +Run artifacts: +- `logs/taskitem_runs/spec_planning_baseline_20260226/fullstack_spec_readiness.json` +- `logs/taskitem_runs/spec_planning_baseline_20260226/deterministic_spec_readiness.json` + +## Results + +- `challenging_fullstack_multifile_2026-02-26.md` + - verdict: `needs_spec_hardening` + - score: `15/100` +- `challenging_deterministic_projects_2026-02-26.md` + - verdict: `needs_spec_hardening` + - score: `8/100` + +## Main Missing Planning Inputs + +- explicit constraints section +- acceptance criteria section +- environment/projection section +- security/performance/observability/rollout sections +- quantitative budgets and test scenarios + +## Interpretation + +These docs are benchmark catalogs, not execution-ready specs. The new planning layer correctly flags that they require hardening before direct taskitem execution. diff --git a/docs/sprint228_231_execution_tracker_2026-02-26.md b/docs/sprint228_231_execution_tracker_2026-02-26.md new file mode 100644 index 0000000..32d91b1 --- /dev/null +++ b/docs/sprint228_231_execution_tracker_2026-02-26.md @@ -0,0 +1,40 @@ +# Sprint 228-231 Execution Tracker - 2026-02-26 + +## Scope +Planned and started: +- `sprint228_plan.md` +- `sprint229_plan.md` +- `sprint230_plan.md` +- `sprint231_plan.md` + +## Implemented So Far + +- Added planning-readiness tool: + - `tools/mcp/spec_planning_readiness.py` +- Tool outputs: + - readiness score (`section_score`, `keyword_score`, `total`) + - verdict (`execution_ready` or `needs_spec_hardening`) + - missing-action list for planning gaps + - recommended markdown template blocks for missing sections + +## Baseline Evidence + +- `logs/taskitem_runs/spec_planning_baseline_20260226/fullstack_spec_readiness.json` +- `logs/taskitem_runs/spec_planning_baseline_20260226/deterministic_spec_readiness.json` + +Observed baseline: +- both sample docs currently return `needs_spec_hardening` +- low readiness scores indicate missing first-class sections for constraints/acceptance/environment/security/performance. + +## Explicit Completion Signal + +- Sprint 228: `PARTIAL` (tool implemented and baseline executed) +- Sprint 229: `PARTIAL` (constraint synthesis mapping implemented in tool) +- Sprint 230: `PARTIAL` (acceptance readiness checks implemented in tool) +- Sprint 231: `PARTIAL` (environment/projection checks implemented in tool) + +## Next Closure Work + +- Bind readiness tool into pre-taskitem pipeline gate. +- Add structured spec hardening pass that applies template scaffolds to candidate specs. +- Re-run benchmarks using hardened specs and measure downstream readiness deltas. diff --git a/editor/src/Sprint228IntegrationSummary.h b/editor/src/Sprint228IntegrationSummary.h new file mode 100644 index 0000000..4169e51 --- /dev/null +++ b/editor/src/Sprint228IntegrationSummary.h @@ -0,0 +1,4 @@ +#pragma once + +// Sprint 228 integration summary: +// - Added deterministic markdown spec readiness scoring tool for pre-execution planning quality. diff --git a/editor/src/Sprint229IntegrationSummary.h b/editor/src/Sprint229IntegrationSummary.h new file mode 100644 index 0000000..c2d055d --- /dev/null +++ b/editor/src/Sprint229IntegrationSummary.h @@ -0,0 +1,4 @@ +#pragma once + +// Sprint 229 integration summary: +// - Added missing-constraint remediation mapping and scaffold-template synthesis in readiness outputs. diff --git a/editor/src/Sprint230IntegrationSummary.h b/editor/src/Sprint230IntegrationSummary.h new file mode 100644 index 0000000..2de57fc --- /dev/null +++ b/editor/src/Sprint230IntegrationSummary.h @@ -0,0 +1,4 @@ +#pragma once + +// Sprint 230 integration summary: +// - Added acceptance-readiness checks and scenario skeleton recommendations. diff --git a/editor/src/Sprint231IntegrationSummary.h b/editor/src/Sprint231IntegrationSummary.h new file mode 100644 index 0000000..f24e011 --- /dev/null +++ b/editor/src/Sprint231IntegrationSummary.h @@ -0,0 +1,4 @@ +#pragma once + +// Sprint 231 integration summary: +// - Added environment/projection planning coverage checks and cross-target scaffold guidance. diff --git a/sprint228_plan.md b/sprint228_plan.md new file mode 100644 index 0000000..e1a1313 --- /dev/null +++ b/sprint228_plan.md @@ -0,0 +1,11 @@ +# Sprint 228 Plan: Spec Readiness Scoring + +## Goal +Make spec-to-execution readiness measurable before taskitem generation. + +## Steps +- Step 2179: Add markdown spec readiness assessor with deterministic section/keyword checks. +- Step 2180: Emit missing-action list for execution blockers. +- Step 2181: Emit recommended spec template blocks for missing sections. +- Step 2182: Run readiness baseline on current challenging spec docs. +- Step 2183: Add `Sprint228IntegrationSummary.h`. diff --git a/sprint229_plan.md b/sprint229_plan.md new file mode 100644 index 0000000..43a535e --- /dev/null +++ b/sprint229_plan.md @@ -0,0 +1,11 @@ +# Sprint 229 Plan: Constraint Synthesis Scaffolding + +## Goal +Produce concrete constraint scaffolding from incomplete specs. + +## Steps +- Step 2184: Add canonical missing-constraint action mapping (security/perf/rollout/environment). +- Step 2185: Generate remediation template snippets keyed to missing sections. +- Step 2186: Persist suggested scaffold in machine-readable outputs. +- Step 2187: Add baseline artifact capture under `logs/taskitem_runs/spec_planning_*`. +- Step 2188: Add `Sprint229IntegrationSummary.h`. diff --git a/sprint230_plan.md b/sprint230_plan.md new file mode 100644 index 0000000..6c31796 --- /dev/null +++ b/sprint230_plan.md @@ -0,0 +1,11 @@ +# Sprint 230 Plan: Acceptance Readiness Derivation + +## Goal +Force acceptance-test shape coverage before execution planning. + +## Steps +- Step 2189: Detect absence of acceptance/done criteria in specs. +- Step 2190: Emit acceptance scenario skeletons in recommendations. +- Step 2191: Add test/readiness keyword gate to scoring model. +- Step 2192: Validate against existing challenging spec docs. +- Step 2193: Add `Sprint230IntegrationSummary.h`. diff --git a/sprint231_plan.md b/sprint231_plan.md new file mode 100644 index 0000000..2f3288b --- /dev/null +++ b/sprint231_plan.md @@ -0,0 +1,11 @@ +# Sprint 231 Plan: Environment/Projection Planning Contract + +## Goal +Treat environment/projection constraints as first-class planning inputs. + +## Steps +- Step 2194: Add environment/projection section detection and missing-action synthesis. +- Step 2195: Add cross-target invariant placeholder generation in recommended template. +- Step 2196: Score spec readiness penalty when projection profile is absent. +- Step 2197: Capture baseline deltas for fullstack and deterministic catalogs. +- Step 2198: Add `Sprint231IntegrationSummary.h`. diff --git a/tools/mcp/spec_planning_readiness.py b/tools/mcp/spec_planning_readiness.py new file mode 100755 index 0000000..e968cc1 --- /dev/null +++ b/tools/mcp/spec_planning_readiness.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import json +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, List + + +SECTION_PATTERNS = { + "goals": [r"^#*\s*goals?\b", r"^#*\s*objective\b", r"^#*\s*problem\b"], + "constraints": [r"^#*\s*constraints?\b", r"^#*\s*non-functional\b", r"^#*\s*requirements\b"], + "acceptance": [r"^#*\s*acceptance\b", r"^#*\s*done criteria\b", r"^#*\s*definition of done\b"], + "environment": [r"^#*\s*environment\b", r"^#*\s*target(s)?\b", r"^#*\s*platform\b"], + "rollout": [r"^#*\s*rollout\b", r"^#*\s*migration\b", r"^#*\s*release\b"], + "observability": [r"^#*\s*observability\b", r"^#*\s*telemetry\b", r"^#*\s*monitoring\b"], + "security": [r"^#*\s*security\b", r"^#*\s*auth\b", r"^#*\s*threat\b"], + "performance": [r"^#*\s*performance\b", r"^#*\s*slo\b", r"^#*\s*latency\b"], +} + +KEYWORD_PATTERNS = { + "numeric_budget": [r"\b\d+\s*(ms|s|sec|mb|gb|rps|qps|%)\b", r"\bp\d{2}\b", r"\berror budget\b"], + "determinism": [r"\bdetermin(istic|ism)\b", r"\breplay\b", r"\brollback\b", r"\bidempotent\b"], + "multifile": [r"\bmulti[- ]file\b", r"\bartifact\b", r"\bopenapi\b", r"\bmigration\b"], + "tests": [r"\btest\b", r"\bassert\b", r"\bscenario\b", r"\bacceptance criteria\b"], +} + + +@dataclass +class Signal: + key: str + present: bool + evidence: List[str] + + +def find_section_signals(lines: List[str]) -> List[Signal]: + out: List[Signal] = [] + for key, pats in SECTION_PATTERNS.items(): + evidence: List[str] = [] + for line in lines: + for pat in pats: + if re.search(pat, line.strip(), flags=re.IGNORECASE): + evidence.append(line.strip()) + break + if evidence: + break + out.append(Signal(key=key, present=bool(evidence), evidence=evidence)) + return out + + +def find_keyword_signals(text: str) -> List[Signal]: + out: List[Signal] = [] + for key, pats in KEYWORD_PATTERNS.items(): + matches: List[str] = [] + for pat in pats: + m = re.search(pat, text, flags=re.IGNORECASE) + if m: + matches.append(m.group(0)) + out.append(Signal(key=key, present=bool(matches), evidence=matches[:5])) + return out + + +def score(section_signals: List[Signal], keyword_signals: List[Signal]) -> Dict[str, int]: + section_count = sum(1 for s in section_signals if s.present) + keyword_count = sum(1 for s in keyword_signals if s.present) + section_score = int(round((section_count / max(1, len(section_signals))) * 70)) + keyword_score = int(round((keyword_count / max(1, len(keyword_signals))) * 30)) + total = min(100, section_score + keyword_score) + return { + "section_score": section_score, + "keyword_score": keyword_score, + "total": total, + } + + +def build_missing_actions(section_signals: List[Signal], keyword_signals: List[Signal]) -> List[Dict[str, str]]: + missing: List[Dict[str, str]] = [] + section_actions = { + "goals": "Add explicit product/behavior goals and out-of-scope boundaries.", + "constraints": "Add hard constraints: language/runtime limits, prohibited APIs, determinism and safety policies.", + "acceptance": "Add acceptance criteria with pass/fail checks and expected artifacts.", + "environment": "Add environment targets/projections and per-target constraints.", + "rollout": "Add rollout/migration/rollback choreography and compatibility window.", + "observability": "Add logging/metrics/tracing expectations and alert thresholds.", + "security": "Add threat/security constraints, default-deny policy, and authn/authz expectations.", + "performance": "Add quantitative budgets (latency, memory, throughput) and regression gates.", + } + for s in section_signals: + if not s.present: + missing.append({"type": "section", "key": s.key, "action": section_actions[s.key]}) + + keyword_actions = { + "numeric_budget": "Add explicit numeric budgets (p95 latency, memory caps, error budgets).", + "determinism": "Add determinism/replay/rollback/idempotency requirements.", + "multifile": "Add required multi-file artifact closure list and dependency graph hints.", + "tests": "Add concrete acceptance-test scenarios and expected outcomes.", + } + for s in keyword_signals: + if not s.present: + missing.append({"type": "keyword", "key": s.key, "action": keyword_actions[s.key]}) + + return missing + + +def template_for_missing(missing: List[Dict[str, str]]) -> str: + keys = {m["key"] for m in missing} + blocks: List[str] = [] + if "goals" in keys: + blocks.append("## Goals\n- Primary objective:\n- Secondary objective:\n- Explicitly out of scope:") + if "constraints" in keys: + blocks.append("## Constraints\n- Language/runtime constraints:\n- Forbidden dependencies/APIs:\n- Determinism/replay requirements:") + if "environment" in keys: + blocks.append("## Environment / Projection Targets\n- Target A:\n- Target B:\n- Cross-target invariants:") + if "performance" in keys or "numeric_budget" in keys: + blocks.append("## Performance Budgets\n- p95 latency: \n- Max memory: \n- Throughput floor: ") + if "security" in keys: + blocks.append("## Security\n- Default deny policy:\n- AuthN/AuthZ model:\n- Sensitive data handling:") + if "observability" in keys: + blocks.append("## Observability\n- Required metrics:\n- Required logs/traces:\n- Alert thresholds:") + if "rollout" in keys: + blocks.append("## Rollout / Migration / Rollback\n- Compatibility window:\n- Migration strategy:\n- Rollback trigger + procedure:") + if "acceptance" in keys or "tests" in keys: + blocks.append("## Acceptance Criteria\n- Scenario 1: given/when/then\n- Scenario 2: given/when/then\n- Required artifacts and pass conditions:") + if "multifile" in keys: + blocks.append("## Required Artifacts\n- file/path/a\n- file/path/b\n- file/path/c") + return "\n\n".join(blocks) + + +def assess(path: Path) -> Dict[str, object]: + text = path.read_text(encoding="utf-8") + lines = text.splitlines() + section_signals = find_section_signals(lines) + keyword_signals = find_keyword_signals(text) + scores = score(section_signals, keyword_signals) + missing = build_missing_actions(section_signals, keyword_signals) + + verdict = "execution_ready" if scores["total"] >= 85 and not missing else "needs_spec_hardening" + return { + "file": str(path), + "verdict": verdict, + "scores": scores, + "sections": [{"key": s.key, "present": s.present, "evidence": s.evidence} for s in section_signals], + "keywords": [{"key": s.key, "present": s.present, "evidence": s.evidence} for s in keyword_signals], + "missing_actions": missing, + "recommended_template": template_for_missing(missing), + } + + +def main() -> None: + ap = argparse.ArgumentParser(description="Assess markdown spec execution-readiness and suggest missing constraints.") + ap.add_argument("--spec", required=True, help="Path to markdown spec") + ap.add_argument("--out", required=True, help="Output JSON file") + args = ap.parse_args() + + result = assess(Path(args.spec)) + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + out.write_text(json.dumps(result, indent=2) + "\n", encoding="utf-8") + print(json.dumps(result, indent=2)) + + +if __name__ == "__main__": + main()