From 5feb05e900d6b712d5048992ce89c75e3e4ed648 Mon Sep 17 00:00:00 2001 From: Bill Date: Thu, 26 Feb 2026 15:59:33 -0700 Subject: [PATCH] Add raw gap backlog synthesis from ladder runs (sprint 254) --- docs/constructive_editing_runtime_plan.md | 4 +- ...rator_readiness_gap_registry_2026-02-26.md | 2 +- docs/progress_log_2026-02-26.md | 15 +++ .../sprint254_execution_tracker_2026-02-26.md | 32 +++++ editor/src/Sprint254IntegrationSummary.h | 6 + sprint254_plan.md | 11 ++ ...run_native_profile_closure_ladder_batch.sh | 11 +- tools/mcp/synthesize_raw_gap_backlog.py | 121 ++++++++++++++++++ 8 files changed, 199 insertions(+), 3 deletions(-) create mode 100644 docs/sprint254_execution_tracker_2026-02-26.md create mode 100644 editor/src/Sprint254IntegrationSummary.h create mode 100644 sprint254_plan.md create mode 100755 tools/mcp/synthesize_raw_gap_backlog.py diff --git a/docs/constructive_editing_runtime_plan.md b/docs/constructive_editing_runtime_plan.md index 176b5ff..1269439 100644 --- a/docs/constructive_editing_runtime_plan.md +++ b/docs/constructive_editing_runtime_plan.md @@ -150,7 +150,7 @@ Parallel planning tranche (active): - `logs/taskitem_runs/TEST_ONLY_spec_planning_baseline_20260226/deterministic_spec_readiness.json` Planning runtime controls (active): -- `sprint232_plan.md` to `sprint253_plan.md` +- `sprint232_plan.md` to `sprint254_plan.md` - semantic bridge + intake augmentation + requirement injection + expansion gating are wired into: - `tools/mcp/run_sprint_taskitem_pipeline.sh` - current policy default is native-first semantic fallback: @@ -210,3 +210,5 @@ Planning runtime controls (active): - closure ladder outcomes can be aggregated in batch: - `tools/mcp/run_native_profile_closure_ladder_batch.sh` - `tools/mcp/analyze_closure_ladder_outcomes.py` +- raw gap backlog can be synthesized from ladder raw-only attempts: + - `tools/mcp/synthesize_raw_gap_backlog.py` diff --git a/docs/generator_readiness_gap_registry_2026-02-26.md b/docs/generator_readiness_gap_registry_2026-02-26.md index be30552..389eb59 100644 --- a/docs/generator_readiness_gap_registry_2026-02-26.md +++ b/docs/generator_readiness_gap_registry_2026-02-26.md @@ -47,7 +47,7 @@ This is the canonical dated registry for "not production-ready" generator gaps. | GR-018 | Performance/security/rollout constrained refactor enforcement gap | `docs/gap_hunt_fullstack_multifile_2026-02-26.md`, `logs/taskitem_runs/challenging_fullstack_multifile_20260226_r5/results.jsonl` | `partial` | Hard checks now enforce migration rollback + data-loss policy, security deny-by-default, SLO p95 presence, and rollout staged+abort policy. Enforcement is contract-level; generator capability under these constraints is still weak in C++ AB path. | Constraint-aware generation follow-up | | GR-019 | Parity-blocked readiness load (gating without capability closure) | `logs/taskitem_runs/challenging_fullstack_multifile_20260226_r7/summary.json`, `logs/taskitem_runs/challenging_subset_prod_20260226_r7/summary.json`, `docs/sprint225_227_execution_tracker_2026-02-26.md` | `partial` | Sprint 225-227 reduced blocked parity load from `6` to `0` on both tracked hard catalogs while keeping unresolved divergence at `0`. This closes immediate safety debt for current corpora, but robustness is still contingent on pattern-driven repair classes. | Generalize repairs beyond queue-shaped transpile outputs | | GR-020 | Semantic fallback overuse masks weak native decomposition | `logs/taskitem_runs/TEST_ONLY_sprint236_semantic_fallback_audit_20260226/semantic_fallback_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint237_semantic_fallback_gate_fail_20260226/semantic_fallback_budget_gate.json`, `logs/taskitem_runs/TEST_ONLY_sprint238_native_gate_fail_20260226.json`, `logs/taskitem_runs/TEST_ONLY_sprint239_semantic_fallback_audit_20260226/semantic_fallback_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint240_retry_20260226_145631/00_summary.json`, `docs/sprint236_execution_tracker_2026-02-26.md`, `docs/sprint237_execution_tracker_2026-02-26.md`, `docs/sprint238_execution_tracker_2026-02-26.md`, `docs/sprint239_execution_tracker_2026-02-26.md`, `docs/sprint240_execution_tracker_2026-02-26.md` | `partial` | Sprint 236 added deterministic fallback-gap auditing with tool + constraint metadata. Sprint 237 added fallback-budget hard gate (`max-fallback-rate`) and produced expected fail/pass artifacts. Sprint 238 added native decomposition hard gate in pipeline (`min task count`, `min semantic signal count`) so weak native generation can be blocked before fallback masking. Sprint 239 added native reason enrichment and improved semantic signal density (`0 -> 6`). Sprint 240 added native decomposition retry with explicit minimum-task policy; on current sample retry attempted but did not improve depth (`2 -> 2`). | Native decomposition task-depth upgrade (increase native task granularity beyond 2) | -| GR-021 | Impact-specific native decomposition coverage not enforced uniformly | `docs/native_decomposition_impact_list_2026-02-26.md`, `tools/mcp/profiles/native_decomposition_impact_profiles.json`, `logs/taskitem_runs/TEST_ONLY_sprint241_fullstack_impact_20260226_150416/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint241_native_impact_aggregate_20260226/native_impact_coverage_aggregate.json`, `logs/taskitem_runs/TEST_ONLY_sprint242_impact_remediation_loop_20260226/remediation_loop_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint243_impact_remediation_tasks_loop_20260226_r2/remediation_loop_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint244_autofill_20260226_151651/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint244_autofill_enforce_20260226_151709/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint245_intrinsic_20260226_151835/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint246_multishot_20260226_153033/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint246_multishot_enforce_20260226_153045/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint247_singleshot_20260226_153230/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint247_singleshot_enforce_20260226_153241/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint248_rawsearch_20260226_153903/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint249_rawguard_20260226_154028/02ae_raw_candidate_search.json`, `logs/taskitem_runs/TEST_ONLY_sprint250_closure_ladder_20260226/closure_ladder_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint251_rawvariants_20260226_154355/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint252_closure_ladder_policy_skip2_20260226/closure_ladder_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint253_closure_ladder_batch_20260226/closure_ladder_batch_summary.json` | `partial` | Sprint 241 introduced impact profiles/gates; 242-243 remediation loops; 244 first-pass autofill; 245 intrinsic constraints only (no uplift); 246 multishot closure; 247 single-shot shaping closure; 248 raw candidate search no uplift; 249 no-uplift guardrail; 250 closure ladder; 251 richer raw variants no uplift; 252 history-aware ladder routing; 253 batch ladder analytics (`single_shot_shape=2`, `multishot=1`, `ok=3`). Residual gap remains in raw generator single-shot quality without overlays. | Improve raw single-shot generator output so profile closure does not rely on shaping/multishot/autofill overlays | +| GR-021 | Impact-specific native decomposition coverage not enforced uniformly | `docs/native_decomposition_impact_list_2026-02-26.md`, `tools/mcp/profiles/native_decomposition_impact_profiles.json`, `logs/taskitem_runs/TEST_ONLY_sprint241_fullstack_impact_20260226_150416/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint241_native_impact_aggregate_20260226/native_impact_coverage_aggregate.json`, `logs/taskitem_runs/TEST_ONLY_sprint242_impact_remediation_loop_20260226/remediation_loop_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint243_impact_remediation_tasks_loop_20260226_r2/remediation_loop_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint244_autofill_20260226_151651/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint244_autofill_enforce_20260226_151709/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint245_intrinsic_20260226_151835/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint246_multishot_20260226_153033/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint246_multishot_enforce_20260226_153045/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint247_singleshot_20260226_153230/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint247_singleshot_enforce_20260226_153241/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint248_rawsearch_20260226_153903/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint249_rawguard_20260226_154028/02ae_raw_candidate_search.json`, `logs/taskitem_runs/TEST_ONLY_sprint250_closure_ladder_20260226/closure_ladder_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint251_rawvariants_20260226_154355/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint252_closure_ladder_policy_skip2_20260226/closure_ladder_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint253_closure_ladder_batch_20260226/closure_ladder_batch_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint254_closure_ladder_batch_20260226_r2/raw_gap_backlog.json` | `partial` | Sprint 241 introduced impact profiles/gates; 242-243 remediation loops; 244 first-pass autofill; 245 intrinsic constraints only (no uplift); 246 multishot closure; 247 single-shot shaping closure; 248 raw candidate search no uplift; 249 no-uplift guardrail; 250 closure ladder; 251 richer raw variants no uplift; 252 history-aware ladder routing; 253 batch ladder analytics; 254 raw gap backlog synthesis from `raw_only` attempts identifies top missing raw signals (e.g., validate_taskitem prereq, deterministic contract fields). Residual gap remains in raw generator single-shot quality without overlays. | Target top raw-gap backlog signals in generator output: prerequisite ops coverage + deterministic executionContract completeness + risk/contract reason density | ## What Was Covered Today (Sprints 175-184) diff --git a/docs/progress_log_2026-02-26.md b/docs/progress_log_2026-02-26.md index 84a97dc..135a6fb 100644 --- a/docs/progress_log_2026-02-26.md +++ b/docs/progress_log_2026-02-26.md @@ -376,3 +376,18 @@ - `selected_mode_counts.single_shot_shape=2` - `selected_mode_counts.multishot=1` - `status_counts.ok=3` + +## Sprint 254 Added (Same Day) + +- Added prioritized raw-gap backlog synthesis from closure ladder runs: + - `tools/mcp/synthesize_raw_gap_backlog.py` + - now mines `raw_only` attempt artifacts for gap signals +- Batch ladder runner now emits: + - `raw_gap_backlog.json` + - `raw_gap_backlog.md` +- Dated validation artifact: + - `logs/taskitem_runs/TEST_ONLY_sprint254_closure_ladder_batch_20260226_r2/raw_gap_backlog.json` +- Current top missing signals in sampled backlog: + - `missing_prerequisite_op:whetstone_validate_taskitem` + - `missing_execution_contract:deterministic` + - `missing_prerequisite_op:whetstone_queue_ready` diff --git a/docs/sprint254_execution_tracker_2026-02-26.md b/docs/sprint254_execution_tracker_2026-02-26.md new file mode 100644 index 0000000..117f368 --- /dev/null +++ b/docs/sprint254_execution_tracker_2026-02-26.md @@ -0,0 +1,32 @@ +# Sprint 254 Execution Tracker - 2026-02-26 + +## Scope +- `sprint254_plan.md` + +## Implemented + +New tool: +- `tools/mcp/synthesize_raw_gap_backlog.py` + - Aggregates closure ladder run artifacts into prioritized raw-gap signals. + - Mines `raw_only` attempt coverage checks where available. + +Batch runner integration: +- `tools/mcp/run_native_profile_closure_ladder_batch.sh` + - now emits: + - `raw_gap_backlog.json` + - `raw_gap_backlog.md` + +## Validation Artifact + +- `logs/taskitem_runs/TEST_ONLY_sprint254_closure_ladder_batch_20260226_r2/raw_gap_backlog.json` + +Observed top missing signals (sample batch): +- `missing_prerequisite_op:whetstone_validate_taskitem` +- `missing_execution_contract:deterministic` +- `missing_prerequisite_op:whetstone_queue_ready` +- `missing_execution_contract:rollbackRequired` +- `missing_execution_contract:replayValidationRequired` + +## Explicit Completion Signal + +- Sprint 254: `DONE` (implemented + prioritized raw-gap backlog verified) diff --git a/editor/src/Sprint254IntegrationSummary.h b/editor/src/Sprint254IntegrationSummary.h new file mode 100644 index 0000000..45dedac --- /dev/null +++ b/editor/src/Sprint254IntegrationSummary.h @@ -0,0 +1,6 @@ +#pragma once + +// Sprint 254 integration summary: +// - Added raw gap backlog synthesis from closure ladder run artifacts. +// - Backlog prioritization now mines raw-only failure signals. +// - Batch ladder runner emits backlog JSON/Markdown artifacts automatically. diff --git a/sprint254_plan.md b/sprint254_plan.md new file mode 100644 index 0000000..ee9da1c --- /dev/null +++ b/sprint254_plan.md @@ -0,0 +1,11 @@ +# Sprint 254 Plan: Raw Gap Backlog Synthesis from Closure Ladder Runs + +## Goal +Create a prioritized raw-generator gap backlog directly from closure ladder attempts, so next intrinsic work targets the highest-frequency missing signals. + +## Steps +- Step 2315: Add backlog synthesizer over ladder run outputs. +- Step 2316: Mine `raw_only` attempt artifacts (not just selected passing runs). +- Step 2317: Integrate backlog generation into ladder batch runner. +- Step 2318: Emit JSON + Markdown backlog artifacts. +- Step 2319: Add `Sprint254IntegrationSummary.h` and execution tracker. diff --git a/tools/mcp/run_native_profile_closure_ladder_batch.sh b/tools/mcp/run_native_profile_closure_ladder_batch.sh index 3c88864..0c19858 100755 --- a/tools/mcp/run_native_profile_closure_ladder_batch.sh +++ b/tools/mcp/run_native_profile_closure_ladder_batch.sh @@ -32,4 +32,13 @@ python3 "$ROOT_DIR/tools/mcp/analyze_closure_ladder_outcomes.py" \ --include-glob "*" \ --out "$OUT_DIR/closure_ladder_batch_summary.json" >/dev/null -jq -n --arg out_dir "$OUT_DIR" --argjson summary "$(cat "$OUT_DIR/closure_ladder_batch_summary.json")" '{status:"ok", out_dir:$out_dir, summary:$summary}' +python3 "$ROOT_DIR/tools/mcp/synthesize_raw_gap_backlog.py" \ + --batch-dir "$OUT_DIR" \ + --out-json "$OUT_DIR/raw_gap_backlog.json" \ + --out-md "$OUT_DIR/raw_gap_backlog.md" >/dev/null + +jq -n \ + --arg out_dir "$OUT_DIR" \ + --argjson summary "$(cat "$OUT_DIR/closure_ladder_batch_summary.json")" \ + --argjson backlog "$(cat "$OUT_DIR/raw_gap_backlog.json")" \ + '{status:"ok", out_dir:$out_dir, summary:$summary, backlog:$backlog}' diff --git a/tools/mcp/synthesize_raw_gap_backlog.py b/tools/mcp/synthesize_raw_gap_backlog.py new file mode 100755 index 0000000..a1e0006 --- /dev/null +++ b/tools/mcp/synthesize_raw_gap_backlog.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +import argparse +import json +from collections import Counter, defaultdict +from pathlib import Path +from typing import Dict, List + + +def load_json(path: Path): + with path.open("r", encoding="utf-8") as f: + return json.load(f) + + +def main() -> int: + p = argparse.ArgumentParser(description="Synthesize prioritized raw-generator gap backlog from closure ladder outputs.") + p.add_argument("--batch-dir", required=True) + p.add_argument("--out-json", required=True) + p.add_argument("--out-md", required=True) + args = p.parse_args() + + batch_dir = Path(args.batch_dir) + ladder_files = sorted(batch_dir.glob("*/closure_ladder_summary.json")) + + mode_counts = Counter() + missing_counter = Counter() + by_profile_missing = defaultdict(Counter) + spec_rows: List[Dict] = [] + + for lf in ladder_files: + try: + summary = load_json(lf) + except Exception: + continue + mode = str(summary.get("selected_mode", "")) + mode_counts[mode] += 1 + + selected_run = str(summary.get("selected_run", "")) + cov_checks = [] + raw_run = "" + raw_rc = "" + # Prefer raw-only attempt artifacts for gap mining. + raw_run_file = lf.parent / "raw_only.run_dir" + raw_rc_file = lf.parent / "raw_only.rc" + if raw_run_file.exists(): + raw_run = raw_run_file.read_text(encoding="utf-8", errors="ignore").strip() + if raw_rc_file.exists(): + raw_rc = raw_rc_file.read_text(encoding="utf-8", errors="ignore").strip() + target_run = raw_run if raw_run else selected_run + if target_run: + sp = Path(target_run) / "00_summary.json" + if sp.exists(): + try: + s = load_json(sp) + cov = (s.get("native_impact_coverage") or {}) + cov_checks = list(cov.get("checks") or []) + except Exception: + cov_checks = [] + + missing_for_spec = [] + for c in cov_checks: + pid = str(c.get("id", "unknown")) + for m in (c.get("missing") or []): + m = str(m) + missing_counter[m] += 1 + by_profile_missing[pid][m] += 1 + missing_for_spec.append({"profile": pid, "missing": m}) + + spec_rows.append( + { + "run_id": lf.parent.name, + "selected_mode": mode, + "selected_run": selected_run, + "raw_only_run": raw_run, + "raw_only_rc": raw_rc, + "missing_items": missing_for_spec, + } + ) + + prioritized_missing = [ + {"missing": k, "count": int(v)} + for k, v in missing_counter.most_common() + ] + + profile_priorities = {} + for pid, ctr in by_profile_missing.items(): + profile_priorities[pid] = [ + {"missing": k, "count": int(v)} for k, v in ctr.most_common() + ] + + out = { + "status": "ok", + "record_count": len(spec_rows), + "selected_mode_counts": dict(mode_counts), + "prioritized_missing": prioritized_missing, + "profile_priorities": profile_priorities, + "records": spec_rows, + } + + with Path(args.out_json).open("w", encoding="utf-8") as f: + json.dump(out, f, indent=2, sort_keys=True) + f.write("\n") + + with Path(args.out_md).open("w", encoding="utf-8") as f: + f.write("# Raw Generator Gap Backlog\n\n") + f.write(f"- Record count: {len(spec_rows)}\n") + f.write("- Selected mode counts:\n") + for mode, count in mode_counts.items(): + f.write(f" - {mode}: {count}\n") + f.write("\n## Prioritized Missing Signals\n\n") + if not prioritized_missing: + f.write("- None\n") + else: + for row in prioritized_missing: + f.write(f"- {row['missing']}: {row['count']}\n") + + print(json.dumps({"status": "ok", "record_count": len(spec_rows), "out_json": args.out_json, "out_md": args.out_md}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())