From 2c422feddab8c1f5236599dfa9daa1c0df39593d Mon Sep 17 00:00:00 2001 From: Bill Date: Thu, 26 Feb 2026 15:42:35 -0700 Subject: [PATCH] Add automatic native profile closure ladder runner (sprint 250) --- docs/constructive_editing_runtime_plan.md | 4 +- ...rator_readiness_gap_registry_2026-02-26.md | 2 +- docs/progress_log_2026-02-26.md | 11 ++ .../sprint250_execution_tracker_2026-02-26.md | 32 ++++++ editor/src/Sprint250IntegrationSummary.h | 6 ++ sprint250_plan.md | 11 ++ .../mcp/run_native_profile_closure_ladder.sh | 101 ++++++++++++++++++ 7 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 docs/sprint250_execution_tracker_2026-02-26.md create mode 100644 editor/src/Sprint250IntegrationSummary.h create mode 100644 sprint250_plan.md create mode 100755 tools/mcp/run_native_profile_closure_ladder.sh diff --git a/docs/constructive_editing_runtime_plan.md b/docs/constructive_editing_runtime_plan.md index bbf218f..2d41a19 100644 --- a/docs/constructive_editing_runtime_plan.md +++ b/docs/constructive_editing_runtime_plan.md @@ -150,7 +150,7 @@ Parallel planning tranche (active): - `logs/taskitem_runs/TEST_ONLY_spec_planning_baseline_20260226/deterministic_spec_readiness.json` Planning runtime controls (active): -- `sprint232_plan.md` to `sprint249_plan.md` +- `sprint232_plan.md` to `sprint250_plan.md` - semantic bridge + intake augmentation + requirement injection + expansion gating are wired into: - `tools/mcp/run_sprint_taskitem_pipeline.sh` - current policy default is native-first semantic fallback: @@ -202,3 +202,5 @@ Planning runtime controls (active): - `WSTONE_NATIVE_RAW_CANDIDATE_MAX_VARIANTS` - `WSTONE_NATIVE_RAW_CANDIDATE_REQUIRE_UPLIFT` - `tools/mcp/score_native_tasks_profile_coverage.py` +- closure ladder can select the minimal passing strategy automatically: + - `tools/mcp/run_native_profile_closure_ladder.sh` diff --git a/docs/generator_readiness_gap_registry_2026-02-26.md b/docs/generator_readiness_gap_registry_2026-02-26.md index 250eee3..cdac8a8 100644 --- a/docs/generator_readiness_gap_registry_2026-02-26.md +++ b/docs/generator_readiness_gap_registry_2026-02-26.md @@ -47,7 +47,7 @@ This is the canonical dated registry for "not production-ready" generator gaps. | GR-018 | Performance/security/rollout constrained refactor enforcement gap | `docs/gap_hunt_fullstack_multifile_2026-02-26.md`, `logs/taskitem_runs/challenging_fullstack_multifile_20260226_r5/results.jsonl` | `partial` | Hard checks now enforce migration rollback + data-loss policy, security deny-by-default, SLO p95 presence, and rollout staged+abort policy. Enforcement is contract-level; generator capability under these constraints is still weak in C++ AB path. | Constraint-aware generation follow-up | | GR-019 | Parity-blocked readiness load (gating without capability closure) | `logs/taskitem_runs/challenging_fullstack_multifile_20260226_r7/summary.json`, `logs/taskitem_runs/challenging_subset_prod_20260226_r7/summary.json`, `docs/sprint225_227_execution_tracker_2026-02-26.md` | `partial` | Sprint 225-227 reduced blocked parity load from `6` to `0` on both tracked hard catalogs while keeping unresolved divergence at `0`. This closes immediate safety debt for current corpora, but robustness is still contingent on pattern-driven repair classes. | Generalize repairs beyond queue-shaped transpile outputs | | GR-020 | Semantic fallback overuse masks weak native decomposition | `logs/taskitem_runs/TEST_ONLY_sprint236_semantic_fallback_audit_20260226/semantic_fallback_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint237_semantic_fallback_gate_fail_20260226/semantic_fallback_budget_gate.json`, `logs/taskitem_runs/TEST_ONLY_sprint238_native_gate_fail_20260226.json`, `logs/taskitem_runs/TEST_ONLY_sprint239_semantic_fallback_audit_20260226/semantic_fallback_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint240_retry_20260226_145631/00_summary.json`, `docs/sprint236_execution_tracker_2026-02-26.md`, `docs/sprint237_execution_tracker_2026-02-26.md`, `docs/sprint238_execution_tracker_2026-02-26.md`, `docs/sprint239_execution_tracker_2026-02-26.md`, `docs/sprint240_execution_tracker_2026-02-26.md` | `partial` | Sprint 236 added deterministic fallback-gap auditing with tool + constraint metadata. Sprint 237 added fallback-budget hard gate (`max-fallback-rate`) and produced expected fail/pass artifacts. Sprint 238 added native decomposition hard gate in pipeline (`min task count`, `min semantic signal count`) so weak native generation can be blocked before fallback masking. Sprint 239 added native reason enrichment and improved semantic signal density (`0 -> 6`). Sprint 240 added native decomposition retry with explicit minimum-task policy; on current sample retry attempted but did not improve depth (`2 -> 2`). | Native decomposition task-depth upgrade (increase native task granularity beyond 2) | -| GR-021 | Impact-specific native decomposition coverage not enforced uniformly | `docs/native_decomposition_impact_list_2026-02-26.md`, `tools/mcp/profiles/native_decomposition_impact_profiles.json`, `logs/taskitem_runs/TEST_ONLY_sprint241_fullstack_impact_20260226_150416/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint241_native_impact_aggregate_20260226/native_impact_coverage_aggregate.json`, `logs/taskitem_runs/TEST_ONLY_sprint242_impact_remediation_loop_20260226/remediation_loop_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint243_impact_remediation_tasks_loop_20260226_r2/remediation_loop_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint244_autofill_20260226_151651/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint244_autofill_enforce_20260226_151709/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint245_intrinsic_20260226_151835/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint246_multishot_20260226_153033/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint246_multishot_enforce_20260226_153045/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint247_singleshot_20260226_153230/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint247_singleshot_enforce_20260226_153241/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint248_rawsearch_20260226_153903/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint249_rawguard_20260226_154028/02ae_raw_candidate_search.json` | `partial` | Sprint 241 introduced impact profiles/gates; 242-243 remediation loops; 244 first-pass autofill; 245 intrinsic constraints only (no uplift); 246 multishot closes hard sample; 247 single-shot shaping closes with multishot+autofill disabled; 248 raw candidate search over 8 variants found no uplift (`selected_variant=0`, `7 -> 7`); 249 added no-uplift guardrail to hard-fail ineffective raw search attempts. Residual gap remains in raw generator single-shot quality without overlays. | Improve raw single-shot generator output so profile closure does not rely on shaping/multishot/autofill overlays | +| GR-021 | Impact-specific native decomposition coverage not enforced uniformly | `docs/native_decomposition_impact_list_2026-02-26.md`, `tools/mcp/profiles/native_decomposition_impact_profiles.json`, `logs/taskitem_runs/TEST_ONLY_sprint241_fullstack_impact_20260226_150416/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint241_native_impact_aggregate_20260226/native_impact_coverage_aggregate.json`, `logs/taskitem_runs/TEST_ONLY_sprint242_impact_remediation_loop_20260226/remediation_loop_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint243_impact_remediation_tasks_loop_20260226_r2/remediation_loop_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint244_autofill_20260226_151651/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint244_autofill_enforce_20260226_151709/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint245_intrinsic_20260226_151835/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint246_multishot_20260226_153033/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint246_multishot_enforce_20260226_153045/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint247_singleshot_20260226_153230/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint247_singleshot_enforce_20260226_153241/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint248_rawsearch_20260226_153903/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint249_rawguard_20260226_154028/02ae_raw_candidate_search.json`, `logs/taskitem_runs/TEST_ONLY_sprint250_closure_ladder_20260226/closure_ladder_summary.json` | `partial` | Sprint 241 introduced impact profiles/gates; 242-243 remediation loops; 244 first-pass autofill; 245 intrinsic constraints only (no uplift); 246 multishot closes hard sample; 247 single-shot shaping closes with multishot+autofill disabled; 248 raw candidate search found no uplift; 249 added no-uplift guardrail; 250 added automatic closure ladder and selected `single_shot_shape` as minimal passing mode on hard sample. Residual gap remains in raw generator single-shot quality without overlays. | Improve raw single-shot generator output so profile closure does not rely on shaping/multishot/autofill overlays | ## What Was Covered Today (Sprints 175-184) diff --git a/docs/progress_log_2026-02-26.md b/docs/progress_log_2026-02-26.md index dfe2b59..03b3e3d 100644 --- a/docs/progress_log_2026-02-26.md +++ b/docs/progress_log_2026-02-26.md @@ -323,3 +323,14 @@ - Current measured result: - baseline `7`, best `7` failing profiles - run blocked correctly under require-uplift mode. + +## Sprint 250 Added (Same Day) + +- Added automatic closure ladder runner: + - `tools/mcp/run_native_profile_closure_ladder.sh` + - mode order: `raw_only -> single_shot_shape -> multishot -> autofill` +- Dated artifact: + - `logs/taskitem_runs/TEST_ONLY_sprint250_closure_ladder_20260226/closure_ladder_summary.json` +- Current measured result on hard fullstack sample: + - selected mode: `single_shot_shape` + - selected run passes impact coverage with `failing_profile_count=0` diff --git a/docs/sprint250_execution_tracker_2026-02-26.md b/docs/sprint250_execution_tracker_2026-02-26.md new file mode 100644 index 0000000..03a372f --- /dev/null +++ b/docs/sprint250_execution_tracker_2026-02-26.md @@ -0,0 +1,32 @@ +# Sprint 250 Execution Tracker - 2026-02-26 + +## Scope +- `sprint250_plan.md` + +## Implemented + +New runner: +- `tools/mcp/run_native_profile_closure_ladder.sh` + - mode order: + 1. `raw_only` + 2. `single_shot_shape` + 3. `multishot` + 4. `autofill` + - runs each mode with enforced impact coverage gate. + - stops at first pass and records selected mode. + - writes deterministic summary: + - `closure_ladder_summary.json` + +## Validation Artifact + +- `logs/taskitem_runs/TEST_ONLY_sprint250_closure_ladder_20260226/closure_ladder_summary.json` + +Observed result on hard fullstack sample: +- selected mode: `single_shot_shape` +- selected run impact coverage: + - `status=ok` + - `failing_profile_count=0` + +## Explicit Completion Signal + +- Sprint 250: `DONE` (implemented + ladder selection verified) diff --git a/editor/src/Sprint250IntegrationSummary.h b/editor/src/Sprint250IntegrationSummary.h new file mode 100644 index 0000000..32333bd --- /dev/null +++ b/editor/src/Sprint250IntegrationSummary.h @@ -0,0 +1,6 @@ +#pragma once + +// Sprint 250 integration summary: +// - Added automatic closure ladder runner for native profile coverage. +// - Tries progressive strategies and selects the minimal passing mode. +// - Emits deterministic summary with selected mode and run artifact path. diff --git a/sprint250_plan.md b/sprint250_plan.md new file mode 100644 index 0000000..07a15cf --- /dev/null +++ b/sprint250_plan.md @@ -0,0 +1,11 @@ +# Sprint 250 Plan: Automatic Native Profile Closure Ladder + +## Goal +Select the minimal legal closure strategy that passes enforced impact coverage for a given spec. + +## Steps +- Step 2295: Add closure ladder runner for progressive strategy modes. +- Step 2296: Encode mode order from least to most assistive. +- Step 2297: Persist per-mode run logs and selected-mode summary. +- Step 2298: Validate ladder on hard fullstack sample. +- Step 2299: Add `Sprint250IntegrationSummary.h` and execution tracker. diff --git a/tools/mcp/run_native_profile_closure_ladder.sh b/tools/mcp/run_native_profile_closure_ladder.sh new file mode 100755 index 0000000..07df4e0 --- /dev/null +++ b/tools/mcp/run_native_profile_closure_ladder.sh @@ -0,0 +1,101 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +SPEC_PATH="${1:-}" +OUT_DIR="${2:-$ROOT_DIR/logs/taskitem_runs/TEST_ONLY_native_profile_closure_ladder_$(date +%Y%m%d_%H%M%S)}" +if [[ -z "$SPEC_PATH" ]]; then + echo "usage: $0 [out_dir]" >&2 + exit 2 +fi +if [[ ! -f "$SPEC_PATH" ]]; then + echo "error: spec not found: $SPEC_PATH" >&2 + exit 2 +fi +mkdir -p "$OUT_DIR" + +run_mode() { + local mode="$1" + local logfile="$OUT_DIR/${mode}.log" + local rcfile="$OUT_DIR/${mode}.rc" + + local env_common=( + WSTONE_NATIVE_IMPACT_COVERAGE_GATE=1 + WSTONE_NATIVE_IMPACT_COVERAGE_ENFORCE=1 + WSTONE_NATIVE_INTRINSIC_BOOST=1 + WSTONE_NATIVE_RAW_CANDIDATE_SEARCH=0 + WSTONE_NATIVE_RAW_CANDIDATE_REQUIRE_UPLIFT=0 + WSTONE_NATIVE_SINGLESHOT_PROFILE_SHAPE=0 + WSTONE_NATIVE_MULTISHOT_DECOMP=0 + WSTONE_NATIVE_PROFILE_AUTOFILL=0 + ) + + case "$mode" in + raw_only) + env_common+=(WSTONE_NATIVE_RAW_CANDIDATE_SEARCH=1) + ;; + single_shot_shape) + env_common+=(WSTONE_NATIVE_SINGLESHOT_PROFILE_SHAPE=1) + ;; + multishot) + env_common+=(WSTONE_NATIVE_MULTISHOT_DECOMP=1) + ;; + autofill) + env_common+=(WSTONE_NATIVE_PROFILE_AUTOFILL=1) + ;; + *) + echo "unknown mode: $mode" >&2 + exit 3 + ;; + esac + + set +e + env "${env_common[@]}" "$ROOT_DIR/tools/mcp/run_sprint_taskitem_pipeline.sh" "$SPEC_PATH" >"$logfile" 2>&1 + local rc=$? + set -e + echo "$rc" > "$rcfile" + + local latest + latest=$(ls -1dt "$ROOT_DIR/logs/taskitem_runs/$(basename "$SPEC_PATH" .md)_"* | head -n1) + echo "$latest" > "$OUT_DIR/${mode}.run_dir" + + if [[ "$rc" -eq 0 ]]; then + return 0 + fi + return "$rc" +} + +modes=(raw_only single_shot_shape multishot autofill) +selected="" +selected_run="" +selected_rc=1 + +for mode in "${modes[@]}"; do + if run_mode "$mode"; then + selected="$mode" + selected_run="$(cat "$OUT_DIR/${mode}.run_dir")" + selected_rc=0 + break + fi +done + +jq -n \ + --arg spec "$SPEC_PATH" \ + --arg out_dir "$OUT_DIR" \ + --arg selected_mode "$selected" \ + --arg selected_run "$selected_run" \ + --argjson selected_rc "$selected_rc" \ + --argjson attempted_modes "$(printf '%s\n' "${modes[@]}" | jq -R . | jq -s .)" \ + '{ + status: (if $selected_rc == 0 then "ok" else "fail" end), + spec:$spec, + out_dir:$out_dir, + attempted_modes:$attempted_modes, + selected_mode:$selected_mode, + selected_run:$selected_run, + selected_rc:$selected_rc + }' | tee "$OUT_DIR/closure_ladder_summary.json" + +if [[ "$selected_rc" -ne 0 ]]; then + exit 21 +fi