From 39575cb38303a9ede96367833f4a96bf1cfac8ef Mon Sep 17 00:00:00 2001 From: Bill Date: Thu, 26 Feb 2026 16:25:57 -0700 Subject: [PATCH] Add top-gap weighted raw candidate selection telemetry (sprint 256) --- docs/constructive_editing_runtime_plan.md | 4 + ...rator_readiness_gap_registry_2026-02-26.md | 2 +- docs/progress_log_2026-02-26.md | 18 +++++ .../sprint256_execution_tracker_2026-02-26.md | 53 ++++++++++++ editor/src/Sprint256IntegrationSummary.h | 6 ++ sprint256_plan.md | 12 +++ tools/mcp/run_sprint_taskitem_pipeline.sh | 34 +++++++- .../score_native_tasks_profile_coverage.py | 80 ++++++++++++++++++- 8 files changed, 204 insertions(+), 5 deletions(-) create mode 100644 docs/sprint256_execution_tracker_2026-02-26.md create mode 100644 editor/src/Sprint256IntegrationSummary.h create mode 100644 sprint256_plan.md diff --git a/docs/constructive_editing_runtime_plan.md b/docs/constructive_editing_runtime_plan.md index ecc72c7..9c90f87 100644 --- a/docs/constructive_editing_runtime_plan.md +++ b/docs/constructive_editing_runtime_plan.md @@ -216,3 +216,7 @@ Planning runtime controls (active): - `tools/mcp/harden_native_tasks_top_gaps.py` - `WSTONE_NATIVE_RAW_HARDEN_TOP_GAPS` - `native_raw_hardening` +- raw candidate selection can be weighted by historical gap priorities: + - `WSTONE_NATIVE_RAW_TOP_GAP_WEIGHTED_SELECT` + - `WSTONE_NATIVE_RAW_TOP_GAP_BACKLOG_FILE` + - scorer `--top-gaps` telemetry (`top_gap_score`) diff --git a/docs/generator_readiness_gap_registry_2026-02-26.md b/docs/generator_readiness_gap_registry_2026-02-26.md index 2fb7318..f012a90 100644 --- a/docs/generator_readiness_gap_registry_2026-02-26.md +++ b/docs/generator_readiness_gap_registry_2026-02-26.md @@ -47,7 +47,7 @@ This is the canonical dated registry for "not production-ready" generator gaps. | GR-018 | Performance/security/rollout constrained refactor enforcement gap | `docs/gap_hunt_fullstack_multifile_2026-02-26.md`, `logs/taskitem_runs/challenging_fullstack_multifile_20260226_r5/results.jsonl` | `partial` | Hard checks now enforce migration rollback + data-loss policy, security deny-by-default, SLO p95 presence, and rollout staged+abort policy. Enforcement is contract-level; generator capability under these constraints is still weak in C++ AB path. | Constraint-aware generation follow-up | | GR-019 | Parity-blocked readiness load (gating without capability closure) | `logs/taskitem_runs/challenging_fullstack_multifile_20260226_r7/summary.json`, `logs/taskitem_runs/challenging_subset_prod_20260226_r7/summary.json`, `docs/sprint225_227_execution_tracker_2026-02-26.md` | `partial` | Sprint 225-227 reduced blocked parity load from `6` to `0` on both tracked hard catalogs while keeping unresolved divergence at `0`. This closes immediate safety debt for current corpora, but robustness is still contingent on pattern-driven repair classes. | Generalize repairs beyond queue-shaped transpile outputs | | GR-020 | Semantic fallback overuse masks weak native decomposition | `logs/taskitem_runs/TEST_ONLY_sprint236_semantic_fallback_audit_20260226/semantic_fallback_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint237_semantic_fallback_gate_fail_20260226/semantic_fallback_budget_gate.json`, `logs/taskitem_runs/TEST_ONLY_sprint238_native_gate_fail_20260226.json`, `logs/taskitem_runs/TEST_ONLY_sprint239_semantic_fallback_audit_20260226/semantic_fallback_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint240_retry_20260226_145631/00_summary.json`, `docs/sprint236_execution_tracker_2026-02-26.md`, `docs/sprint237_execution_tracker_2026-02-26.md`, `docs/sprint238_execution_tracker_2026-02-26.md`, `docs/sprint239_execution_tracker_2026-02-26.md`, `docs/sprint240_execution_tracker_2026-02-26.md` | `partial` | Sprint 236 added deterministic fallback-gap auditing with tool + constraint metadata. Sprint 237 added fallback-budget hard gate (`max-fallback-rate`) and produced expected fail/pass artifacts. Sprint 238 added native decomposition hard gate in pipeline (`min task count`, `min semantic signal count`) so weak native generation can be blocked before fallback masking. Sprint 239 added native reason enrichment and improved semantic signal density (`0 -> 6`). Sprint 240 added native decomposition retry with explicit minimum-task policy; on current sample retry attempted but did not improve depth (`2 -> 2`). | Native decomposition task-depth upgrade (increase native task granularity beyond 2) | -| GR-021 | Impact-specific native decomposition coverage not enforced uniformly | `docs/native_decomposition_impact_list_2026-02-26.md`, `tools/mcp/profiles/native_decomposition_impact_profiles.json`, `logs/taskitem_runs/TEST_ONLY_sprint241_fullstack_impact_20260226_150416/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint241_native_impact_aggregate_20260226/native_impact_coverage_aggregate.json`, `logs/taskitem_runs/TEST_ONLY_sprint242_impact_remediation_loop_20260226/remediation_loop_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint243_impact_remediation_tasks_loop_20260226_r2/remediation_loop_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint244_autofill_20260226_151651/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint244_autofill_enforce_20260226_151709/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint245_intrinsic_20260226_151835/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint246_multishot_20260226_153033/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint246_multishot_enforce_20260226_153045/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint247_singleshot_20260226_153230/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint247_singleshot_enforce_20260226_153241/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint248_rawsearch_20260226_153903/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint249_rawguard_20260226_154028/02ae_raw_candidate_search.json`, `logs/taskitem_runs/TEST_ONLY_sprint250_closure_ladder_20260226/closure_ladder_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint251_rawvariants_20260226_154355/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint252_closure_ladder_policy_skip2_20260226/closure_ladder_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint253_closure_ladder_batch_20260226/closure_ladder_batch_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint254_closure_ladder_batch_20260226_r2/raw_gap_backlog.json`, `logs/taskitem_runs/TEST_ONLY_sprint255_rawharden_20260226_161924/00_summary.json` | `partial` | Sprint 241 introduced impact profiles/gates; 242-243 remediation loops; 244 first-pass autofill; 245 intrinsic constraints only (no uplift); 246 multishot closure; 247 single-shot shaping closure; 248 raw candidate search no uplift; 249 no-uplift guardrail; 250 closure ladder; 251 richer raw variants no uplift; 252 history-aware ladder routing; 253 batch ladder analytics; 254 raw gap backlog synthesis from `raw_only` attempts identifies top missing raw signals; 255 adds first-pass raw top-gap hardening that closes hard-sample profile coverage (`failing_profile_count 7 -> 0`) by injecting missing ops/contracts/reasons and minimum task depth. Residual gap remains in intrinsic raw generator quality because hardening is compensatory post-processing. | Improve intrinsic raw candidate selection/generation so backlog signal closure is achieved without post-hardening overlays | +| GR-021 | Impact-specific native decomposition coverage not enforced uniformly | `docs/native_decomposition_impact_list_2026-02-26.md`, `tools/mcp/profiles/native_decomposition_impact_profiles.json`, `logs/taskitem_runs/TEST_ONLY_sprint241_fullstack_impact_20260226_150416/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint241_native_impact_aggregate_20260226/native_impact_coverage_aggregate.json`, `logs/taskitem_runs/TEST_ONLY_sprint242_impact_remediation_loop_20260226/remediation_loop_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint243_impact_remediation_tasks_loop_20260226_r2/remediation_loop_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint244_autofill_20260226_151651/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint244_autofill_enforce_20260226_151709/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint245_intrinsic_20260226_151835/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint246_multishot_20260226_153033/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint246_multishot_enforce_20260226_153045/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint247_singleshot_20260226_153230/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint247_singleshot_enforce_20260226_153241/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint248_rawsearch_20260226_153903/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint249_rawguard_20260226_154028/02ae_raw_candidate_search.json`, `logs/taskitem_runs/TEST_ONLY_sprint250_closure_ladder_20260226/closure_ladder_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint251_rawvariants_20260226_154355/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint252_closure_ladder_policy_skip2_20260226/closure_ladder_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint253_closure_ladder_batch_20260226/closure_ladder_batch_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint254_closure_ladder_batch_20260226_r2/raw_gap_backlog.json`, `logs/taskitem_runs/TEST_ONLY_sprint255_rawharden_20260226_161924/00_summary.json`, `logs/taskitem_runs/01a_fallback_intake_spec_20260226_162423/00_summary.json` | `partial` | Sprint 241 introduced impact profiles/gates; 242-243 remediation loops; 244 first-pass autofill; 245 intrinsic constraints only (no uplift); 246 multishot closure; 247 single-shot shaping closure; 248 raw candidate search no uplift; 249 no-uplift guardrail; 250 closure ladder; 251 richer raw variants no uplift; 252 history-aware ladder routing; 253 batch ladder analytics; 254 raw gap backlog synthesis from `raw_only` attempts identifies top missing raw signals; 255 adds first-pass raw top-gap hardening that closes hard-sample profile coverage (`failing_profile_count 7 -> 0`) by injecting missing ops/contracts/reasons and minimum task depth; 256 adds top-gap weighted candidate selection telemetry and tie-break policy, but measured sample still shows no intrinsic raw uplift (`selected_variant=0`). Residual gap remains in intrinsic raw generator quality because coverage is still recovered primarily via post-generation hardening/shaping paths. | Add intrinsic raw generation hard gate requiring minimum top-gap score uplift (or fail fast) before promoting raw-only path | ## What Was Covered Today (Sprints 175-184) diff --git a/docs/progress_log_2026-02-26.md b/docs/progress_log_2026-02-26.md index cadd3b4..59230ec 100644 --- a/docs/progress_log_2026-02-26.md +++ b/docs/progress_log_2026-02-26.md @@ -409,3 +409,21 @@ - Current measured result on hard fullstack sample: - `failing_profile_count 7 -> 0` - `effective_task_count 2 -> 6` + +## Sprint 256 Added (Same Day) + +- Added top-gap weighted raw candidate scoring/selection: + - scorer: `tools/mcp/score_native_tasks_profile_coverage.py --top-gaps` + - pipeline controls: + - `WSTONE_NATIVE_RAW_TOP_GAP_WEIGHTED_SELECT` + - `WSTONE_NATIVE_RAW_TOP_GAP_BACKLOG_FILE` +- Raw search summary now includes: + - `baseline_top_gap_score` + - `best_top_gap_score` + - weighted-select metadata +- Dated A/B artifacts (weighted select OFF/ON): + - `logs/taskitem_runs/01a_fallback_intake_spec_20260226_162422/00_summary.json` + - `logs/taskitem_runs/01a_fallback_intake_spec_20260226_162423/00_summary.json` +- Current measured result on hard sample: + - no intrinsic raw uplift yet (`selected_variant=0`, `best_failing_profile_count=1`) + - scorer top-gap telemetry emitted and usable for ranking policy. diff --git a/docs/sprint256_execution_tracker_2026-02-26.md b/docs/sprint256_execution_tracker_2026-02-26.md new file mode 100644 index 0000000..1cf408e --- /dev/null +++ b/docs/sprint256_execution_tracker_2026-02-26.md @@ -0,0 +1,53 @@ +# Sprint 256 Execution Tracker - 2026-02-26 + +## Scope +- `sprint256_plan.md` + +## Implemented + +Scorer enhancement: +- `tools/mcp/score_native_tasks_profile_coverage.py` + - Added optional `--top-gaps `. + - Computes: + - `top_gap_signal_count` + - `top_gap_signal_hits` + - `top_gap_signal_weight_total` + - `top_gap_score` + +Pipeline integration in `tools/mcp/run_sprint_taskitem_pipeline.sh`: +- Added: + - `WSTONE_NATIVE_RAW_TOP_GAP_WEIGHTED_SELECT` + - `WSTONE_NATIVE_RAW_TOP_GAP_BACKLOG_FILE` +- Raw search summary now emits: + - `baseline_top_gap_score` + - `best_top_gap_score` + - `top_gap_weighted_select` + - `top_gap_backlog_file` +- Candidate selection policy: + - first minimize `failing_profile_count` + - if tied and weighted mode enabled, maximize `top_gap_score` + - then maximize `task_count` + +## Validation Artifacts + +Raw search A/B (weighted selector off/on): +- OFF: + - `logs/taskitem_runs/01a_fallback_intake_spec_20260226_162422/00_summary.json` +- ON: + - `logs/taskitem_runs/01a_fallback_intake_spec_20260226_162423/00_summary.json` + +Observed result on hard sample: +- `selected_variant` unchanged (`0`) +- `best_failing_profile_count` unchanged (`1`) +- `best_top_gap_score` remained `0` for this sample + +Scorer sanity check on hardened tasks: +- `python3 tools/mcp/score_native_tasks_profile_coverage.py ... --top-gaps ...` +- output: `/tmp/sprint256_score_check.json` +- observed: + - `top_gap_signal_hits=18/20` + - `top_gap_score=86` + +## Explicit Completion Signal + +- Sprint 256: `DONE` (implemented + measured; no intrinsic uplift yet on this hard sample) diff --git a/editor/src/Sprint256IntegrationSummary.h b/editor/src/Sprint256IntegrationSummary.h new file mode 100644 index 0000000..30bf459 --- /dev/null +++ b/editor/src/Sprint256IntegrationSummary.h @@ -0,0 +1,6 @@ +#pragma once + +// Sprint 256 integration summary: +// - Added top-gap weighted scoring in raw candidate coverage scorer. +// - Raw candidate search can now select variants using weighted backlog signal closure. +// - Summary telemetry now includes baseline/best top-gap scores for forensic tuning. diff --git a/sprint256_plan.md b/sprint256_plan.md new file mode 100644 index 0000000..4813161 --- /dev/null +++ b/sprint256_plan.md @@ -0,0 +1,12 @@ +# Sprint 256 Plan: Top-Gap Weighted Raw Candidate Selection + +## Goal +Use prioritized raw-gap backlog signals directly in raw-only candidate scoring so variant selection is guided by historically missing intrinsic signals, not only profile fail count. + +## Steps +- Step 2326: Extend raw candidate scorer with optional top-gap backlog input. +- Step 2327: Compute weighted `top_gap_score` and hit counts from candidate tasks. +- Step 2328: Add pipeline controls for weighted raw selection. +- Step 2329: Make raw candidate tie-break use `top_gap_score` when enabled. +- Step 2330: Emit baseline/best top-gap score telemetry in raw search summary. +- Step 2331: Add `Sprint256IntegrationSummary.h` and execution tracker. diff --git a/tools/mcp/run_sprint_taskitem_pipeline.sh b/tools/mcp/run_sprint_taskitem_pipeline.sh index f1ed7a8..c38e3a5 100755 --- a/tools/mcp/run_sprint_taskitem_pipeline.sh +++ b/tools/mcp/run_sprint_taskitem_pipeline.sh @@ -53,6 +53,8 @@ NATIVE_RAW_CANDIDATE_SEARCH="${WSTONE_NATIVE_RAW_CANDIDATE_SEARCH:-0}" NATIVE_RAW_CANDIDATE_MAX_VARIANTS="${WSTONE_NATIVE_RAW_CANDIDATE_MAX_VARIANTS:-8}" NATIVE_RAW_CANDIDATE_REQUIRE_UPLIFT="${WSTONE_NATIVE_RAW_CANDIDATE_REQUIRE_UPLIFT:-0}" NATIVE_RAW_HARDEN_TOP_GAPS="${WSTONE_NATIVE_RAW_HARDEN_TOP_GAPS:-0}" +NATIVE_RAW_TOP_GAP_WEIGHTED_SELECT="${WSTONE_NATIVE_RAW_TOP_GAP_WEIGHTED_SELECT:-0}" +NATIVE_RAW_TOP_GAP_BACKLOG_FILE="${WSTONE_NATIVE_RAW_TOP_GAP_BACKLOG_FILE:-}" EXTRA_NORMALIZED_REQUIREMENTS_FILE="${WSTONE_EXTRA_NORMALIZED_REQUIREMENTS_FILE:-}" EXTRA_TASKS_FILE="${WSTONE_EXTRA_TASKS_FILE:-}" CAPABILITY_SIGNALS_JSON="${WSTONE_CAPABILITY_SIGNALS_JSON:-}" @@ -392,6 +394,12 @@ else '{attempted:$attempted, applied:$applied, initial_task_count:$initial_task_count, target_min_task_count:$target_min_task_count}')" fi if [[ "$NATIVE_RAW_CANDIDATE_SEARCH" == "1" ]]; then + top_gap_score_args=() + use_top_gap_weighted_select=false + if [[ "$NATIVE_RAW_TOP_GAP_WEIGHTED_SELECT" == "1" && -n "$NATIVE_RAW_TOP_GAP_BACKLOG_FILE" && -f "$NATIVE_RAW_TOP_GAP_BACKLOG_FILE" ]]; then + top_gap_score_args=(--top-gaps "$NATIVE_RAW_TOP_GAP_BACKLOG_FILE") + use_top_gap_weighted_select=true + fi python3 "$ROOT_DIR/tools/mcp/synthesize_raw_candidate_requirement_variants.py" \ --spec "$INPUT_FILE" \ --profiles "$NATIVE_IMPACT_COVERAGE_PROFILES" \ @@ -404,12 +412,15 @@ if [[ "$NATIVE_RAW_CANDIDATE_SEARCH" == "1" ]]; then --spec "$INPUT_FILE" \ --profiles "$NATIVE_IMPACT_COVERAGE_PROFILES" \ --tasks "$OUT_DIR/02ae_candidate_0_tasks.json" \ + "${top_gap_score_args[@]}" \ --out "$OUT_DIR/02ae_candidate_0_score.json" >/dev/null best_variant="0" best_fail="$(jq '.failing_profile_count // 999' "$OUT_DIR/02ae_candidate_0_score.json")" best_task_count="$(jq '.task_count // 0' "$OUT_DIR/02ae_candidate_0_score.json")" + best_top_gap_score="$(jq '.top_gap_score // 0' "$OUT_DIR/02ae_candidate_0_score.json")" baseline_fail="$best_fail" baseline_task_count="$best_task_count" + baseline_top_gap_score="$best_top_gap_score" attempted=1 successful=1 @@ -434,13 +445,28 @@ if [[ "$NATIVE_RAW_CANDIDATE_SEARCH" == "1" ]]; then --spec "$INPUT_FILE" \ --profiles "$NATIVE_IMPACT_COVERAGE_PROFILES" \ --tasks "$OUT_DIR/02ae_candidate_${variant}_tasks.json" \ + "${top_gap_score_args[@]}" \ --out "$OUT_DIR/02ae_candidate_${variant}_score.json" >/dev/null cand_fail="$(jq '.failing_profile_count // 999' "$OUT_DIR/02ae_candidate_${variant}_score.json")" cand_task_count="$(jq '.task_count // 0' "$OUT_DIR/02ae_candidate_${variant}_score.json")" - if [[ "$cand_fail" -lt "$best_fail" || ( "$cand_fail" -eq "$best_fail" && "$cand_task_count" -gt "$best_task_count" ) ]]; then + cand_top_gap_score="$(jq '.top_gap_score // 0' "$OUT_DIR/02ae_candidate_${variant}_score.json")" + should_select=false + if [[ "$cand_fail" -lt "$best_fail" ]]; then + should_select=true + elif [[ "$cand_fail" -eq "$best_fail" ]]; then + if [[ "$use_top_gap_weighted_select" == true && "$cand_top_gap_score" -gt "$best_top_gap_score" ]]; then + should_select=true + elif [[ "$cand_top_gap_score" -eq "$best_top_gap_score" && "$cand_task_count" -gt "$best_task_count" ]]; then + should_select=true + elif [[ "$use_top_gap_weighted_select" != true && "$cand_task_count" -gt "$best_task_count" ]]; then + should_select=true + fi + fi + if [[ "$should_select" == true ]]; then best_variant="$variant" best_fail="$cand_fail" best_task_count="$cand_task_count" + best_top_gap_score="$cand_top_gap_score" TASKS="$cand_tasks" fi successful=$((successful + 1)) @@ -456,10 +482,14 @@ if [[ "$NATIVE_RAW_CANDIDATE_SEARCH" == "1" ]]; then --arg best_variant "$best_variant" \ --argjson baseline_failing_profile_count "$baseline_fail" \ --argjson baseline_task_count "$baseline_task_count" \ + --argjson baseline_top_gap_score "$baseline_top_gap_score" \ --argjson best_failing_profile_count "$best_fail" \ --argjson best_task_count "$best_task_count" \ + --argjson best_top_gap_score "$best_top_gap_score" \ --argjson available_variants "$variant_total" \ - '{enabled:$enabled, attempted_variants:$attempted, successful_variants:$successful, available_variants:$available_variants, selected_variant:$best_variant, baseline_failing_profile_count:$baseline_failing_profile_count, baseline_task_count:$baseline_task_count, best_failing_profile_count:$best_failing_profile_count, best_task_count:$best_task_count}')" + --argjson top_gap_weighted_select "$use_top_gap_weighted_select" \ + --arg top_gap_backlog_file "$NATIVE_RAW_TOP_GAP_BACKLOG_FILE" \ + '{enabled:$enabled, attempted_variants:$attempted, successful_variants:$successful, available_variants:$available_variants, selected_variant:$best_variant, baseline_failing_profile_count:$baseline_failing_profile_count, baseline_task_count:$baseline_task_count, baseline_top_gap_score:$baseline_top_gap_score, best_failing_profile_count:$best_failing_profile_count, best_task_count:$best_task_count, best_top_gap_score:$best_top_gap_score, top_gap_weighted_select:$top_gap_weighted_select, top_gap_backlog_file:$top_gap_backlog_file}')" printf '%s\n' "$NATIVE_RAW_CANDIDATE_SEARCH_JSON" > "$OUT_DIR/02ae_raw_candidate_search.json" if [[ "$NATIVE_RAW_CANDIDATE_REQUIRE_UPLIFT" == "1" && "$best_fail" -ge "$baseline_fail" ]]; then echo "error: raw candidate search did not improve failing profile count" >&2 diff --git a/tools/mcp/score_native_tasks_profile_coverage.py b/tools/mcp/score_native_tasks_profile_coverage.py index 50417f3..9cfb45f 100755 --- a/tools/mcp/score_native_tasks_profile_coverage.py +++ b/tools/mcp/score_native_tasks_profile_coverage.py @@ -2,7 +2,7 @@ import argparse import json from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Optional, Tuple def load_json(path: Path): @@ -35,6 +35,55 @@ def collect_task_unions(tasks: List[Dict]): return reasons, prereq_ops, contracts +def parse_missing_signal(signal: str) -> Tuple[str, str]: + if ":" not in signal: + return signal, "" + k, v = signal.split(":", 1) + return k, v + + +def signal_present(signal: str, tasks: List[Dict]) -> bool: + if signal.startswith("native_task_count<"): + try: + min_count = int(signal.split("<", 1)[1]) + except (ValueError, IndexError): + return False + return len(tasks) >= min_count + + kind, value = parse_missing_signal(signal) + reasons, prereq_ops, contracts = collect_task_unions(tasks) + + if kind == "missing_prerequisite_op": + return value in set(prereq_ops) + if kind == "missing_execution_contract": + for ec in contracts: + if isinstance(ec.get(value), bool): + if ec.get(value): + return True + elif value in ec: + return True + return False + if kind == "missing_reason_keyword": + v = value.lower() + return any(v in r for r in reasons) + return False + + +def load_top_gap_items(path: Optional[Path]) -> List[Dict]: + if path is None or not path.exists(): + return [] + data = load_json(path) + rows = list(data.get("prioritized_missing") or []) + out: List[Dict] = [] + for row in rows: + signal = str(row.get("missing", "")) + if not signal: + continue + weight = int(row.get("count", 1) or 1) + out.append({"missing": signal, "weight": weight}) + return out + + def check_profile(profile: Dict, tasks: List[Dict]) -> Dict: reasons, prereq_ops, contracts = collect_task_unions(tasks) missing: List[str] = [] @@ -74,6 +123,7 @@ def main() -> int: p.add_argument("--profiles", required=True) p.add_argument("--tasks", required=True) p.add_argument("--out", required=True) + p.add_argument("--top-gaps", default="", help="Optional raw gap backlog JSON with prioritized_missing.") args = p.parse_args() spec_text = Path(args.spec).read_text(encoding="utf-8", errors="ignore") @@ -87,19 +137,45 @@ def main() -> int: checks = [check_profile(pf, tasks) for pf in active] failing = [c for c in checks if not c.get("passed", False)] + top_gap_items = load_top_gap_items(Path(args.top_gaps) if args.top_gaps else None) + top_gap_signal_hits = 0 + top_gap_score = 0 + top_gap_signal_weight_total = 0 + for item in top_gap_items: + sig = str(item.get("missing", "")) + w = int(item.get("weight", 1) or 1) + top_gap_signal_weight_total += w + if signal_present(sig, tasks): + top_gap_signal_hits += 1 + top_gap_score += w result = { "status": "ok" if not failing else "fail", "task_count": len(tasks), "active_profile_count": len(active), "failing_profile_count": len(failing), + "top_gap_signal_count": len(top_gap_items), + "top_gap_signal_hits": top_gap_signal_hits, + "top_gap_signal_weight_total": top_gap_signal_weight_total, + "top_gap_score": top_gap_score, "checks": checks, } with Path(args.out).open("w", encoding="utf-8") as f: json.dump(result, f, indent=2, sort_keys=True) f.write("\n") - print(json.dumps({"status": result["status"], "task_count": result["task_count"], "failing_profile_count": result["failing_profile_count"], "out": args.out}, sort_keys=True)) + print( + json.dumps( + { + "status": result["status"], + "task_count": result["task_count"], + "failing_profile_count": result["failing_profile_count"], + "top_gap_score": result["top_gap_score"], + "out": args.out, + }, + sort_keys=True, + ) + ) return 0