From 8e0018ef801ae45a382fcee3e24cb6f40c61e7ac Mon Sep 17 00:00:00 2001 From: Bill Date: Thu, 26 Feb 2026 15:41:20 -0700 Subject: [PATCH] Add raw candidate search scoring and no-uplift guardrail (sprints 248-249) --- docs/constructive_editing_runtime_plan.md | 7 +- ...rator_readiness_gap_registry_2026-02-26.md | 2 +- docs/progress_log_2026-02-26.md | 28 +++++ .../sprint248_execution_tracker_2026-02-26.md | 38 +++++++ .../sprint249_execution_tracker_2026-02-26.md | 30 +++++ editor/src/Sprint248IntegrationSummary.h | 6 + editor/src/Sprint249IntegrationSummary.h | 6 + sprint248_plan.md | 11 ++ sprint249_plan.md | 11 ++ tools/mcp/run_sprint_taskitem_pipeline.sh | 84 ++++++++++++++ .../score_native_tasks_profile_coverage.py | 107 ++++++++++++++++++ 11 files changed, 328 insertions(+), 2 deletions(-) create mode 100644 docs/sprint248_execution_tracker_2026-02-26.md create mode 100644 docs/sprint249_execution_tracker_2026-02-26.md create mode 100644 editor/src/Sprint248IntegrationSummary.h create mode 100644 editor/src/Sprint249IntegrationSummary.h create mode 100644 sprint248_plan.md create mode 100644 sprint249_plan.md create mode 100755 tools/mcp/score_native_tasks_profile_coverage.py diff --git a/docs/constructive_editing_runtime_plan.md b/docs/constructive_editing_runtime_plan.md index d3ea81f..bbf218f 100644 --- a/docs/constructive_editing_runtime_plan.md +++ b/docs/constructive_editing_runtime_plan.md @@ -150,7 +150,7 @@ Parallel planning tranche (active): - `logs/taskitem_runs/TEST_ONLY_spec_planning_baseline_20260226/deterministic_spec_readiness.json` Planning runtime controls (active): -- `sprint232_plan.md` to `sprint247_plan.md` +- `sprint232_plan.md` to `sprint249_plan.md` - semantic bridge + intake augmentation + requirement injection + expansion gating are wired into: - `tools/mcp/run_sprint_taskitem_pipeline.sh` - current policy default is native-first semantic fallback: @@ -197,3 +197,8 @@ Planning runtime controls (active): - `WSTONE_NATIVE_SINGLESHOT_PROFILE_SHAPE` - `tools/mcp/synthesize_single_shot_profile_shape_tasks.py` - `native_single_shot_profile_shape` +- raw-only candidate search (no overlays) can benchmark intrinsic generator variants: + - `WSTONE_NATIVE_RAW_CANDIDATE_SEARCH` + - `WSTONE_NATIVE_RAW_CANDIDATE_MAX_VARIANTS` + - `WSTONE_NATIVE_RAW_CANDIDATE_REQUIRE_UPLIFT` + - `tools/mcp/score_native_tasks_profile_coverage.py` diff --git a/docs/generator_readiness_gap_registry_2026-02-26.md b/docs/generator_readiness_gap_registry_2026-02-26.md index 7cb938a..250eee3 100644 --- a/docs/generator_readiness_gap_registry_2026-02-26.md +++ b/docs/generator_readiness_gap_registry_2026-02-26.md @@ -47,7 +47,7 @@ This is the canonical dated registry for "not production-ready" generator gaps. | GR-018 | Performance/security/rollout constrained refactor enforcement gap | `docs/gap_hunt_fullstack_multifile_2026-02-26.md`, `logs/taskitem_runs/challenging_fullstack_multifile_20260226_r5/results.jsonl` | `partial` | Hard checks now enforce migration rollback + data-loss policy, security deny-by-default, SLO p95 presence, and rollout staged+abort policy. Enforcement is contract-level; generator capability under these constraints is still weak in C++ AB path. | Constraint-aware generation follow-up | | GR-019 | Parity-blocked readiness load (gating without capability closure) | `logs/taskitem_runs/challenging_fullstack_multifile_20260226_r7/summary.json`, `logs/taskitem_runs/challenging_subset_prod_20260226_r7/summary.json`, `docs/sprint225_227_execution_tracker_2026-02-26.md` | `partial` | Sprint 225-227 reduced blocked parity load from `6` to `0` on both tracked hard catalogs while keeping unresolved divergence at `0`. This closes immediate safety debt for current corpora, but robustness is still contingent on pattern-driven repair classes. | Generalize repairs beyond queue-shaped transpile outputs | | GR-020 | Semantic fallback overuse masks weak native decomposition | `logs/taskitem_runs/TEST_ONLY_sprint236_semantic_fallback_audit_20260226/semantic_fallback_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint237_semantic_fallback_gate_fail_20260226/semantic_fallback_budget_gate.json`, `logs/taskitem_runs/TEST_ONLY_sprint238_native_gate_fail_20260226.json`, `logs/taskitem_runs/TEST_ONLY_sprint239_semantic_fallback_audit_20260226/semantic_fallback_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint240_retry_20260226_145631/00_summary.json`, `docs/sprint236_execution_tracker_2026-02-26.md`, `docs/sprint237_execution_tracker_2026-02-26.md`, `docs/sprint238_execution_tracker_2026-02-26.md`, `docs/sprint239_execution_tracker_2026-02-26.md`, `docs/sprint240_execution_tracker_2026-02-26.md` | `partial` | Sprint 236 added deterministic fallback-gap auditing with tool + constraint metadata. Sprint 237 added fallback-budget hard gate (`max-fallback-rate`) and produced expected fail/pass artifacts. Sprint 238 added native decomposition hard gate in pipeline (`min task count`, `min semantic signal count`) so weak native generation can be blocked before fallback masking. Sprint 239 added native reason enrichment and improved semantic signal density (`0 -> 6`). Sprint 240 added native decomposition retry with explicit minimum-task policy; on current sample retry attempted but did not improve depth (`2 -> 2`). | Native decomposition task-depth upgrade (increase native task granularity beyond 2) | -| GR-021 | Impact-specific native decomposition coverage not enforced uniformly | `docs/native_decomposition_impact_list_2026-02-26.md`, `tools/mcp/profiles/native_decomposition_impact_profiles.json`, `logs/taskitem_runs/TEST_ONLY_sprint241_fullstack_impact_20260226_150416/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint241_native_impact_aggregate_20260226/native_impact_coverage_aggregate.json`, `logs/taskitem_runs/TEST_ONLY_sprint242_impact_remediation_loop_20260226/remediation_loop_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint243_impact_remediation_tasks_loop_20260226_r2/remediation_loop_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint244_autofill_20260226_151651/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint244_autofill_enforce_20260226_151709/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint245_intrinsic_20260226_151835/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint246_multishot_20260226_153033/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint246_multishot_enforce_20260226_153045/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint247_singleshot_20260226_153230/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint247_singleshot_enforce_20260226_153241/00_summary.json` | `partial` | Sprint 241 added one-by-one impact profiles and native impact coverage gate. Sprint 242-243 added remediation synthesis and task-bundle closure loop (`7 -> 0`). Sprint 244 added first-pass profile autofill. Sprint 245 intrinsic-constraint-only path showed no uplift (`7 -> 7`). Sprint 246 intrinsic multishot path closed hard sample with autofill disabled (`7 -> 0`). Sprint 247 added single-shot profile shaping and closes hard sample with both multishot and autofill disabled (`7 -> 0`) while enforced gate still fails when shaping is disabled (`7`). Remaining risk is that closure depends on deterministic shaping overlays, not purely raw generator single-shot behavior. | Improve raw single-shot generator output so profile closure does not rely on post-generation shaping overlays | +| GR-021 | Impact-specific native decomposition coverage not enforced uniformly | `docs/native_decomposition_impact_list_2026-02-26.md`, `tools/mcp/profiles/native_decomposition_impact_profiles.json`, `logs/taskitem_runs/TEST_ONLY_sprint241_fullstack_impact_20260226_150416/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint241_native_impact_aggregate_20260226/native_impact_coverage_aggregate.json`, `logs/taskitem_runs/TEST_ONLY_sprint242_impact_remediation_loop_20260226/remediation_loop_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint243_impact_remediation_tasks_loop_20260226_r2/remediation_loop_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint244_autofill_20260226_151651/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint244_autofill_enforce_20260226_151709/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint245_intrinsic_20260226_151835/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint246_multishot_20260226_153033/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint246_multishot_enforce_20260226_153045/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint247_singleshot_20260226_153230/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint247_singleshot_enforce_20260226_153241/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint248_rawsearch_20260226_153903/00_summary.json`, `logs/taskitem_runs/TEST_ONLY_sprint249_rawguard_20260226_154028/02ae_raw_candidate_search.json` | `partial` | Sprint 241 introduced impact profiles/gates; 242-243 remediation loops; 244 first-pass autofill; 245 intrinsic constraints only (no uplift); 246 multishot closes hard sample; 247 single-shot shaping closes with multishot+autofill disabled; 248 raw candidate search over 8 variants found no uplift (`selected_variant=0`, `7 -> 7`); 249 added no-uplift guardrail to hard-fail ineffective raw search attempts. Residual gap remains in raw generator single-shot quality without overlays. | Improve raw single-shot generator output so profile closure does not rely on shaping/multishot/autofill overlays | ## What Was Covered Today (Sprints 175-184) diff --git a/docs/progress_log_2026-02-26.md b/docs/progress_log_2026-02-26.md index d84fbf6..dfe2b59 100644 --- a/docs/progress_log_2026-02-26.md +++ b/docs/progress_log_2026-02-26.md @@ -295,3 +295,31 @@ - Enforced gate verification: - pass ON: `logs/taskitem_runs/TEST_ONLY_sprint247_singleshot_enforce_20260226_153241/00_summary.json` - fail OFF: `logs/taskitem_runs/TEST_ONLY_sprint247_singleshot_enforce_20260226_153242/06_native_impact_coverage.json` + +## Sprint 248 Added (Same Day) + +- Added raw generator candidate search path (no shaping/autofill/multishot overlays): + - `tools/mcp/score_native_tasks_profile_coverage.py` + - `WSTONE_NATIVE_RAW_CANDIDATE_SEARCH` + - `WSTONE_NATIVE_RAW_CANDIDATE_MAX_VARIANTS` + - summary packet: `native_raw_candidate_search` +- A/B artifacts: + - OFF: `logs/taskitem_runs/TEST_ONLY_sprint248_rawsearch_20260226_153902/00_summary.json` + - ON: `logs/taskitem_runs/TEST_ONLY_sprint248_rawsearch_20260226_153903/00_summary.json` +- Current measured result: + - `attempted_variants=8`, `selected_variant=0` + - `failing_profile_count 7 -> 7` + - no raw-only uplift on this hard sample. + +## Sprint 249 Added (Same Day) + +- Added raw candidate no-uplift guardrail: + - `WSTONE_NATIVE_RAW_CANDIDATE_REQUIRE_UPLIFT` + - artifact: `02ae_raw_candidate_search.json` + - hard fail code: `17` when uplift is required but absent +- Dated artifacts: + - guard OFF: `logs/taskitem_runs/TEST_ONLY_sprint249_rawguard_20260226_154026/00_summary.json` + - guard ON (expected fail): `logs/taskitem_runs/TEST_ONLY_sprint249_rawguard_20260226_154028/02ae_raw_candidate_search.json` +- Current measured result: + - baseline `7`, best `7` failing profiles + - run blocked correctly under require-uplift mode. diff --git a/docs/sprint248_execution_tracker_2026-02-26.md b/docs/sprint248_execution_tracker_2026-02-26.md new file mode 100644 index 0000000..3f22d19 --- /dev/null +++ b/docs/sprint248_execution_tracker_2026-02-26.md @@ -0,0 +1,38 @@ +# Sprint 248 Execution Tracker - 2026-02-26 + +## Scope +- `sprint248_plan.md` + +## Implemented + +New tool: +- `tools/mcp/score_native_tasks_profile_coverage.py` + - Scores any raw task set against active impact profiles. + +Pipeline integration in `tools/mcp/run_sprint_taskitem_pipeline.sh`: +- Added controls: + - `WSTONE_NATIVE_RAW_CANDIDATE_SEARCH` (default `0`) + - `WSTONE_NATIVE_RAW_CANDIDATE_MAX_VARIANTS` (default `8`) +- Added raw candidate generation/scoring artifacts: + - `02ae_candidate_.json` + - `02ae_candidate__tasks.json` + - `02ae_candidate__score.json` +- Added summary packet: + - `native_raw_candidate_search` + +## A/B Validation (no overlays) + +- candidate search OFF: + - `logs/taskitem_runs/TEST_ONLY_sprint248_rawsearch_20260226_153902/00_summary.json` +- candidate search ON: + - `logs/taskitem_runs/TEST_ONLY_sprint248_rawsearch_20260226_153903/00_summary.json` + +Observed result: +- attempted variants: `8` +- selected variant: `0` (base) +- `failing_profile_count: 7 -> 7` +- no raw candidate uplift on this hard sample. + +## Explicit Completion Signal + +- Sprint 248: `DONE` (implemented + measured; no uplift on current sample) diff --git a/docs/sprint249_execution_tracker_2026-02-26.md b/docs/sprint249_execution_tracker_2026-02-26.md new file mode 100644 index 0000000..b9f4cba --- /dev/null +++ b/docs/sprint249_execution_tracker_2026-02-26.md @@ -0,0 +1,30 @@ +# Sprint 249 Execution Tracker - 2026-02-26 + +## Scope +- `sprint249_plan.md` + +## Implemented + +Pipeline updates in `tools/mcp/run_sprint_taskitem_pipeline.sh`: +- Added `WSTONE_NATIVE_RAW_CANDIDATE_REQUIRE_UPLIFT` (default `0`). +- `native_raw_candidate_search` now reports: + - `baseline_failing_profile_count` + - `best_failing_profile_count` + - baseline/best task counts +- Added artifact: + - `02ae_raw_candidate_search.json` +- Added hard-fail behavior (`exit 17`) when uplift is required but not achieved. + +## Validation Artifacts + +- guard disabled (records no-uplift signal, continues): + - `logs/taskitem_runs/TEST_ONLY_sprint249_rawguard_20260226_154026/00_summary.json` +- guard enabled (expected hard fail): + - `logs/taskitem_runs/TEST_ONLY_sprint249_rawguard_20260226_154028/02ae_raw_candidate_search.json` + +Observed signal: +- baseline and best both `7` failing profiles (`selected_variant=0`), so no uplift. + +## Explicit Completion Signal + +- Sprint 249: `DONE` (implemented + expected no-uplift guard behavior verified) diff --git a/editor/src/Sprint248IntegrationSummary.h b/editor/src/Sprint248IntegrationSummary.h new file mode 100644 index 0000000..baa658c --- /dev/null +++ b/editor/src/Sprint248IntegrationSummary.h @@ -0,0 +1,6 @@ +#pragma once + +// Sprint 248 integration summary: +// - Added raw candidate search over multiple profile-guided generation variants. +// - Selection chooses best raw variant by profile-failure score then task count. +// - On current hard sample, no raw-variant uplift was observed. diff --git a/editor/src/Sprint249IntegrationSummary.h b/editor/src/Sprint249IntegrationSummary.h new file mode 100644 index 0000000..7c550bf --- /dev/null +++ b/editor/src/Sprint249IntegrationSummary.h @@ -0,0 +1,6 @@ +#pragma once + +// Sprint 249 integration summary: +// - Added no-uplift guardrail for raw candidate search mode. +// - Raw search now records baseline and best profile-failure metrics. +// - Optional hard-fail mode blocks runs when raw search provides no uplift. diff --git a/sprint248_plan.md b/sprint248_plan.md new file mode 100644 index 0000000..f333a40 --- /dev/null +++ b/sprint248_plan.md @@ -0,0 +1,11 @@ +# Sprint 248 Plan: Raw Generator Candidate Search (No Overlays) + +## Goal +Probe whether raw generator outputs can be improved without shaping/autofill/multishot by searching multiple profile-guided requirement variants and selecting the best raw candidate. + +## Steps +- Step 2285: Add raw task-set profile coverage scoring tool. +- Step 2286: Add optional raw candidate search path in pipeline. +- Step 2287: Generate and score multiple raw generation variants. +- Step 2288: Select best raw variant by failing-profile score and task count. +- Step 2289: Add `Sprint248IntegrationSummary.h` and execution tracker. diff --git a/sprint249_plan.md b/sprint249_plan.md new file mode 100644 index 0000000..db18939 --- /dev/null +++ b/sprint249_plan.md @@ -0,0 +1,11 @@ +# Sprint 249 Plan: Raw Candidate No-Uplift Guardrail + +## Goal +Prevent false confidence from raw candidate search runs that try variants but produce no measurable uplift. + +## Steps +- Step 2290: Add baseline vs best raw candidate comparison telemetry. +- Step 2291: Emit raw candidate search artifact for every enabled run. +- Step 2292: Add optional hard-fail guard when no failing-profile uplift is achieved. +- Step 2293: Validate expected pass/fail behavior for guardrail mode. +- Step 2294: Add `Sprint249IntegrationSummary.h` and execution tracker. diff --git a/tools/mcp/run_sprint_taskitem_pipeline.sh b/tools/mcp/run_sprint_taskitem_pipeline.sh index b600bb6..360abb9 100755 --- a/tools/mcp/run_sprint_taskitem_pipeline.sh +++ b/tools/mcp/run_sprint_taskitem_pipeline.sh @@ -49,6 +49,9 @@ NATIVE_SINGLESHOT_PROFILE_SHAPE="${WSTONE_NATIVE_SINGLESHOT_PROFILE_SHAPE:-0}" NATIVE_MULTISHOT_DECOMP="${WSTONE_NATIVE_MULTISHOT_DECOMP:-0}" NATIVE_MULTISHOT_MAX_PROFILES="${WSTONE_NATIVE_MULTISHOT_MAX_PROFILES:-12}" NATIVE_MULTISHOT_APPLY_MODE="${WSTONE_NATIVE_MULTISHOT_APPLY_MODE:-if_improves}" +NATIVE_RAW_CANDIDATE_SEARCH="${WSTONE_NATIVE_RAW_CANDIDATE_SEARCH:-0}" +NATIVE_RAW_CANDIDATE_MAX_VARIANTS="${WSTONE_NATIVE_RAW_CANDIDATE_MAX_VARIANTS:-8}" +NATIVE_RAW_CANDIDATE_REQUIRE_UPLIFT="${WSTONE_NATIVE_RAW_CANDIDATE_REQUIRE_UPLIFT:-0}" EXTRA_NORMALIZED_REQUIREMENTS_FILE="${WSTONE_EXTRA_NORMALIZED_REQUIREMENTS_FILE:-}" EXTRA_TASKS_FILE="${WSTONE_EXTRA_TASKS_FILE:-}" CAPABILITY_SIGNALS_JSON="${WSTONE_CAPABILITY_SIGNALS_JSON:-}" @@ -110,6 +113,7 @@ NATIVE_PROFILE_AUTOFILL_JSON='{}' NATIVE_INTRINSIC_BOOST_JSON='{}' NATIVE_SINGLESHOT_PROFILE_SHAPE_JSON='{}' NATIVE_MULTISHOT_JSON='{}' +NATIVE_RAW_CANDIDATE_SEARCH_JSON='{}' INTRINSIC_REQS_JSON='[]' EXTRA_NORMALIZED_REQUIREMENTS_JSON='[]' EXTRA_TASKS_JSON='[]' @@ -385,6 +389,84 @@ else --argjson target_min_task_count "$NATIVE_DECOMP_TARGET_MIN_TASKS" \ '{attempted:$attempted, applied:$applied, initial_task_count:$initial_task_count, target_min_task_count:$target_min_task_count}')" fi +if [[ "$NATIVE_RAW_CANDIDATE_SEARCH" == "1" ]]; then + if [[ "$(printf '%s' "$INTRINSIC_REQS_JSON" | jq 'length')" -eq 0 ]]; then + python3 "$ROOT_DIR/tools/mcp/synthesize_native_intrinsic_boost_requirements.py" \ + --spec "$INPUT_FILE" \ + --profiles "$NATIVE_IMPACT_COVERAGE_PROFILES" \ + --out "$OUT_DIR/02ae_raw_candidate_requirements.json" >/dev/null + INTRINSIC_REQS_JSON="$(cat "$OUT_DIR/02ae_raw_candidate_requirements.json")" + fi + + printf '%s\n' "$TASKS" > "$OUT_DIR/02ae_candidate_0_tasks.json" + python3 "$ROOT_DIR/tools/mcp/score_native_tasks_profile_coverage.py" \ + --spec "$INPUT_FILE" \ + --profiles "$NATIVE_IMPACT_COVERAGE_PROFILES" \ + --tasks "$OUT_DIR/02ae_candidate_0_tasks.json" \ + --out "$OUT_DIR/02ae_candidate_0_score.json" >/dev/null + best_variant="0" + best_fail="$(jq '.failing_profile_count // 999' "$OUT_DIR/02ae_candidate_0_score.json")" + best_task_count="$(jq '.task_count // 0' "$OUT_DIR/02ae_candidate_0_score.json")" + baseline_fail="$best_fail" + baseline_task_count="$best_task_count" + attempted=1 + successful=1 + + req_total="$(printf '%s' "$INTRINSIC_REQS_JSON" | jq 'length')" + max_variants="$NATIVE_RAW_CANDIDATE_MAX_VARIANTS" + idx=0 + variant=1 + while [[ "$variant" -lt "$max_variants" && "$idx" -lt "$req_total" ]]; do + req_i="$(printf '%s' "$INTRINSIC_REQS_JSON" | jq ".[$idx]")" + cand_reqs="$(jq -nc --argjson base "$NORMALIZED_REQS" --argjson req "$req_i" '$base + [$req]')" + cand_args="$(jq -nc --argjson nr "$cand_reqs" --argjson cf "$CONFLICTS" --arg strict "$STRICT_EXECUTION_CONTRACT" \ + '{normalizedRequirements:$nr,conflicts:$cf,strictExecutionContract:($strict == "1")}')" + cand_raw="$(call_tool "whetstone_generate_taskitems" "$cand_args")" + printf '%s\n' "$cand_raw" > "$OUT_DIR/02ae_candidate_${variant}_raw.ndjson.json" + cand_json="$(extract_tool_text_json "$cand_raw")" + printf '%s\n' "$cand_json" > "$OUT_DIR/02ae_candidate_${variant}.json" + attempted=$((attempted + 1)) + if [[ "$(printf '%s' "$cand_json" | jq -r '.success // false')" == "true" ]]; then + cand_tasks="$(printf '%s' "$cand_json" | jq '.tasks // []')" + printf '%s\n' "$cand_tasks" > "$OUT_DIR/02ae_candidate_${variant}_tasks.json" + python3 "$ROOT_DIR/tools/mcp/score_native_tasks_profile_coverage.py" \ + --spec "$INPUT_FILE" \ + --profiles "$NATIVE_IMPACT_COVERAGE_PROFILES" \ + --tasks "$OUT_DIR/02ae_candidate_${variant}_tasks.json" \ + --out "$OUT_DIR/02ae_candidate_${variant}_score.json" >/dev/null + cand_fail="$(jq '.failing_profile_count // 999' "$OUT_DIR/02ae_candidate_${variant}_score.json")" + cand_task_count="$(jq '.task_count // 0' "$OUT_DIR/02ae_candidate_${variant}_score.json")" + if [[ "$cand_fail" -lt "$best_fail" || ( "$cand_fail" -eq "$best_fail" && "$cand_task_count" -gt "$best_task_count" ) ]]; then + best_variant="$variant" + best_fail="$cand_fail" + best_task_count="$cand_task_count" + TASKS="$cand_tasks" + fi + successful=$((successful + 1)) + fi + idx=$((idx + 1)) + variant=$((variant + 1)) + done + + NATIVE_RAW_CANDIDATE_SEARCH_JSON="$(jq -nc \ + --argjson enabled true \ + --argjson attempted "$attempted" \ + --argjson successful "$successful" \ + --arg best_variant "$best_variant" \ + --argjson baseline_failing_profile_count "$baseline_fail" \ + --argjson baseline_task_count "$baseline_task_count" \ + --argjson best_failing_profile_count "$best_fail" \ + --argjson best_task_count "$best_task_count" \ + '{enabled:$enabled, attempted_variants:$attempted, successful_variants:$successful, selected_variant:$best_variant, baseline_failing_profile_count:$baseline_failing_profile_count, baseline_task_count:$baseline_task_count, best_failing_profile_count:$best_failing_profile_count, best_task_count:$best_task_count}')" + printf '%s\n' "$NATIVE_RAW_CANDIDATE_SEARCH_JSON" > "$OUT_DIR/02ae_raw_candidate_search.json" + if [[ "$NATIVE_RAW_CANDIDATE_REQUIRE_UPLIFT" == "1" && "$best_fail" -ge "$baseline_fail" ]]; then + echo "error: raw candidate search did not improve failing profile count" >&2 + echo "error: see $OUT_DIR/02ae_raw_candidate_search.json" >&2 + exit 17 + fi +else + NATIVE_RAW_CANDIDATE_SEARCH_JSON='{"enabled":false}' +fi if [[ "$NATIVE_SINGLESHOT_PROFILE_SHAPE" == "1" ]]; then printf '%s\n' "$TASKS" > "$OUT_DIR/02ad_single_shot_base_tasks.json" python3 "$ROOT_DIR/tools/mcp/synthesize_single_shot_profile_shape_tasks.py" \ @@ -704,6 +786,7 @@ SUMMARY_JSON="$(jq -nc \ --argjson native_decomposition_retry "$NATIVE_DECOMP_RETRY_JSON" \ --argjson native_profile_autofill "$NATIVE_PROFILE_AUTOFILL_JSON" \ --argjson native_intrinsic_boost "$NATIVE_INTRINSIC_BOOST_JSON" \ + --argjson native_raw_candidate_search "$NATIVE_RAW_CANDIDATE_SEARCH_JSON" \ --argjson native_single_shot_profile_shape "$NATIVE_SINGLESHOT_PROFILE_SHAPE_JSON" \ --argjson native_multishot "$NATIVE_MULTISHOT_JSON" \ --argjson extra_normalized_requirements "$EXTRA_NORMALIZED_REQUIREMENTS_JSON" \ @@ -731,6 +814,7 @@ SUMMARY_JSON="$(jq -nc \ native_decomposition_retry: $native_decomposition_retry, native_profile_autofill: $native_profile_autofill, native_intrinsic_boost: $native_intrinsic_boost, + native_raw_candidate_search: $native_raw_candidate_search, native_single_shot_profile_shape: $native_single_shot_profile_shape, native_multishot: $native_multishot, extra_normalized_requirements: $extra_normalized_requirements, diff --git a/tools/mcp/score_native_tasks_profile_coverage.py b/tools/mcp/score_native_tasks_profile_coverage.py new file mode 100755 index 0000000..50417f3 --- /dev/null +++ b/tools/mcp/score_native_tasks_profile_coverage.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +import argparse +import json +from pathlib import Path +from typing import Dict, List + + +def load_json(path: Path): + with path.open("r", encoding="utf-8") as f: + return json.load(f) + + +def activate_profiles(profiles: List[Dict], spec_text: str) -> List[Dict]: + s = spec_text.lower() + out = [] + for p in profiles: + keys = [str(k).lower() for k in (p.get("trigger_keywords") or [])] + if keys and any(k in s for k in keys): + out.append(p) + return out + + +def collect_task_unions(tasks: List[Dict]): + reasons: List[str] = [] + prereq_ops: List[str] = [] + contracts: List[Dict] = [] + for t in tasks: + for r in (t.get("reasons") or []): + reasons.append(str(r).lower()) + for op in (t.get("prerequisiteOps") or []): + prereq_ops.append(str(op)) + ec = t.get("executionContract") or {} + if isinstance(ec, dict): + contracts.append(ec) + return reasons, prereq_ops, contracts + + +def check_profile(profile: Dict, tasks: List[Dict]) -> Dict: + reasons, prereq_ops, contracts = collect_task_unions(tasks) + missing: List[str] = [] + + min_tasks = int(profile.get("min_native_task_count", 0) or 0) + if len(tasks) < min_tasks: + missing.append(f"native_task_count<{min_tasks}") + + for key in [str(x).lower() for x in (profile.get("required_reason_keywords") or [])]: + if not any(key in r for r in reasons): + missing.append(f"missing_reason_keyword:{key}") + + ops_set = set(prereq_ops) + for op in [str(x) for x in (profile.get("required_prerequisite_ops") or [])]: + if op not in ops_set: + missing.append(f"missing_prerequisite_op:{op}") + + for field in [str(x) for x in (profile.get("required_execution_contract") or [])]: + ok = False + for ec in contracts: + if isinstance(ec.get(field), bool): + if ec.get(field): + ok = True + break + elif field in ec: + ok = True + break + if not ok: + missing.append(f"missing_execution_contract:{field}") + + return {"id": profile.get("id", "unknown"), "passed": len(missing) == 0, "missing": missing} + + +def main() -> int: + p = argparse.ArgumentParser(description="Score raw task set against active impact profiles.") + p.add_argument("--spec", required=True) + p.add_argument("--profiles", required=True) + p.add_argument("--tasks", required=True) + p.add_argument("--out", required=True) + args = p.parse_args() + + spec_text = Path(args.spec).read_text(encoding="utf-8", errors="ignore") + profiles_doc = load_json(Path(args.profiles)) + profiles = list(profiles_doc.get("profiles") or []) + active = activate_profiles(profiles, spec_text) + + tasks = load_json(Path(args.tasks)) + if not isinstance(tasks, list): + raise SystemExit("--tasks must be JSON array") + + checks = [check_profile(pf, tasks) for pf in active] + failing = [c for c in checks if not c.get("passed", False)] + + result = { + "status": "ok" if not failing else "fail", + "task_count": len(tasks), + "active_profile_count": len(active), + "failing_profile_count": len(failing), + "checks": checks, + } + with Path(args.out).open("w", encoding="utf-8") as f: + json.dump(result, f, indent=2, sort_keys=True) + f.write("\n") + + print(json.dumps({"status": result["status"], "task_count": result["task_count"], "failing_profile_count": result["failing_profile_count"], "out": args.out}, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())