bill/whetstone_DSL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add richer raw variants and history-aware ladder routing (sprints 251-252)

Browse Source
This commit is contained in:
Bill
2026-02-26 15:46:11 -07:00
parent 2c422fedda
commit 28b29dfd30
13 changed files with 368 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
tools/mcp/analyze_raw_candidate_uplift_history.py Executable file
View File

@@ -0,0 +1,64 @@
#!/usr/bin/env python3
import argparse
import json
from pathlib import Path
from typing import Dict, List
def load_json(path: Path):
with path.open("r", encoding="utf-8") as f:
return json.load(f)
def main() -> int:
p = argparse.ArgumentParser(description="Analyze recent raw candidate search uplift history.")
p.add_argument("--runs-root", default="logs/taskitem_runs")
p.add_argument("--include-glob", default="*")
p.add_argument("--max-runs", type=int, default=20)
p.add_argument("--out", required=True)
args = p.parse_args()
root = Path(args.runs_root)
summaries: List[Path] = sorted(root.glob(f"{args.include_glob}/00_summary.json"), key=lambda x: x.stat().st_mtime, reverse=True)
rows = []
for path in summaries[: max(0, args.max_runs)]:
try:
s = load_json(path)
except Exception:
continue
raw = s.get("native_raw_candidate_search") or {}
if not isinstance(raw, dict) or not raw.get("enabled", False):
continue
baseline = int(raw.get("baseline_failing_profile_count", 999) or 999)
best = int(raw.get("best_failing_profile_count", 999) or 999)
rows.append({
"run_id": path.parent.name,
"baseline_failing_profile_count": baseline,
"best_failing_profile_count": best,
"uplift": int(best < baseline),
})
total = len(rows)
uplift_count = sum(int(r.get("uplift", 0)) for r in rows)
no_uplift_count = total - uplift_count
out = {
"status": "ok",
"record_count": total,
"uplift_count": uplift_count,
"no_uplift_count": no_uplift_count,
"uplift_rate": round((uplift_count / total), 4) if total else 0.0,
"records": rows,
}
with Path(args.out).open("w", encoding="utf-8") as f:
json.dump(out, f, indent=2, sort_keys=True)
f.write("\n")
print(json.dumps({"status": "ok", "record_count": total, "uplift_rate": out["uplift_rate"], "out": args.out}, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())

30
tools/mcp/run_native_profile_closure_ladder.sh
View File

@@ -14,6 +14,31 @@ if [[ ! -f "$SPEC_PATH" ]]; then
fi
mkdir -p "$OUT_DIR"
SKIP_RAW_WHEN_NO_UPLIFT="${WSTONE_CLOSURE_LADDER_SKIP_RAW_WHEN_NO_UPLIFT:-1}"
RAW_HISTORY_GLOB="${WSTONE_CLOSURE_LADDER_RAW_HISTORY_GLOB:-TEST_ONLY_sprint24*_raw*}"
RAW_HISTORY_MAX_RUNS="${WSTONE_CLOSURE_LADDER_RAW_HISTORY_MAX_RUNS:-20}"
RAW_MIN_UPLIFT_RATE="${WSTONE_CLOSURE_LADDER_RAW_MIN_UPLIFT_RATE:-0.05}"
RAW_MIN_RECORDS="${WSTONE_CLOSURE_LADDER_RAW_MIN_RECORDS:-3}"
effective_modes=(raw_only single_shot_shape multishot autofill)
if [[ "$SKIP_RAW_WHEN_NO_UPLIFT" == "1" ]]; then
python3 "$ROOT_DIR/tools/mcp/analyze_raw_candidate_uplift_history.py" \
--runs-root "$ROOT_DIR/logs/taskitem_runs" \
--include-glob "$RAW_HISTORY_GLOB" \
--max-runs "$RAW_HISTORY_MAX_RUNS" \
--out "$OUT_DIR/raw_uplift_history.json" >/dev/null || true
if [[ -f "$OUT_DIR/raw_uplift_history.json" ]]; then
raw_record_count="$(jq '.record_count // 0' "$OUT_DIR/raw_uplift_history.json")"
raw_uplift_rate="$(jq '.uplift_rate // 0' "$OUT_DIR/raw_uplift_history.json")"
if [[ "$raw_record_count" -ge "$RAW_MIN_RECORDS" ]]; then
awk_cmp="$(awk -v a="$raw_uplift_rate" -v b="$RAW_MIN_UPLIFT_RATE" 'BEGIN{if (a < b) print 1; else print 0;}')"
if [[ "$awk_cmp" == "1" ]]; then
effective_modes=(single_shot_shape multishot autofill)
fi
fi
fi
fi
run_mode() {
local mode="$1"
local logfile="$OUT_DIR/${mode}.log"
@@ -65,12 +90,11 @@ run_mode() {
return "$rc"
}
modes=(raw_only single_shot_shape multishot autofill)
selected=""
selected_run=""
selected_rc=1
for mode in "${modes[@]}"; do
for mode in "${effective_modes[@]}"; do
if run_mode "$mode"; then
selected="$mode"
selected_run="$(cat "$OUT_DIR/${mode}.run_dir")"
@@ -85,7 +109,7 @@ jq -n \
--arg selected_mode "$selected" \
--arg selected_run "$selected_run" \
--argjson selected_rc "$selected_rc" \
--argjson attempted_modes "$(printf '%s\n' "${modes[@]}" | jq -R . | jq -s .)" \
--argjson attempted_modes "$(printf '%s\n' "${effective_modes[@]}" | jq -R . | jq -s .)" \
'{
status: (if $selected_rc == 0 then "ok" else "fail" end),
spec:$spec,

24
tools/mcp/run_sprint_taskitem_pipeline.sh
View File

@@ -390,13 +390,12 @@ else
'{attempted:$attempted, applied:$applied, initial_task_count:$initial_task_count, target_min_task_count:$target_min_task_count}')"
fi
if [[ "$NATIVE_RAW_CANDIDATE_SEARCH" == "1" ]]; then
if [[ "$(printf '%s' "$INTRINSIC_REQS_JSON" | jq 'length')" -eq 0 ]]; then
python3 "$ROOT_DIR/tools/mcp/synthesize_native_intrinsic_boost_requirements.py" \
--spec "$INPUT_FILE" \
--profiles "$NATIVE_IMPACT_COVERAGE_PROFILES" \
--out "$OUT_DIR/02ae_raw_candidate_requirements.json" >/dev/null
INTRINSIC_REQS_JSON="$(cat "$OUT_DIR/02ae_raw_candidate_requirements.json")"
fi
python3 "$ROOT_DIR/tools/mcp/synthesize_raw_candidate_requirement_variants.py" \
--spec "$INPUT_FILE" \
--profiles "$NATIVE_IMPACT_COVERAGE_PROFILES" \
--max-variants "$NATIVE_RAW_CANDIDATE_MAX_VARIANTS" \
--out "$OUT_DIR/02ae_raw_candidate_variants.json" >/dev/null
VARIANTS_JSON="$(cat "$OUT_DIR/02ae_raw_candidate_variants.json")"
printf '%s\n' "$TASKS" > "$OUT_DIR/02ae_candidate_0_tasks.json"
python3 "$ROOT_DIR/tools/mcp/score_native_tasks_profile_coverage.py" \
@@ -412,13 +411,13 @@ if [[ "$NATIVE_RAW_CANDIDATE_SEARCH" == "1" ]]; then
attempted=1
successful=1
req_total="$(printf '%s' "$INTRINSIC_REQS_JSON" | jq 'length')"
variant_total="$(printf '%s' "$VARIANTS_JSON" | jq '.variants | length')"
max_variants="$NATIVE_RAW_CANDIDATE_MAX_VARIANTS"
idx=0
variant=1
while [[ "$variant" -lt "$max_variants" && "$idx" -lt "$req_total" ]]; do
req_i="$(printf '%s' "$INTRINSIC_REQS_JSON" | jq ".[$idx]")"
cand_reqs="$(jq -nc --argjson base "$NORMALIZED_REQS" --argjson req "$req_i" '$base + [$req]')"
while [[ "$variant" -lt "$max_variants" && "$idx" -lt "$variant_total" ]]; do
req_bundle="$(printf '%s' "$VARIANTS_JSON" | jq ".variants[$idx]")"
cand_reqs="$(jq -nc --argjson base "$NORMALIZED_REQS" --argjson bundle "$req_bundle" '$base + $bundle')"
cand_args="$(jq -nc --argjson nr "$cand_reqs" --argjson cf "$CONFLICTS" --arg strict "$STRICT_EXECUTION_CONTRACT" \
'{normalizedRequirements:$nr,conflicts:$cf,strictExecutionContract:($strict == "1")}')"
cand_raw="$(call_tool "whetstone_generate_taskitems" "$cand_args")"
@@ -457,7 +456,8 @@ if [[ "$NATIVE_RAW_CANDIDATE_SEARCH" == "1" ]]; then
--argjson baseline_task_count "$baseline_task_count" \
--argjson best_failing_profile_count "$best_fail" \
--argjson best_task_count "$best_task_count" \
'{enabled:$enabled, attempted_variants:$attempted, successful_variants:$successful, selected_variant:$best_variant, baseline_failing_profile_count:$baseline_failing_profile_count, baseline_task_count:$baseline_task_count, best_failing_profile_count:$best_failing_profile_count, best_task_count:$best_task_count}')"
--argjson available_variants "$variant_total" \
'{enabled:$enabled, attempted_variants:$attempted, successful_variants:$successful, available_variants:$available_variants, selected_variant:$best_variant, baseline_failing_profile_count:$baseline_failing_profile_count, baseline_task_count:$baseline_task_count, best_failing_profile_count:$best_failing_profile_count, best_task_count:$best_task_count}')"
printf '%s\n' "$NATIVE_RAW_CANDIDATE_SEARCH_JSON" > "$OUT_DIR/02ae_raw_candidate_search.json"
if [[ "$NATIVE_RAW_CANDIDATE_REQUIRE_UPLIFT" == "1" && "$best_fail" -ge "$baseline_fail" ]]; then
echo "error: raw candidate search did not improve failing profile count" >&2

131
tools/mcp/synthesize_raw_candidate_requirement_variants.py Executable file
View File

@@ -0,0 +1,131 @@
#!/usr/bin/env python3
import argparse
import json
from pathlib import Path
from typing import Dict, List
def load_json(path: Path):
with path.open("r", encoding="utf-8") as f:
return json.load(f)
def write_json(path: Path, obj) -> None:
with path.open("w", encoding="utf-8") as f:
json.dump(obj, f, indent=2, sort_keys=True)
f.write("\n")
def activate_profiles(profiles: List[Dict], spec_text: str) -> List[Dict]:
s = spec_text.lower()
out = []
for p in profiles:
keys = [str(k).lower() for k in (p.get("trigger_keywords") or [])]
if keys and any(k in s for k in keys):
out.append(p)
return out
def req_from_profile(p: Dict, idx: int) -> Dict:
pid = str(p.get("id", f"profile_{idx}"))
min_tasks = int(p.get("min_native_task_count", 0) or 0)
reasons = [str(x) for x in (p.get("required_reason_keywords") or [])]
ops = [str(x) for x in (p.get("required_prerequisite_ops") or [])]
contract = [str(x) for x in (p.get("required_execution_contract") or [])]
text = (
f"raw candidate policy {pid}: produce >= {min_tasks} concrete taskitems; "
f"reasons include {', '.join(reasons) if reasons else 'none'}; "
f"ops include {', '.join(ops) if ops else 'none'}; "
f"execution contract include {', '.join(contract) if contract else 'none'}; "
"avoid generic or umbrella tasks."
)
return {
"requirementId": f"raw-variant-{pid}",
"kind": "constraint",
"normalizedText": text,
"anchor": "raw_candidate_variant",
"sourceLine": 0,
"ambiguous": False,
}
def global_decomposition_req(active_count: int, min_target: int) -> Dict:
return {
"requirementId": "raw-variant-global-decomposition",
"kind": "constraint",
"normalizedText": (
f"raw decomposition hard policy: output at least {min_target} concrete taskitems "
f"for {active_count} active impact profiles; each task must have deterministic executionContract and explicit prerequisiteOps."
),
"anchor": "raw_candidate_variant",
"sourceLine": 0,
"ambiguous": False,
}
def build_variants(active: List[Dict], max_variants: int) -> List[List[Dict]]:
if max_variants <= 0:
return []
profile_reqs = [req_from_profile(p, i) for i, p in enumerate(active)]
min_target = max(5, len(active) + 2)
g = global_decomposition_req(len(active), min_target)
variants: List[List[Dict]] = []
# Variant 1: global only
variants.append([g])
# Variant 2: global + all profiles
variants.append([g] + profile_reqs)
# Variant 3+: one profile at a time + global
for r in profile_reqs:
variants.append([g, r])
# Variant: paired profile bundles
for i in range(0, len(profile_reqs), 2):
bundle = [g] + profile_reqs[i:i+2]
variants.append(bundle)
# Deduplicate by requirementId tuple ordering
seen = set()
out: List[List[Dict]] = []
for v in variants:
key = tuple(x.get("requirementId", "") for x in v)
if key in seen:
continue
seen.add(key)
out.append(v)
if len(out) >= max_variants:
break
return out
def main() -> int:
p = argparse.ArgumentParser(description="Synthesize raw candidate requirement variants from active profiles.")
p.add_argument("--spec", required=True)
p.add_argument("--profiles", required=True)
p.add_argument("--max-variants", type=int, default=8)
p.add_argument("--out", required=True)
args = p.parse_args()
spec_text = Path(args.spec).read_text(encoding="utf-8", errors="ignore")
profiles_doc = load_json(Path(args.profiles))
profiles = list(profiles_doc.get("profiles") or [])
active = activate_profiles(profiles, spec_text)
variants = build_variants(active, args.max_variants)
payload = {
"status": "ok",
"active_profile_count": len(active),
"variant_count": len(variants),
"variants": variants,
}
write_json(Path(args.out), payload)
print(json.dumps({"status": "ok", "active_profile_count": len(active), "variant_count": len(variants), "out": args.out}, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())