bill/whetstone_DSL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add impact-by-impact native decomposition coverage gates (sprint 241)

Browse Source
This commit is contained in:
Bill
2026-02-26 15:05:21 -07:00
parent e9dd06205e
commit 3c04216a54
12 changed files with 559 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
tools/mcp/analyze_native_impact_coverage.py Executable file
View File

@@ -0,0 +1,61 @@
#!/usr/bin/env python3
import argparse
import json
from pathlib import Path
from typing import Dict
def load_json(path: Path) -> Dict:
with path.open("r", encoding="utf-8") as f:
return json.load(f)
def write_json(path: Path, obj: Dict) -> None:
with path.open("w", encoding="utf-8") as f:
json.dump(obj, f, indent=2, sort_keys=True)
f.write("\n")
def main() -> int:
p = argparse.ArgumentParser(description="Aggregate native impact coverage reports across runs.")
p.add_argument("--runs-root", default="logs/taskitem_runs")
p.add_argument("--include-glob", default="*")
p.add_argument("--out", required=True)
args = p.parse_args()
root = Path(args.runs_root)
profile_counts: Dict[str, Dict[str, int]] = {}
total_runs = 0
failed_runs = 0
for report_path in sorted(root.glob(f"{args.include_glob}/06_native_impact_coverage.json")):
try:
report = load_json(report_path)
except Exception:
continue
total_runs += 1
if report.get("status") == "fail":
failed_runs += 1
for c in report.get("checks", []):
pid = str(c.get("id", "unknown"))
row = profile_counts.setdefault(pid, {"active": 0, "passed": 0, "failed": 0})
row["active"] += 1
if c.get("passed", False):
row["passed"] += 1
else:
row["failed"] += 1
out = {
"status": "ok",
"total_runs": total_runs,
"failed_runs": failed_runs,
"fail_rate": round((failed_runs / total_runs), 4) if total_runs else 0.0,
"profile_counts": profile_counts,
}
write_json(Path(args.out), out)
print(json.dumps({"status": "ok", "total_runs": total_runs, "failed_runs": failed_runs, "out": args.out}, sort_keys=True))
return 0
if __name__ == "__main__":
raise SystemExit(main())

166
tools/mcp/check_native_decomposition_impact_coverage.py Executable file
View File

@@ -0,0 +1,166 @@
#!/usr/bin/env python3
import argparse
import json
from pathlib import Path
from typing import Dict, List, Tuple
def load_json(path: Path) -> Dict:
with path.open("r", encoding="utf-8") as f:
return json.load(f)
def write_json(path: Path, obj: Dict) -> None:
with path.open("w", encoding="utf-8") as f:
json.dump(obj, f, indent=2, sort_keys=True)
f.write("\n")
def load_native_tasks(run_dir: Path, summary: Dict) -> List[Dict]:
retry = summary.get("native_decomposition_retry") or {}
retry_applied = bool(retry.get("applied", False))
retry_path = run_dir / "02aa_generate_taskitems_retry.json"
base_path = run_dir / "02_generate_taskitems.json"
if retry_applied and retry_path.exists():
data = load_json(retry_path)
if isinstance(data, dict):
return list(data.get("tasks") or [])
if base_path.exists():
data = load_json(base_path)
if isinstance(data, dict):
return list(data.get("tasks") or [])
return []
def load_spec_text(summary: Dict, repo_root: Path) -> str:
input_file = str(summary.get("input_file", "")).strip()
if not input_file:
return ""
p = Path(input_file)
if not p.is_absolute():
p = repo_root / input_file
try:
return p.read_text(encoding="utf-8", errors="ignore")
except Exception:
return ""
def activate_profiles(profiles: List[Dict], spec_text: str) -> List[Dict]:
s = spec_text.lower()
active = []
for profile in profiles:
keys = [str(k).lower() for k in (profile.get("trigger_keywords") or [])]
if not keys:
continue
if any(k in s for k in keys):
active.append(profile)
return active
def collect_task_unions(tasks: List[Dict]) -> Tuple[List[str], List[str], List[Dict]]:
reasons: List[str] = []
prereq_ops: List[str] = []
contracts: List[Dict] = []
for t in tasks:
rs = t.get("reasons") or []
po = t.get("prerequisiteOps") or []
ec = t.get("executionContract") or {}
for r in rs:
reasons.append(str(r).lower())
for op in po:
prereq_ops.append(str(op))
if isinstance(ec, dict):
contracts.append(ec)
return reasons, prereq_ops, contracts
def check_profile(profile: Dict, tasks: List[Dict]) -> Dict:
reasons, prereq_ops, contracts = collect_task_unions(tasks)
missing: List[str] = []
min_tasks = int(profile.get("min_native_task_count", 0) or 0)
if len(tasks) < min_tasks:
missing.append(f"native_task_count<{min_tasks}")
req_reason_keys = [str(x).lower() for x in (profile.get("required_reason_keywords") or [])]
for key in req_reason_keys:
if not any(key in r for r in reasons):
missing.append(f"missing_reason_keyword:{key}")
req_ops = [str(x) for x in (profile.get("required_prerequisite_ops") or [])]
ops_set = set(prereq_ops)
for op in req_ops:
if op not in ops_set:
missing.append(f"missing_prerequisite_op:{op}")
req_contract = [str(x) for x in (profile.get("required_execution_contract") or [])]
for field in req_contract:
has_field_true = False
for ec in contracts:
if isinstance(ec.get(field), bool):
if ec.get(field) is True:
has_field_true = True
break
elif field in ec:
has_field_true = True
break
if not has_field_true:
missing.append(f"missing_execution_contract:{field}")
return {
"id": profile.get("id", "unknown"),
"description": profile.get("description", ""),
"passed": len(missing) == 0,
"native_task_count": len(tasks),
"missing": missing,
}
def main() -> int:
parser = argparse.ArgumentParser(description="Check native decomposition coverage against impact profiles.")
parser.add_argument("--run-dir", required=True, help="Pipeline run directory containing 00_summary.json")
parser.add_argument("--profiles", default="tools/mcp/profiles/native_decomposition_impact_profiles.json", help="Profile JSON path")
parser.add_argument("--out", required=True, help="Output report path")
parser.add_argument("--enforce", action="store_true", help="Exit nonzero on failures")
args = parser.parse_args()
run_dir = Path(args.run_dir)
summary_path = run_dir / "00_summary.json"
if not summary_path.exists():
raise SystemExit(f"missing summary: {summary_path}")
summary = load_json(summary_path)
profiles_doc = load_json(Path(args.profiles))
profiles = list(profiles_doc.get("profiles") or [])
repo_root = Path(__file__).resolve().parents[2]
spec_text = load_spec_text(summary, repo_root)
active = activate_profiles(profiles, spec_text)
tasks = load_native_tasks(run_dir, summary)
checks = [check_profile(p, tasks) for p in active]
failing = [c for c in checks if not c.get("passed", False)]
result = {
"status": "ok" if not failing else "fail",
"run_dir": str(run_dir),
"active_profile_count": len(active),
"failing_profile_count": len(failing),
"active_profiles": [p.get("id", "unknown") for p in active],
"checks": checks,
}
write_json(Path(args.out), result)
print(json.dumps({
"status": result["status"],
"active_profile_count": result["active_profile_count"],
"failing_profile_count": result["failing_profile_count"],
"out": args.out,
}, sort_keys=True))
if args.enforce and failing:
return 12
return 0
if __name__ == "__main__":
raise SystemExit(main())

113
tools/mcp/profiles/native_decomposition_impact_profiles.json Normal file
View File

@@ -0,0 +1,113 @@
{
"schema": "native_decomposition_impact_profiles_v1",
"profiles": [
{
"id": "impact_api_contract_evolution",
"description": "API/schema evolution with compatibility obligations.",
"trigger_keywords": ["api", "openapi", "schema", "contract", "compatibility", "version"],
"min_native_task_count": 5,
"required_reason_keywords": ["contract", "compatibility"],
"required_prerequisite_ops": ["whetstone_generate_taskitems", "whetstone_validate_taskitem", "whetstone_queue_ready"],
"required_execution_contract": ["deterministic", "rollbackRequired", "replayValidationRequired"]
},
{
"id": "impact_state_migration_rollback",
"description": "State/data migrations that require rollback and data safety choreography.",
"trigger_keywords": ["migration", "rollback", "backfill", "data loss", "compatibility window"],
"min_native_task_count": 6,
"required_reason_keywords": ["migration", "risk"],
"required_prerequisite_ops": ["whetstone_generate_taskitems", "whetstone_validate_taskitem", "whetstone_queue_ready"],
"required_execution_contract": ["deterministic", "rollbackRequired", "replayValidationRequired"]
},
{
"id": "impact_security_policy_propagation",
"description": "Cross-layer security policy propagation and deny-by-default handling.",
"trigger_keywords": ["security", "auth", "permission", "deny-by-default", "threat"],
"min_native_task_count": 5,
"required_reason_keywords": ["security", "risk"],
"required_prerequisite_ops": ["whetstone_validate_taskitem", "whetstone_queue_ready"],
"required_execution_contract": ["deterministic", "rollbackRequired"]
},
{
"id": "impact_slo_latency_budget",
"description": "Performance/SLO constrained changes.",
"trigger_keywords": ["latency", "slo", "p95", "throughput", "budget"],
"min_native_task_count": 4,
"required_reason_keywords": ["performance", "risk"],
"required_prerequisite_ops": ["whetstone_validate_taskitem"],
"required_execution_contract": ["deterministic", "replayValidationRequired"]
},
{
"id": "impact_rollout_choreography",
"description": "Feature flags, staged rollout, and abort choreography.",
"trigger_keywords": ["rollout", "stage", "feature flag", "canary", "abort"],
"min_native_task_count": 5,
"required_reason_keywords": ["rollout", "risk"],
"required_prerequisite_ops": ["whetstone_queue_ready", "whetstone_validate_taskitem"],
"required_execution_contract": ["deterministic", "rollbackRequired"]
},
{
"id": "impact_multifile_transactional_edit",
"description": "Cross-file transactional edits where partial updates are unsafe.",
"trigger_keywords": ["multi-file", "cross-file", "transaction", "refactor", "across files"],
"min_native_task_count": 6,
"required_reason_keywords": ["transaction", "contract"],
"required_prerequisite_ops": ["whetstone_queue_ready", "whetstone_validate_taskitem"],
"required_execution_contract": ["deterministic", "rollbackRequired", "replayValidationRequired"]
},
{
"id": "impact_concurrency_locking",
"description": "Threading, locking, races, and scheduling semantics.",
"trigger_keywords": ["concurrency", "thread", "mutex", "lock", "race", "deadlock"],
"min_native_task_count": 5,
"required_reason_keywords": ["concurrency", "risk"],
"required_prerequisite_ops": ["whetstone_validate_taskitem", "whetstone_queue_ready"],
"required_execution_contract": ["deterministic", "replayValidationRequired"]
},
{
"id": "impact_projection_environment_constraints",
"description": "Target projection/environment constraints (embedded/mobile/serverless/hardware limits).",
"trigger_keywords": ["arduino", "embedded", "mobile", "ios", "android", "serverless", "memory limit", "hardware"],
"min_native_task_count": 4,
"required_reason_keywords": ["capability", "constraint"],
"required_prerequisite_ops": ["whetstone_architect_intake", "whetstone_validate_taskitem"],
"required_execution_contract": ["deterministic"]
},
{
"id": "impact_polyglot_boundary_changes",
"description": "Cross-language boundary and API client/server regeneration work.",
"trigger_keywords": ["polyglot", "cross-language", "sdk", "client", "backend", "frontend"],
"min_native_task_count": 5,
"required_reason_keywords": ["compatibility", "contract"],
"required_prerequisite_ops": ["whetstone_generate_taskitems", "whetstone_validate_taskitem"],
"required_execution_contract": ["deterministic", "replayValidationRequired"]
},
{
"id": "impact_observability_telemetry_integrity",
"description": "Logging/metrics/tracing changes requiring consistent telemetry semantics.",
"trigger_keywords": ["observability", "telemetry", "metrics", "tracing", "logging"],
"min_native_task_count": 4,
"required_reason_keywords": ["semantic", "contract"],
"required_prerequisite_ops": ["whetstone_validate_taskitem"],
"required_execution_contract": ["deterministic"]
},
{
"id": "impact_data_integrity_precision",
"description": "Financial/scientific precision and data-integrity sensitive changes.",
"trigger_keywords": ["financial", "precision", "scientific", "numerical", "rounding"],
"min_native_task_count": 5,
"required_reason_keywords": ["risk", "contract"],
"required_prerequisite_ops": ["whetstone_validate_taskitem"],
"required_execution_contract": ["deterministic", "replayValidationRequired"]
},
{
"id": "impact_disaster_recovery_replay",
"description": "Recovery/replay/rollback critical workflows.",
"trigger_keywords": ["recovery", "replay", "rollback", "incident", "failover"],
"min_native_task_count": 5,
"required_reason_keywords": ["risk", "replay"],
"required_prerequisite_ops": ["whetstone_validate_taskitem", "whetstone_queue_ready"],
"required_execution_contract": ["deterministic", "rollbackRequired", "replayValidationRequired"]
}
]
}

28
tools/mcp/run_native_impact_coverage_gate.sh Executable file
View File

@@ -0,0 +1,28 @@
#!/usr/bin/env bash
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
RUN_DIR="${1:-}"
if [[ -z "$RUN_DIR" ]]; then
echo "usage: $0 <run_dir> [out_json]" >&2
exit 2
fi
OUT_JSON="${2:-$RUN_DIR/06_native_impact_coverage.json}"
PROFILES="${WSTONE_NATIVE_IMPACT_COVERAGE_PROFILES:-$ROOT_DIR/tools/mcp/profiles/native_decomposition_impact_profiles.json}"
ENFORCE="${WSTONE_NATIVE_IMPACT_COVERAGE_ENFORCE:-1}"
args=(
--run-dir "$RUN_DIR"
--profiles "$PROFILES"
--out "$OUT_JSON"
)
if [[ "$ENFORCE" == "1" ]]; then
args+=(--enforce)
fi
if python3 "$ROOT_DIR/tools/mcp/check_native_decomposition_impact_coverage.py" "${args[@]}" >/dev/null; then
jq -n --arg run_dir "$RUN_DIR" --argjson report "$(cat "$OUT_JSON")" '{status:"pass", run_dir:$run_dir, report:$report}'
else
jq -n --arg run_dir "$RUN_DIR" --argjson report "$(cat "$OUT_JSON")" '{status:"fail", run_dir:$run_dir, report:$report}'
exit 12
fi

24
tools/mcp/run_sprint_taskitem_pipeline.sh
View File

@@ -39,6 +39,9 @@ NATIVE_SEMANTIC_SIGNAL_MIN="${WSTONE_NATIVE_SEMANTIC_SIGNAL_MIN:-0}"
NATIVE_REASON_ENRICHMENT="${WSTONE_NATIVE_REASON_ENRICHMENT:-1}"
NATIVE_DECOMP_RETRY="${WSTONE_NATIVE_DECOMP_RETRY:-1}"
NATIVE_DECOMP_TARGET_MIN_TASKS="${WSTONE_NATIVE_DECOMP_TARGET_MIN_TASKS:-5}"
NATIVE_IMPACT_COVERAGE_GATE="${WSTONE_NATIVE_IMPACT_COVERAGE_GATE:-0}"
NATIVE_IMPACT_COVERAGE_ENFORCE="${WSTONE_NATIVE_IMPACT_COVERAGE_ENFORCE:-0}"
NATIVE_IMPACT_COVERAGE_PROFILES="${WSTONE_NATIVE_IMPACT_COVERAGE_PROFILES:-$ROOT_DIR/tools/mcp/profiles/native_decomposition_impact_profiles.json}"
CAPABILITY_SIGNALS_JSON="${WSTONE_CAPABILITY_SIGNALS_JSON:-}"
if [[ -z "$CAPABILITY_SIGNALS_JSON" ]]; then
CAPABILITY_SIGNALS_JSON='{}'
@@ -93,6 +96,7 @@ SEMANTIC_GATE_JSON='{}'
NATIVE_DECOMP_GATE_JSON='{}'
NATIVE_REASON_ENRICHMENT_JSON='{}'
NATIVE_DECOMP_RETRY_JSON='{}'
NATIVE_IMPACT_COVERAGE_JSON='{}'
if [[ "$SEMANTIC_PLANNING_BRIDGE" == "1" ]]; then
python3 "$ROOT_DIR/tools/mcp/markdown_to_semantic_annotations.py" \
--spec "$INPUT_FILE" \
@@ -571,6 +575,26 @@ SUMMARY_JSON="$(jq -nc \
}
}')"
if [[ "$NATIVE_IMPACT_COVERAGE_GATE" == "1" ]]; then
# Emit a provisional summary so coverage checker can read run metadata.
printf '%s\n' "$SUMMARY_JSON" > "$OUT_DIR/00_summary.json"
if python3 "$ROOT_DIR/tools/mcp/check_native_decomposition_impact_coverage.py" \
--run-dir "$OUT_DIR" \
--profiles "$NATIVE_IMPACT_COVERAGE_PROFILES" \
--out "$OUT_DIR/06_native_impact_coverage.json" \
$([[ "$NATIVE_IMPACT_COVERAGE_ENFORCE" == "1" ]] && echo --enforce) >/dev/null; then
NATIVE_IMPACT_COVERAGE_JSON="$(cat "$OUT_DIR/06_native_impact_coverage.json")"
else
NATIVE_IMPACT_COVERAGE_JSON="$(cat "$OUT_DIR/06_native_impact_coverage.json")"
SUMMARY_JSON="$(printf '%s' "$SUMMARY_JSON" | jq --argjson nic "$NATIVE_IMPACT_COVERAGE_JSON" '.native_impact_coverage = $nic')"
printf '%s\n' "$SUMMARY_JSON" > "$OUT_DIR/00_summary.json"
echo "error: native impact coverage gate failed" >&2
echo "error: see $OUT_DIR/06_native_impact_coverage.json" >&2
exit 11
fi
SUMMARY_JSON="$(printf '%s' "$SUMMARY_JSON" | jq --argjson nic "$NATIVE_IMPACT_COVERAGE_JSON" '.native_impact_coverage = $nic')"
fi
if [[ "$CALIBRATE_AFTER_RUN" == "1" ]]; then
CALIBRATION_OUT_DIR="$OUT_DIR/calibration"
CALIBRATION_JSON="$(python3 "$ROOT_DIR/tools/mcp/analyze_taskitem_calibration.py" \