Implement sprints 170-171 strict gates and autonomous remediation loop
This commit is contained in:
Bill
@@ -1,18 +1,17 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Evaluate production gates for generated code.
|
||||
"""
|
||||
"""Evaluate production gates for generated code with real compile/test execution."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
from typing import Dict, List, Tuple
|
||||
|
||||
|
||||
PLACEHOLDER_PATTERNS = [
|
||||
@@ -24,6 +23,23 @@ PLACEHOLDER_PATTERNS = [
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExecResult:
|
||||
passed: bool
|
||||
skipped: bool
|
||||
reason: str
|
||||
return_code: int
|
||||
stdout: str
|
||||
stderr: str
|
||||
command: List[str]
|
||||
|
||||
|
||||
def truncate(text: str, limit: int = 6000) -> str:
|
||||
if len(text) <= limit:
|
||||
return text
|
||||
return text[: limit - 32] + "\n...<truncated>...\n"
|
||||
|
||||
|
||||
def detect_placeholders(code: str) -> Dict[str, object]:
|
||||
findings: List[Dict[str, object]] = []
|
||||
for pat in PLACEHOLDER_PATTERNS:
|
||||
@@ -32,51 +48,284 @@ def detect_placeholders(code: str) -> Dict[str, object]:
|
||||
return {"passed": len(findings) == 0, "count": len(findings), "findings": findings}
|
||||
|
||||
|
||||
def compile_check(code: str, language: str) -> Dict[str, object]:
|
||||
def run_cmd(cmd: List[str], timeout_s: int) -> ExecResult:
|
||||
try:
|
||||
p = subprocess.run(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
timeout=timeout_s,
|
||||
check=False,
|
||||
)
|
||||
return ExecResult(
|
||||
passed=(p.returncode == 0),
|
||||
skipped=False,
|
||||
reason="ok" if p.returncode == 0 else "non_zero_exit",
|
||||
return_code=p.returncode,
|
||||
stdout=truncate(p.stdout),
|
||||
stderr=truncate(p.stderr),
|
||||
command=cmd,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
return ExecResult(
|
||||
passed=False,
|
||||
skipped=True,
|
||||
reason="tool_missing",
|
||||
return_code=127,
|
||||
stdout="",
|
||||
stderr="",
|
||||
command=cmd,
|
||||
)
|
||||
except subprocess.TimeoutExpired as e:
|
||||
return ExecResult(
|
||||
passed=False,
|
||||
skipped=False,
|
||||
reason="timeout",
|
||||
return_code=124,
|
||||
stdout=truncate(e.stdout or ""),
|
||||
stderr=truncate(e.stderr or ""),
|
||||
command=cmd,
|
||||
)
|
||||
|
||||
|
||||
def language_tools(language: str) -> Dict[str, str]:
|
||||
lang = language.lower()
|
||||
with tempfile.TemporaryDirectory(prefix="whetstone_gate_") as td:
|
||||
if lang in ("cpp", "c++"):
|
||||
cxx = shutil.which("g++") or shutil.which("clang++") or ""
|
||||
return {"compile": cxx, "test": cxx}
|
||||
if lang == "python":
|
||||
py = shutil.which("python3") or shutil.which("python") or ""
|
||||
return {"compile": py, "test": py}
|
||||
if lang == "rust":
|
||||
rustc = shutil.which("rustc") or ""
|
||||
return {"compile": rustc, "test": rustc}
|
||||
if lang == "go":
|
||||
go = shutil.which("go") or ""
|
||||
return {"compile": go, "test": go}
|
||||
return {"compile": "", "test": ""}
|
||||
|
||||
|
||||
def detect_cpp_namespace_prefix(code: str) -> str:
|
||||
m = re.search(r"namespace\\s+([A-Za-z_][A-Za-z0-9_]*)\\s*\\{", code)
|
||||
if m:
|
||||
return m.group(1) + "::"
|
||||
q = re.search(r"([A-Za-z_][A-Za-z0-9_]*)::PriorityQueue", code)
|
||||
if q:
|
||||
return q.group(1) + "::"
|
||||
return ""
|
||||
|
||||
|
||||
def cpp_harness(code: str) -> str:
|
||||
prefix = detect_cpp_namespace_prefix(code)
|
||||
return (
|
||||
code
|
||||
+ "\nint main() {\n"
|
||||
+ f" {prefix}PriorityQueue q;\n"
|
||||
+ f" {prefix}WorkItem w(\"job\", 1, \"payload\");\n"
|
||||
+ " (void)q;\n"
|
||||
+ " (void)w;\n"
|
||||
+ " return 0;\n"
|
||||
+ "}\n"
|
||||
)
|
||||
|
||||
|
||||
def rust_harness(code: str) -> str:
|
||||
return (
|
||||
code
|
||||
+ "\nfn main() {\n"
|
||||
+ " let q = PriorityQueue{};\n"
|
||||
+ " let w = WorkItem{job_id: \"job\".to_string(), priority: 1, payload: \"payload\".to_string()};\n"
|
||||
+ " let _ = q;\n"
|
||||
+ " let _ = w;\n"
|
||||
+ "}\n"
|
||||
)
|
||||
|
||||
|
||||
def go_harness(code: str) -> str:
|
||||
return (
|
||||
code
|
||||
+ "\nfunc main() {\n"
|
||||
+ " q := PriorityQueue{}\n"
|
||||
+ " _ = q\n"
|
||||
+ "}\n"
|
||||
)
|
||||
|
||||
|
||||
def python_harness(code: str) -> str:
|
||||
return (
|
||||
code
|
||||
+ "\nif __name__ == '__main__':\n"
|
||||
+ " q = PriorityQueue()\n"
|
||||
+ " w = WorkItem('job', 1, 'payload')\n"
|
||||
+ " _ = q\n"
|
||||
+ " _ = w\n"
|
||||
)
|
||||
|
||||
|
||||
def compile_check(code: str, language: str, strict: bool) -> Dict[str, object]:
|
||||
lang = language.lower()
|
||||
tools = language_tools(lang)
|
||||
if lang not in ("cpp", "c++", "python", "rust", "go"):
|
||||
return {
|
||||
"passed": False,
|
||||
"skipped": True,
|
||||
"reason": "unsupported_language",
|
||||
"strict_blocking": bool(strict),
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="whetstone_gate_compile_") as td:
|
||||
td_path = Path(td)
|
||||
if lang in ("cpp", "c++"):
|
||||
src = td_path / "generated.cpp"
|
||||
src.write_text(code)
|
||||
cmd = ["g++", "-std=c++20", "-fsyntax-only", str(src)]
|
||||
elif lang in ("python", "py"):
|
||||
cmd = [tools["compile"] or "g++", "-std=c++20", "-fsyntax-only", str(src)]
|
||||
elif lang == "python":
|
||||
src = td_path / "generated.py"
|
||||
src.write_text(code)
|
||||
cmd = ["python3", "-m", "py_compile", str(src)]
|
||||
else:
|
||||
return {"passed": False, "skipped": True, "reason": f"compile gate not implemented for {language}"}
|
||||
cmd = [tools["compile"] or "python3", "-m", "py_compile", str(src)]
|
||||
elif lang == "rust":
|
||||
src = td_path / "generated.rs"
|
||||
src.write_text(code)
|
||||
cmd = [tools["compile"] or "rustc", "--crate-type", "lib", str(src), "-o", str(td_path / "libgenerated.rlib")]
|
||||
else: # go
|
||||
src = td_path / "generated.go"
|
||||
src.write_text(code)
|
||||
cmd = [tools["compile"] or "go", "build", str(src)]
|
||||
|
||||
try:
|
||||
p = subprocess.run(
|
||||
cmd,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
timeout=20,
|
||||
check=False,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
return {"passed": False, "skipped": True, "reason": f"compiler missing for {language}"}
|
||||
except subprocess.TimeoutExpired:
|
||||
return {"passed": False, "skipped": False, "reason": "compile timeout"}
|
||||
res = run_cmd(cmd, timeout_s=20)
|
||||
payload = {
|
||||
"passed": res.passed,
|
||||
"skipped": res.skipped,
|
||||
"reason": res.reason,
|
||||
"return_code": res.return_code,
|
||||
"command": res.command,
|
||||
"stdout": res.stdout,
|
||||
"stderr": res.stderr,
|
||||
"strict_blocking": bool(strict and (res.skipped or not res.passed)),
|
||||
}
|
||||
return payload
|
||||
|
||||
|
||||
def test_check(code: str, language: str, strict: bool) -> Dict[str, object]:
|
||||
lang = language.lower()
|
||||
queue_spec = ("priorityqueue" in code.lower())
|
||||
if not queue_spec:
|
||||
return {
|
||||
"passed": p.returncode == 0,
|
||||
"passed": True,
|
||||
"skipped": True,
|
||||
"reason": "queue_harness_not_required",
|
||||
"strict_blocking": False,
|
||||
}
|
||||
|
||||
required = ["enqueue", "dequeue", "peek", "size", "empty", "workitem"]
|
||||
text = code.lower()
|
||||
missing = [r for r in required if r not in text]
|
||||
if missing:
|
||||
return {
|
||||
"passed": False,
|
||||
"skipped": False,
|
||||
"return_code": p.returncode,
|
||||
"stdout": p.stdout[-4000:],
|
||||
"stderr": p.stderr[-4000:],
|
||||
"reason": "missing_required_queue_members",
|
||||
"missing": missing,
|
||||
"strict_blocking": bool(strict),
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="whetstone_gate_test_") as td:
|
||||
td_path = Path(td)
|
||||
if lang in ("cpp", "c++"):
|
||||
src = td_path / "harness.cpp"
|
||||
src.write_text(cpp_harness(code))
|
||||
cxx = shutil.which("g++") or shutil.which("clang++")
|
||||
cmd = [cxx or "g++", "-std=c++20", "-fsyntax-only", str(src)]
|
||||
elif lang == "python":
|
||||
src = td_path / "harness.py"
|
||||
src.write_text(python_harness(code))
|
||||
py = shutil.which("python3") or shutil.which("python")
|
||||
cmd = [py or "python3", str(src)]
|
||||
elif lang == "rust":
|
||||
src = td_path / "harness.rs"
|
||||
src.write_text(rust_harness(code))
|
||||
rustc = shutil.which("rustc")
|
||||
cmd = [rustc or "rustc", str(src), "-o", str(td_path / "harness")]
|
||||
elif lang == "go":
|
||||
src = td_path / "harness.go"
|
||||
src.write_text(go_harness(code))
|
||||
go = shutil.which("go")
|
||||
cmd = [go or "go", "build", str(src)]
|
||||
else:
|
||||
return {
|
||||
"passed": False,
|
||||
"skipped": True,
|
||||
"reason": "unsupported_language",
|
||||
"strict_blocking": bool(strict),
|
||||
}
|
||||
|
||||
res = run_cmd(cmd, timeout_s=25)
|
||||
return {
|
||||
"passed": res.passed,
|
||||
"skipped": res.skipped,
|
||||
"reason": res.reason,
|
||||
"return_code": res.return_code,
|
||||
"command": res.command,
|
||||
"stdout": res.stdout,
|
||||
"stderr": res.stderr,
|
||||
"strict_blocking": bool(strict and (res.skipped or not res.passed)),
|
||||
}
|
||||
|
||||
|
||||
def test_check(code: str, language: str) -> Dict[str, object]:
|
||||
text = code.lower()
|
||||
if "priorityqueue" in text:
|
||||
required = ["enqueue", "dequeue", "peek", "size", "empty"]
|
||||
missing = [x for x in required if x not in text]
|
||||
return {"passed": not missing, "missing": missing}
|
||||
return {"passed": True, "missing": []}
|
||||
def normalize_diagnostics(compile_res: Dict[str, object], test_res: Dict[str, object]) -> List[Dict[str, object]]:
|
||||
patterns: List[Tuple[str, re.Pattern[str]]] = [
|
||||
("gcc_clang", re.compile(r"^(?P<file>[^:\n]+):(\d+):(\d+):\s*(?P<sev>warning|error):\s*(?P<msg>.+)$", re.MULTILINE)),
|
||||
("rustc", re.compile(r"^(?P<sev>error|warning)(\[[^\]]+\])?:\s*(?P<msg>.+)$", re.MULTILINE)),
|
||||
("python", re.compile(r"File \"(?P<file>[^\"]+)\", line (?P<line>\d+).*(?P<msg>SyntaxError:.*)$", re.MULTILINE)),
|
||||
]
|
||||
|
||||
out: List[Dict[str, object]] = []
|
||||
|
||||
def scan(source: str, category: str, text: str) -> None:
|
||||
seen = set()
|
||||
for family, pat in patterns:
|
||||
for m in pat.finditer(text or ""):
|
||||
file_name = m.groupdict().get("file", "")
|
||||
sev = m.groupdict().get("sev", "error")
|
||||
msg = m.groupdict().get("msg", "parse failure")
|
||||
line = int(m.group(2)) if m.lastindex and m.lastindex >= 2 and m.group(2) and m.group(2).isdigit() else None
|
||||
col = int(m.group(3)) if m.lastindex and m.lastindex >= 3 and m.group(3) and m.group(3).isdigit() else None
|
||||
key = (source, category, file_name, line, col, msg)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
out.append(
|
||||
{
|
||||
"source": source,
|
||||
"family": family,
|
||||
"severity": "warning" if sev == "warning" else "error",
|
||||
"category": category,
|
||||
"file": file_name,
|
||||
"line": line,
|
||||
"column": col,
|
||||
"message": msg.strip() or "diagnostic",
|
||||
"raw_excerpt": truncate(m.group(0), 300),
|
||||
}
|
||||
)
|
||||
|
||||
scan("compile", "compile", str(compile_res.get("stderr", "")))
|
||||
scan("tests", "tests", str(test_res.get("stderr", "")))
|
||||
if not out and (not compile_res.get("passed", True) or not test_res.get("passed", True)):
|
||||
out.append(
|
||||
{
|
||||
"source": "gate",
|
||||
"family": "fallback",
|
||||
"severity": "error",
|
||||
"category": "gates",
|
||||
"file": "",
|
||||
"line": None,
|
||||
"column": None,
|
||||
"message": "Gate failed without parseable diagnostic output",
|
||||
"raw_excerpt": "",
|
||||
}
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def main() -> None:
|
||||
@@ -84,6 +333,7 @@ def main() -> None:
|
||||
ap.add_argument("--code-file", help="Path to generated code file", default="")
|
||||
ap.add_argument("--code", help="Inline generated code", default="")
|
||||
ap.add_argument("--language", required=True)
|
||||
ap.add_argument("--strict", action="store_true", help="Strict mode: missing tools/skips are blocking")
|
||||
ap.add_argument("--out", default="")
|
||||
args = ap.parse_args()
|
||||
|
||||
@@ -92,16 +342,35 @@ def main() -> None:
|
||||
code = Path(args.code_file).read_text()
|
||||
|
||||
placeholders = detect_placeholders(code)
|
||||
compile_res = compile_check(code, args.language)
|
||||
test_res = test_check(code, args.language)
|
||||
compile_res = compile_check(code, args.language, strict=args.strict)
|
||||
test_res = test_check(code, args.language, strict=args.strict)
|
||||
diagnostics = normalize_diagnostics(compile_res, test_res)
|
||||
|
||||
compile_pass = bool(compile_res.get("passed", False))
|
||||
test_pass = bool(test_res.get("passed", False))
|
||||
if args.strict:
|
||||
compile_pass = compile_pass and not bool(compile_res.get("strict_blocking", False))
|
||||
test_pass = test_pass and not bool(test_res.get("strict_blocking", False))
|
||||
|
||||
overall = bool(placeholders["passed"] and compile_pass and test_pass)
|
||||
|
||||
failure_reasons: List[str] = []
|
||||
if not placeholders["passed"]:
|
||||
failure_reasons.append("placeholder_or_todo_detected")
|
||||
if not compile_pass:
|
||||
failure_reasons.append(f"compile_failed:{compile_res.get('reason', 'unknown')}")
|
||||
if not test_pass:
|
||||
failure_reasons.append(f"tests_failed:{test_res.get('reason', 'unknown')}")
|
||||
|
||||
overall = bool(placeholders["passed"] and compile_res.get("passed", False) and test_res.get("passed", False))
|
||||
payload = {
|
||||
"language": args.language,
|
||||
"strict_mode": bool(args.strict),
|
||||
"diagnostics": diagnostics,
|
||||
"gates": {
|
||||
"compile": compile_res,
|
||||
"tests": test_res,
|
||||
"placeholder": placeholders,
|
||||
"failure_reasons": failure_reasons,
|
||||
"overall_ready": overall,
|
||||
},
|
||||
}
|
||||
@@ -116,4 +385,3 @@ def main() -> None:
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
|
||||
118
tools/mcp/remediation_router.py
Executable file
118
tools/mcp/remediation_router.py
Executable file
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Deterministic remediation routing from gate diagnostics to action plan."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Dict, List
|
||||
|
||||
|
||||
def load_json(path: str) -> Dict[str, object]:
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
return {}
|
||||
return json.loads(p.read_text())
|
||||
|
||||
|
||||
def classify(gates: Dict[str, object]) -> Dict[str, object]:
|
||||
g = gates.get("gates", {}) if isinstance(gates, dict) else {}
|
||||
reasons = list(g.get("failure_reasons", [])) if isinstance(g, dict) else []
|
||||
compile_reason = str((g.get("compile") or {}).get("reason", ""))
|
||||
tests_reason = str((g.get("tests") or {}).get("reason", ""))
|
||||
|
||||
actions: List[Dict[str, object]] = []
|
||||
|
||||
nonrecoverable = "tool_missing" in compile_reason or "tool_missing" in tests_reason
|
||||
if nonrecoverable:
|
||||
actions.append(
|
||||
{
|
||||
"action": "stop_blocked",
|
||||
"tool": "none",
|
||||
"reason": "missing_toolchain",
|
||||
"confidence": 1.0,
|
||||
"priority": 100,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"blocked": True,
|
||||
"blocked_reason": "missing_toolchain",
|
||||
"actions": actions,
|
||||
}
|
||||
|
||||
if any(r.startswith("placeholder_or_todo") for r in reasons):
|
||||
actions.append(
|
||||
{
|
||||
"action": "strengthen_spec",
|
||||
"tool": "whetstone_generate_code",
|
||||
"hint": "Eliminate placeholders/TODO markers and provide concrete field/method types and bodies.",
|
||||
"confidence": 0.95,
|
||||
"priority": 90,
|
||||
}
|
||||
)
|
||||
|
||||
if any(r.startswith("compile_failed") for r in reasons):
|
||||
actions.append(
|
||||
{
|
||||
"action": "diagnose_pipeline",
|
||||
"tool": "whetstone_run_pipeline",
|
||||
"hint": "Run same-language pipeline pass to collect parse/validation diagnostics and refine signatures/types.",
|
||||
"confidence": 0.9,
|
||||
"priority": 80,
|
||||
}
|
||||
)
|
||||
actions.append(
|
||||
{
|
||||
"action": "strengthen_spec",
|
||||
"tool": "whetstone_generate_code",
|
||||
"hint": "Emit compile-clean code for target language with explicit imports/includes and fully typed declarations.",
|
||||
"confidence": 0.85,
|
||||
"priority": 70,
|
||||
}
|
||||
)
|
||||
|
||||
if any(r.startswith("tests_failed") for r in reasons):
|
||||
actions.append(
|
||||
{
|
||||
"action": "strengthen_spec",
|
||||
"tool": "whetstone_generate_code",
|
||||
"hint": "Implement queue behaviors: enqueue/dequeue/peek/size/empty with deterministic semantics.",
|
||||
"confidence": 0.9,
|
||||
"priority": 75,
|
||||
}
|
||||
)
|
||||
|
||||
actions.sort(key=lambda a: (-int(a["priority"]), a["action"]))
|
||||
return {
|
||||
"blocked": False,
|
||||
"blocked_reason": "",
|
||||
"actions": actions,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--gates-json", required=True)
|
||||
ap.add_argument("--out", default="")
|
||||
args = ap.parse_args()
|
||||
|
||||
gates = load_json(args.gates_json)
|
||||
plan = classify(gates)
|
||||
payload = {
|
||||
"router_version": "1",
|
||||
"input_failure_reasons": (gates.get("gates", {}) or {}).get("failure_reasons", []),
|
||||
"blocked": plan["blocked"],
|
||||
"blocked_reason": plan["blocked_reason"],
|
||||
"actions": plan["actions"],
|
||||
}
|
||||
text = json.dumps(payload, indent=2, sort_keys=True)
|
||||
if args.out:
|
||||
out = Path(args.out)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(text + "\n")
|
||||
print(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
69
tools/mcp/run_production_benchmark_suite.sh
Executable file
69
tools/mcp/run_production_benchmark_suite.sh
Executable file
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
OUT_DIR="${OUT_DIR:-$ROOT_DIR/logs/taskitem_runs/production_benchmark_$(date +%Y%m%d_%H%M%S)}"
|
||||
RUNS="${RUNS:-3}"
|
||||
STRICT_MODE="${STRICT_MODE:-1}"
|
||||
LANGUAGE="${WSTONE_LANGUAGE:-cpp}"
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
spec_a="Generate WorkItem and PriorityQueue classes with enqueue, dequeue, peek, size, empty"
|
||||
spec_b="Generate DataStore class with put/get/delete/contains methods and concrete typed fields"
|
||||
|
||||
run_case() {
|
||||
local name="$1"
|
||||
local spec="$2"
|
||||
local pass=0
|
||||
local blocked=0
|
||||
local total_ms=0
|
||||
|
||||
for i in $(seq 1 "$RUNS"); do
|
||||
local run_dir="$OUT_DIR/${name}_run_${i}"
|
||||
mkdir -p "$run_dir"
|
||||
local start_ms
|
||||
start_ms=$(date +%s%3N)
|
||||
OUT_DIR="$run_dir" STRICT_MODE="$STRICT_MODE" WSTONE_LANGUAGE="$LANGUAGE" \
|
||||
"$ROOT_DIR/tools/mcp/run_production_completion_loop.sh" "$spec" > "$run_dir/stdout.log"
|
||||
local end_ms
|
||||
end_ms=$(date +%s%3N)
|
||||
local dur=$((end_ms - start_ms))
|
||||
total_ms=$((total_ms + dur))
|
||||
|
||||
local status
|
||||
status=$(jq -r '.status' "$run_dir/00_summary.json")
|
||||
if [[ "$status" == "green" ]]; then
|
||||
pass=$((pass + 1))
|
||||
else
|
||||
blocked=$((blocked + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
jq -nc \
|
||||
--arg name "$name" \
|
||||
--arg spec "$spec" \
|
||||
--argjson runs "$RUNS" \
|
||||
--argjson pass "$pass" \
|
||||
--argjson blocked "$blocked" \
|
||||
--argjson avg_ms "$((total_ms / RUNS))" \
|
||||
'{name:$name,spec:$spec,runs:$runs,pass:$pass,blocked:$blocked,pass_rate:($pass/$runs),avg_latency_ms:$avg_ms}'
|
||||
}
|
||||
|
||||
case_a=$(run_case "priorityqueue" "$spec_a")
|
||||
case_b=$(run_case "datastore" "$spec_b")
|
||||
|
||||
jq -nc \
|
||||
--arg out_dir "$OUT_DIR" \
|
||||
--arg language "$LANGUAGE" \
|
||||
--argjson strict_mode "$STRICT_MODE" \
|
||||
--argjson a "$case_a" \
|
||||
--argjson b "$case_b" \
|
||||
'{
|
||||
out_dir:$out_dir,
|
||||
language:$language,
|
||||
strict_mode:($strict_mode==1),
|
||||
cases:[$a,$b]
|
||||
}' > "$OUT_DIR/00_benchmark_summary.json"
|
||||
|
||||
echo "Benchmark suite output: $OUT_DIR"
|
||||
cat "$OUT_DIR/00_benchmark_summary.json"
|
||||
@@ -6,6 +6,7 @@ BIN="${WSTONE_MCP_BIN:-$ROOT_DIR/editor/build-native/whetstone_mcp_stable}"
|
||||
WORKSPACE="${WSTONE_WORKSPACE:-$ROOT_DIR}"
|
||||
LANGUAGE="${WSTONE_LANGUAGE:-cpp}"
|
||||
MAX_ITERS="${MAX_ITERS:-3}"
|
||||
STRICT_MODE="${STRICT_MODE:-1}"
|
||||
SPEC="${1:-}"
|
||||
|
||||
if [[ -z "$SPEC" ]]; then
|
||||
@@ -20,6 +21,7 @@ fi
|
||||
|
||||
OUT_DIR="${OUT_DIR:-$ROOT_DIR/logs/taskitem_runs/production_loop_$(date +%Y%m%d_%H%M%S)}"
|
||||
mkdir -p "$OUT_DIR"
|
||||
TRACE_FILE="$OUT_DIR/trace.jsonl"
|
||||
|
||||
call_tool() {
|
||||
local tool_name="$1"
|
||||
@@ -37,54 +39,115 @@ extract_json() {
|
||||
printf '%s' "$raw" | jq -r '.result.content[0].text // "{}"' | jq '.'
|
||||
}
|
||||
|
||||
log_trace() {
|
||||
local phase="$1"
|
||||
local iter="$2"
|
||||
local payload="$3"
|
||||
jq -nc --arg phase "$phase" --arg iter "$iter" --argjson payload "$payload" \
|
||||
'{phase:$phase, iteration:$iter, payload:$payload}' >> "$TRACE_FILE"
|
||||
}
|
||||
|
||||
loop_spec="$SPEC"
|
||||
status="blocked"
|
||||
blocked_reason="max_iterations_exhausted"
|
||||
final_ready=0
|
||||
|
||||
for i in $(seq 1 "$MAX_ITERS"); do
|
||||
iter_id="iter_$(printf '%02d' "$i")"
|
||||
|
||||
args="$(jq -nc --arg s "$loop_spec" '{spec:$s,preferImports:true}')"
|
||||
raw="$(call_tool whetstone_generate_code "$args")"
|
||||
printf '%s\n' "$raw" > "$OUT_DIR/iter_${i}_generate_raw.json"
|
||||
printf '%s\n' "$raw" > "$OUT_DIR/${iter_id}_generate_raw.json"
|
||||
parsed="$(extract_json "$raw")"
|
||||
printf '%s\n' "$parsed" > "$OUT_DIR/iter_${i}_generate.json"
|
||||
printf '%s\n' "$parsed" > "$OUT_DIR/${iter_id}_generate.json"
|
||||
log_trace "generate" "$iter_id" "$parsed"
|
||||
|
||||
code="$(printf '%s' "$parsed" | jq -r '.generatedCode // ""')"
|
||||
if [[ -z "$code" ]]; then
|
||||
code="$(printf '%s' "$parsed" | jq -r '.note // ""')"
|
||||
fi
|
||||
printf '%s\n' "$code" > "$OUT_DIR/iter_${i}_generated_code.txt"
|
||||
printf '%s\n' "$code" > "$OUT_DIR/${iter_id}_generated_code.txt"
|
||||
|
||||
strict_arg=()
|
||||
if [[ "$STRICT_MODE" == "1" ]]; then
|
||||
strict_arg+=(--strict)
|
||||
fi
|
||||
|
||||
python3 "$ROOT_DIR/tools/mcp/evaluate_generated_code_gates.py" \
|
||||
--code-file "$OUT_DIR/iter_${i}_generated_code.txt" \
|
||||
--code-file "$OUT_DIR/${iter_id}_generated_code.txt" \
|
||||
--language "$LANGUAGE" \
|
||||
--out "$OUT_DIR/iter_${i}_gates.json" >/tmp/production_loop_gate_${i}.json
|
||||
"${strict_arg[@]}" \
|
||||
--out "$OUT_DIR/${iter_id}_gates.json" >/tmp/production_loop_gate_${i}.json
|
||||
|
||||
ready="$(jq -r '.gates.overall_ready' "$OUT_DIR/iter_${i}_gates.json")"
|
||||
gates_payload="$(cat "$OUT_DIR/${iter_id}_gates.json")"
|
||||
log_trace "gates" "$iter_id" "$gates_payload"
|
||||
|
||||
ready="$(jq -r '.gates.overall_ready' "$OUT_DIR/${iter_id}_gates.json")"
|
||||
if [[ "$ready" == "true" ]]; then
|
||||
status="green"
|
||||
blocked_reason=""
|
||||
final_ready=1
|
||||
break
|
||||
fi
|
||||
|
||||
# Simple remediation strategy: force class/module details + no placeholders.
|
||||
# Run same-language pipeline for diagnostics when compile/test fail.
|
||||
pipeline_args="$(jq -nc --arg src "$code" --arg lang "$LANGUAGE" '{source:$src,sourceLanguage:$lang,targetLanguage:$lang}')"
|
||||
pipeline_raw="$(call_tool whetstone_run_pipeline "$pipeline_args")"
|
||||
printf '%s\n' "$pipeline_raw" > "$OUT_DIR/${iter_id}_pipeline_raw.json"
|
||||
pipeline_parsed="$(extract_json "$pipeline_raw")"
|
||||
printf '%s\n' "$pipeline_parsed" > "$OUT_DIR/${iter_id}_pipeline.json"
|
||||
log_trace "pipeline_diagnostics" "$iter_id" "$pipeline_parsed"
|
||||
|
||||
python3 "$ROOT_DIR/tools/mcp/remediation_router.py" \
|
||||
--gates-json "$OUT_DIR/${iter_id}_gates.json" \
|
||||
--out "$OUT_DIR/${iter_id}_route.json" >/tmp/production_loop_route_${i}.json
|
||||
|
||||
route_payload="$(cat "$OUT_DIR/${iter_id}_route.json")"
|
||||
log_trace "route" "$iter_id" "$route_payload"
|
||||
|
||||
blocked="$(jq -r '.blocked' "$OUT_DIR/${iter_id}_route.json")"
|
||||
if [[ "$blocked" == "true" ]]; then
|
||||
status="blocked"
|
||||
blocked_reason="$(jq -r '.blocked_reason' "$OUT_DIR/${iter_id}_route.json")"
|
||||
break
|
||||
fi
|
||||
|
||||
hints="$(jq -r '[.actions[].hint] | unique | join("\n")' "$OUT_DIR/${iter_id}_route.json")"
|
||||
loop_spec="$loop_spec
|
||||
Return production-ready code only:
|
||||
|
||||
Remediation directives (iteration $i):
|
||||
$hints
|
||||
|
||||
Requirements:
|
||||
- no TODO/FIXME/placeholder markers
|
||||
- concrete types for fields and methods
|
||||
- class bodies for queue operations
|
||||
- compile-clean output."
|
||||
- concrete field and method types
|
||||
- compile-clean output for target language
|
||||
- queue operations must be behaviorally complete: enqueue/dequeue/peek/size/empty"
|
||||
done
|
||||
|
||||
if [[ "$status" == "green" ]]; then
|
||||
blocked_reason=""
|
||||
fi
|
||||
|
||||
jq -nc \
|
||||
--arg out_dir "$OUT_DIR" \
|
||||
--arg spec "$SPEC" \
|
||||
--arg language "$LANGUAGE" \
|
||||
--arg status "$status" \
|
||||
--arg blocked_reason "$blocked_reason" \
|
||||
--argjson strict_mode "$STRICT_MODE" \
|
||||
--argjson max_iters "$MAX_ITERS" \
|
||||
--argjson overall_ready "$final_ready" \
|
||||
'{
|
||||
out_dir:$out_dir,
|
||||
spec:$spec,
|
||||
language:$language,
|
||||
strict_mode:($strict_mode==1),
|
||||
max_iters:$max_iters,
|
||||
overall_ready:($overall_ready==1)
|
||||
overall_ready:($overall_ready==1),
|
||||
status:$status,
|
||||
blocked_reason:$blocked_reason
|
||||
}' > "$OUT_DIR/00_summary.json"
|
||||
|
||||
echo "Production completion loop output: $OUT_DIR"
|
||||
cat "$OUT_DIR/00_summary.json"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user