133 lines
4.3 KiB
Bash
Executable File
133 lines
4.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# sweep_all_gates.sh
|
|
#
|
|
# Run capacity sweeps for all 4 specialists + prereq_op factorizability test.
|
|
# Runs sweeps sequentially (one GPU job at a time).
|
|
#
|
|
# Usage:
|
|
# bash specialists/scripts/sweep_all_gates.sh [--pilot-only] [--gates worker_type,prereq_op]
|
|
#
|
|
# Defaults: full sweep for all gates.
|
|
#
|
|
# Output:
|
|
# specialists/eval/results/<gate>_sweep.json — per-gate sweep table
|
|
# specialists/eval/results/prereq_op_factorize.json — factorize comparison
|
|
# specialists/eval/results/summary.json — all gates combined
|
|
|
|
set -euo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
PYTHON="/home/bill/Documents/CLionProjects/whetstone_DSL/.venv/bin/python3"
|
|
RUNS_DIR="/mnt/storage/fabricate_runs"
|
|
RESULTS_DIR="$ROOT/specialists/eval/results"
|
|
mkdir -p "$RESULTS_DIR"
|
|
|
|
PILOT_ONLY=""
|
|
GATES="verification_type,worker_type,prereq_op,automatability"
|
|
TIERS="small_plus,medium"
|
|
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--pilot-only) PILOT_ONLY="--pilot-only" ;;
|
|
--gates=*) GATES="${arg#--gates=}" ;;
|
|
--tiers=*) TIERS="${arg#--tiers=}" ;;
|
|
esac
|
|
done
|
|
|
|
echo "=== Whetstone Specialist Capacity Sweep ==="
|
|
echo " Gates: $GATES"
|
|
echo " Mode: ${PILOT_ONLY:-(full sweep)}"
|
|
echo " Results: $RESULTS_DIR"
|
|
echo ""
|
|
|
|
run_sweep() {
|
|
local name="$1"; local train="$2"; local eval="$3"; local labels="$4"
|
|
if echo "$GATES" | grep -qw "$name"; then
|
|
echo ""
|
|
echo ">>> Gate: $name"
|
|
"$PYTHON" "$ROOT/specialists/scripts/capacity_sweep.py" \
|
|
--name "$name" \
|
|
--train "$train" \
|
|
--eval "$eval" \
|
|
--labels "$labels" \
|
|
--runs-dir "$RUNS_DIR" \
|
|
--results-out "$RESULTS_DIR/${name}_sweep.json" \
|
|
--tiers "$TIERS" \
|
|
$PILOT_ONLY \
|
|
2>&1 | tee "$RESULTS_DIR/${name}_sweep.log"
|
|
echo ""
|
|
fi
|
|
}
|
|
|
|
run_sweep "verification_type" \
|
|
"$ROOT/specialists/data/generated/verification_type_train.tsv" \
|
|
"$ROOT/specialists/data/generated/verification_type_eval.tsv" \
|
|
"unit,integration,schema,smoke,docs"
|
|
|
|
run_sweep "worker_type" \
|
|
"$ROOT/specialists/data/generated/worker_type_train.tsv" \
|
|
"$ROOT/specialists/data/generated/worker_type_eval.tsv" \
|
|
"implementer,reviewer,architect,qa"
|
|
|
|
run_sweep "prereq_op" \
|
|
"$ROOT/specialists/data/combined/prereq_op_train.tsv" \
|
|
"$ROOT/specialists/data/combined/prereq_op_eval.tsv" \
|
|
"standard,needs_review,needs_approval,full_gates"
|
|
|
|
run_sweep "automatability" \
|
|
"$ROOT/specialists/data/generated/automatability_train.tsv" \
|
|
"$ROOT/specialists/data/generated/automatability_eval.tsv" \
|
|
"deterministic,template,specialist,slm,llm,human"
|
|
|
|
# prereq_op factorizability test (runs alongside the sweep)
|
|
if echo "$GATES" | grep -qw "prereq_op"; then
|
|
echo ""
|
|
echo ">>> prereq_op: factorizability test (two binary classifiers)"
|
|
"$PYTHON" "$ROOT/specialists/scripts/prereq_op_binary_split.py" \
|
|
--data-dir "$ROOT/specialists/data/combined" \
|
|
--runs-dir "$RUNS_DIR" \
|
|
--results-out "$RESULTS_DIR/prereq_op_factorize.json" \
|
|
2>&1 | tee "$RESULTS_DIR/prereq_op_factorize.log"
|
|
fi
|
|
|
|
# --- Combine into summary ---
|
|
"$PYTHON" - <<'PYEOF'
|
|
import json, glob, sys
|
|
from pathlib import Path
|
|
|
|
results_dir = Path("specialists/eval/results")
|
|
summary = {"gates": {}}
|
|
|
|
for path in sorted(results_dir.glob("*_sweep.json")):
|
|
gate = path.stem.replace("_sweep", "")
|
|
data = json.loads(path.read_text())
|
|
best_tier = max(
|
|
(t for t in data["tiers"] if t.get("eval_accuracy")),
|
|
key=lambda t: t.get("eval_accuracy", 0),
|
|
default=None,
|
|
)
|
|
summary["gates"][gate] = {
|
|
"best_tier": best_tier["tier"] if best_tier else None,
|
|
"best_accuracy": best_tier["eval_accuracy"] if best_tier else None,
|
|
"verdict": best_tier.get("verdict") if best_tier else None,
|
|
"guardrail_threshold": best_tier.get("guardrail_threshold") if best_tier else None,
|
|
}
|
|
|
|
# Add factorize comparison if available
|
|
factorize_path = results_dir / "prereq_op_factorize.json"
|
|
if factorize_path.exists():
|
|
data = json.loads(factorize_path.read_text())
|
|
summary["gates"]["prereq_op"]["factorized_accuracy"] = (
|
|
data.get("factorized", {}).get("combined_4way", {}).get("overall_accuracy")
|
|
)
|
|
|
|
out = results_dir / "summary.json"
|
|
out.write_text(json.dumps(summary, indent=2))
|
|
print(f"\nSummary → {out}")
|
|
print(json.dumps(summary, indent=2))
|
|
PYEOF
|
|
|
|
echo ""
|
|
echo "=== Sweep complete ==="
|
|
echo " Full results in: $RESULTS_DIR"
|