296 lines
12 KiB
Bash
296 lines
12 KiB
Bash
|
#!/usr/bin/env bash
|
||
|
|
set -euo pipefail
|
||
|
|
|
||
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||
|
|
CATALOG="${1:-$ROOT_DIR/datasets/project_benchmarks/common_projects_100.jsonl}"
|
||
|
|
OUT_DIR="${OUT_DIR:-$ROOT_DIR/logs/taskitem_runs/project_benchmark_matrix_$(date +%Y%m%d_%H%M%S)}"
|
||
|
|
LANG_MATRIX="${LANG_MATRIX:-cpp,python,go,rust}"
|
||
|
|
RUN_AB="${RUN_AB:-1}"
|
||
|
|
RUN_PROD="${RUN_PROD:-0}"
|
||
|
|
STRICT_MODE="${STRICT_MODE:-1}"
|
||
|
|
LIMIT="${LIMIT:-0}"
|
||
|
|
REQUIRE_AB_PARITY="${REQUIRE_AB_PARITY:-1}"
|
||
|
|
|
||
|
|
if [[ ! -f "$CATALOG" ]]; then
|
||
|
|
echo "error: catalog not found: $CATALOG" >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
if ! command -v jq >/dev/null 2>&1; then
|
||
|
|
echo "error: jq required" >&2
|
||
|
|
exit 1
|
||
|
|
fi
|
||
|
|
|
||
|
|
mkdir -p "$OUT_DIR"
|
||
|
|
RESULTS_JSONL="$OUT_DIR/results.jsonl"
|
||
|
|
touch "$RESULTS_JSONL"
|
||
|
|
|
||
|
|
IFS=',' read -r -a LANGS <<< "$LANG_MATRIX"
|
||
|
|
total_specs=$(wc -l < "$CATALOG" | tr -d ' ')
|
||
|
|
if [[ "$LIMIT" -gt 0 && "$LIMIT" -lt "$total_specs" ]]; then
|
||
|
|
total_specs="$LIMIT"
|
||
|
|
fi
|
||
|
|
|
||
|
|
spec_idx=0
|
||
|
|
while IFS= read -r row; do
|
||
|
|
spec_idx=$((spec_idx + 1))
|
||
|
|
if [[ "$LIMIT" -gt 0 && "$spec_idx" -gt "$LIMIT" ]]; then
|
||
|
|
break
|
||
|
|
fi
|
||
|
|
|
||
|
|
id="$(printf '%s' "$row" | jq -r '.id')"
|
||
|
|
project="$(printf '%s' "$row" | jq -r '.project')"
|
||
|
|
category="$(printf '%s' "$row" | jq -r '.category')"
|
||
|
|
language_declared="$(printf '%s' "$row" | jq -r '.language')"
|
||
|
|
spec="$(printf '%s' "$row" | jq -r '.spec')"
|
||
|
|
core_semantics="$(printf '%s' "$row" | jq -c '.core_semantics // {}')"
|
||
|
|
projection_targets="$(printf '%s' "$row" | jq -c '.projection_targets // []')"
|
||
|
|
cross_target_invariants="$(printf '%s' "$row" | jq -c '.cross_target_invariants // []')"
|
||
|
|
degradation_policy="$(printf '%s' "$row" | jq -c '.degradation_policy // {}')"
|
||
|
|
constraints="$(printf '%s' "$row" | jq -c '.constraints // {}')"
|
||
|
|
|
||
|
|
projection_contract_ok="true"
|
||
|
|
projection_contract_reason=""
|
||
|
|
if [[ "$category" == "projection_constraints" ]]; then
|
||
|
|
target_count="$(printf '%s' "$projection_targets" | jq 'length')"
|
||
|
|
invariants_count="$(printf '%s' "$core_semantics" | jq '.invariants // [] | length')"
|
||
|
|
if [[ "$target_count" -lt 1 ]]; then
|
||
|
|
projection_contract_ok="false"
|
||
|
|
projection_contract_reason="projection_targets_missing"
|
||
|
|
elif [[ "$invariants_count" -lt 1 ]]; then
|
||
|
|
projection_contract_ok="false"
|
||
|
|
projection_contract_reason="core_semantics_invariants_missing"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
fullstack_contract_ok="true"
|
||
|
|
fullstack_contract_reason=""
|
||
|
|
required_artifact_count="$(printf '%s' "$constraints" | jq '.required_artifacts // [] | length')"
|
||
|
|
require_multifile="$(printf '%s' "$constraints" | jq '.require_multifile // false')"
|
||
|
|
if [[ "$require_multifile" == "true" && "$required_artifact_count" -lt 3 ]]; then
|
||
|
|
fullstack_contract_ok="false"
|
||
|
|
fullstack_contract_reason="required_artifacts_insufficient_for_multifile"
|
||
|
|
fi
|
||
|
|
if [[ "$category" == "fullstack_api_evolution" ]]; then
|
||
|
|
compat_days="$(printf '%s' "$constraints" | jq '.compatibility.backward_compatible_window_days // 0')"
|
||
|
|
if [[ "$compat_days" -le 0 ]]; then
|
||
|
|
fullstack_contract_ok="false"
|
||
|
|
fullstack_contract_reason="compatibility_window_missing"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
if [[ "$category" == "state_migration" ]]; then
|
||
|
|
rollback_required="$(printf '%s' "$constraints" | jq '.migration.rollback_required // false')"
|
||
|
|
if [[ "$rollback_required" != "true" ]]; then
|
||
|
|
fullstack_contract_ok="false"
|
||
|
|
fullstack_contract_reason="migration_rollback_requirement_missing"
|
||
|
|
fi
|
||
|
|
data_loss_allowed="$(printf '%s' "$constraints" | jq -r 'if (.migration | type) == "object" and (.migration | has("data_loss_allowed")) then .migration.data_loss_allowed else true end')"
|
||
|
|
if [[ "$data_loss_allowed" != "false" ]]; then
|
||
|
|
fullstack_contract_ok="false"
|
||
|
|
fullstack_contract_reason="migration_data_loss_policy_invalid"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
if [[ "$category" == "security_propagation" ]]; then
|
||
|
|
deny_default="$(printf '%s' "$constraints" | jq '.security.deny_by_default // false')"
|
||
|
|
if [[ "$deny_default" != "true" ]]; then
|
||
|
|
fullstack_contract_ok="false"
|
||
|
|
fullstack_contract_reason="security_deny_by_default_missing"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
if [[ "$category" == "performance_budgeted_change" ]]; then
|
||
|
|
p95="$(printf '%s' "$constraints" | jq '.slo.p95_latency_ms // 0')"
|
||
|
|
if [[ "$p95" -le 0 ]]; then
|
||
|
|
fullstack_contract_ok="false"
|
||
|
|
fullstack_contract_reason="slo_p95_latency_missing"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
if [[ "$category" == "rollout_choreography" ]]; then
|
||
|
|
rollout_staged="$(printf '%s' "$constraints" | jq '.rollout.staged // false')"
|
||
|
|
rollout_abort_on_breach="$(printf '%s' "$constraints" | jq '.rollout.auto_abort_on_error_budget_breach // false')"
|
||
|
|
if [[ "$rollout_staged" != "true" ]]; then
|
||
|
|
fullstack_contract_ok="false"
|
||
|
|
fullstack_contract_reason="rollout_staged_policy_missing"
|
||
|
|
elif [[ "$rollout_abort_on_breach" != "true" ]]; then
|
||
|
|
fullstack_contract_ok="false"
|
||
|
|
fullstack_contract_reason="rollout_auto_abort_policy_missing"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
for lang in "${LANGS[@]}"; do
|
||
|
|
run_key="${id}_${lang}"
|
||
|
|
run_dir="$OUT_DIR/$run_key"
|
||
|
|
mkdir -p "$run_dir"
|
||
|
|
|
||
|
|
ab_status="skipped"
|
||
|
|
ab_ready_a="false"
|
||
|
|
ab_ready_b="false"
|
||
|
|
ab_fail_a="[]"
|
||
|
|
ab_fail_b="[]"
|
||
|
|
ab_tok_a=0
|
||
|
|
ab_tok_b=0
|
||
|
|
ab_out_dir=""
|
||
|
|
ab_err=""
|
||
|
|
|
||
|
|
if [[ "$RUN_AB" == "1" ]]; then
|
||
|
|
ab_out_dir="$run_dir/ab"
|
||
|
|
mkdir -p "$ab_out_dir"
|
||
|
|
if [[ "$STRICT_MODE" == "1" && "$projection_contract_ok" != "true" ]]; then
|
||
|
|
ab_status="failed"
|
||
|
|
ab_err="projection_contract_invalid:${projection_contract_reason}"
|
||
|
|
elif [[ "$STRICT_MODE" == "1" && "$fullstack_contract_ok" != "true" ]]; then
|
||
|
|
ab_status="failed"
|
||
|
|
ab_err="fullstack_contract_invalid:${fullstack_contract_reason}"
|
||
|
|
elif OUT_DIR="$ab_out_dir" WSTONE_LANGUAGE="$lang" STRICT_MODE="$STRICT_MODE" \
|
||
|
|
"$ROOT_DIR/tools/mcp/run_ab_test_ast_vs_language_first.sh" "$spec" > "$run_dir/ab_stdout.log" 2>"$run_dir/ab_stderr.log"; then
|
||
|
|
ab_status="ok"
|
||
|
|
else
|
||
|
|
ab_status="failed"
|
||
|
|
ab_err="$(tail -n 5 "$run_dir/ab_stderr.log" | tr '\n' ' ' | sed 's/"/\\"/g')"
|
||
|
|
fi
|
||
|
|
if [[ -f "$ab_out_dir/00_summary.json" ]]; then
|
||
|
|
ab_ready_a="$(jq -r '.path_a.gate_overall_ready // false' "$ab_out_dir/00_summary.json")"
|
||
|
|
ab_ready_b="$(jq -r '.path_b.gate_overall_ready // false' "$ab_out_dir/00_summary.json")"
|
||
|
|
ab_fail_a="$(jq -c '.path_a.failure_reasons // []' "$ab_out_dir/00_summary.json")"
|
||
|
|
ab_fail_b="$(jq -c '.path_b.failure_reasons // []' "$ab_out_dir/00_summary.json")"
|
||
|
|
ab_tok_a="$(jq -r '.path_a.token_accounting.total_tokens // 0' "$ab_out_dir/00_summary.json")"
|
||
|
|
ab_tok_b="$(jq -r '.path_b.token_accounting.total_tokens // 0' "$ab_out_dir/00_summary.json")"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
prod_status="skipped"
|
||
|
|
prod_ready="false"
|
||
|
|
prod_blocked_reason=""
|
||
|
|
prod_gate_evidence_complete="false"
|
||
|
|
prod_compile_pass="false"
|
||
|
|
prod_tests_pass="false"
|
||
|
|
prod_ab_divergence="false"
|
||
|
|
prod_ab_consistency_blocked="false"
|
||
|
|
prod_out_dir=""
|
||
|
|
prod_err=""
|
||
|
|
if [[ "$RUN_PROD" == "1" ]]; then
|
||
|
|
prod_out_dir="$run_dir/prod"
|
||
|
|
mkdir -p "$prod_out_dir"
|
||
|
|
if [[ "$STRICT_MODE" == "1" && "$projection_contract_ok" != "true" ]]; then
|
||
|
|
prod_status="failed"
|
||
|
|
prod_err="projection_contract_invalid:${projection_contract_reason}"
|
||
|
|
elif [[ "$STRICT_MODE" == "1" && "$fullstack_contract_ok" != "true" ]]; then
|
||
|
|
prod_status="failed"
|
||
|
|
prod_err="fullstack_contract_invalid:${fullstack_contract_reason}"
|
||
|
|
elif OUT_DIR="$prod_out_dir" WSTONE_LANGUAGE="$lang" STRICT_MODE="$STRICT_MODE" \
|
||
|
|
"$ROOT_DIR/tools/mcp/run_production_completion_loop.sh" "$spec" > "$run_dir/prod_stdout.log" 2>"$run_dir/prod_stderr.log"; then
|
||
|
|
prod_status="ok"
|
||
|
|
else
|
||
|
|
prod_status="failed"
|
||
|
|
prod_err="$(tail -n 5 "$run_dir/prod_stderr.log" | tr '\n' ' ' | sed 's/"/\\"/g')"
|
||
|
|
fi
|
||
|
|
if [[ -f "$prod_out_dir/00_summary.json" ]]; then
|
||
|
|
prod_ready="$(jq -r '.overall_ready // false' "$prod_out_dir/00_summary.json")"
|
||
|
|
prod_blocked_reason="$(jq -r '.blocked_reason // ""' "$prod_out_dir/00_summary.json" | sed 's/"/\\"/g')"
|
||
|
|
prod_gate_evidence_complete="$(jq -r '.gate_evidence_complete // false' "$prod_out_dir/00_summary.json")"
|
||
|
|
prod_compile_pass="$(jq -r '.gate_proofs.compile.passed // false' "$prod_out_dir/00_summary.json")"
|
||
|
|
prod_tests_pass="$(jq -r '.gate_proofs.tests.passed // false' "$prod_out_dir/00_summary.json")"
|
||
|
|
# Divergence flag: language-first AB failed compile/tests while production reports both compile+tests passed.
|
||
|
|
if [[ "$ab_ready_b" == "false" ]] &&
|
||
|
|
[[ "$ab_fail_b" == *"compile_failed:non_zero_exit"* || "$ab_fail_b" == *"tests_failed:non_zero_exit"* ]] &&
|
||
|
|
[[ "$prod_compile_pass" == "true" && "$prod_tests_pass" == "true" ]]; then
|
||
|
|
prod_ab_divergence="true"
|
||
|
|
if [[ "$REQUIRE_AB_PARITY" == "1" ]]; then
|
||
|
|
prod_ab_consistency_blocked="true"
|
||
|
|
prod_ab_divergence="false"
|
||
|
|
prod_ready="false"
|
||
|
|
prod_status="failed"
|
||
|
|
prod_blocked_reason="ab_parity_blocked:path_b_compile_or_test_failed"
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
fi
|
||
|
|
|
||
|
|
jq -nc \
|
||
|
|
--arg id "$id" \
|
||
|
|
--arg project "$project" \
|
||
|
|
--arg category "$category" \
|
||
|
|
--arg language_declared "$language_declared" \
|
||
|
|
--arg language_exec "$lang" \
|
||
|
|
--arg spec "$spec" \
|
||
|
|
--argjson core_semantics "$core_semantics" \
|
||
|
|
--argjson projection_targets "$projection_targets" \
|
||
|
|
--argjson cross_target_invariants "$cross_target_invariants" \
|
||
|
|
--argjson degradation_policy "$degradation_policy" \
|
||
|
|
--argjson constraints "$constraints" \
|
||
|
|
--argjson projection_contract_ok "$projection_contract_ok" \
|
||
|
|
--arg projection_contract_reason "$projection_contract_reason" \
|
||
|
|
--argjson fullstack_contract_ok "$fullstack_contract_ok" \
|
||
|
|
--arg fullstack_contract_reason "$fullstack_contract_reason" \
|
||
|
|
--arg ab_status "$ab_status" \
|
||
|
|
--argjson ab_ready_a "$ab_ready_a" \
|
||
|
|
--argjson ab_ready_b "$ab_ready_b" \
|
||
|
|
--argjson ab_fail_a "$ab_fail_a" \
|
||
|
|
--argjson ab_fail_b "$ab_fail_b" \
|
||
|
|
--argjson ab_tokens_a "$ab_tok_a" \
|
||
|
|
--argjson ab_tokens_b "$ab_tok_b" \
|
||
|
|
--arg ab_out_dir "$ab_out_dir" \
|
||
|
|
--arg ab_err "$ab_err" \
|
||
|
|
--arg prod_status "$prod_status" \
|
||
|
|
--argjson prod_ready "$prod_ready" \
|
||
|
|
--arg prod_blocked_reason "$prod_blocked_reason" \
|
||
|
|
--argjson prod_gate_evidence_complete "$prod_gate_evidence_complete" \
|
||
|
|
--argjson prod_compile_pass "$prod_compile_pass" \
|
||
|
|
--argjson prod_tests_pass "$prod_tests_pass" \
|
||
|
|
--argjson prod_ab_divergence "$prod_ab_divergence" \
|
||
|
|
--argjson prod_ab_consistency_blocked "$prod_ab_consistency_blocked" \
|
||
|
|
--arg prod_out_dir "$prod_out_dir" \
|
||
|
|
--arg prod_err "$prod_err" \
|
||
|
|
'{
|
||
|
|
id:$id,
|
||
|
|
project:$project,
|
||
|
|
category:$category,
|
||
|
|
language_declared:$language_declared,
|
||
|
|
language_exec:$language_exec,
|
||
|
|
spec:$spec,
|
||
|
|
projection_contract:{
|
||
|
|
ok:$projection_contract_ok,
|
||
|
|
reason:$projection_contract_reason,
|
||
|
|
core_semantics:$core_semantics,
|
||
|
|
projection_targets:$projection_targets,
|
||
|
|
cross_target_invariants:$cross_target_invariants,
|
||
|
|
degradation_policy:$degradation_policy
|
||
|
|
},
|
||
|
|
fullstack_contract:{
|
||
|
|
ok:$fullstack_contract_ok,
|
||
|
|
reason:$fullstack_contract_reason,
|
||
|
|
constraints:$constraints
|
||
|
|
},
|
||
|
|
ab:{
|
||
|
|
status:$ab_status,
|
||
|
|
path_a_ready:$ab_ready_a,
|
||
|
|
path_b_ready:$ab_ready_b,
|
||
|
|
path_a_failure_reasons:$ab_fail_a,
|
||
|
|
path_b_failure_reasons:$ab_fail_b,
|
||
|
|
path_a_total_tokens:$ab_tokens_a,
|
||
|
|
path_b_total_tokens:$ab_tokens_b,
|
||
|
|
out_dir:$ab_out_dir,
|
||
|
|
error:$ab_err
|
||
|
|
},
|
||
|
|
production_loop:{
|
||
|
|
status:$prod_status,
|
||
|
|
overall_ready:$prod_ready,
|
||
|
|
blocked_reason:$prod_blocked_reason,
|
||
|
|
gate_evidence_complete:$prod_gate_evidence_complete,
|
||
|
|
compile_pass:$prod_compile_pass,
|
||
|
|
tests_pass:$prod_tests_pass,
|
||
|
|
ab_divergence:$prod_ab_divergence,
|
||
|
|
ab_consistency_blocked:$prod_ab_consistency_blocked,
|
||
|
|
out_dir:$prod_out_dir,
|
||
|
|
error:$prod_err
|
||
|
|
}
|
||
|
|
}' >> "$RESULTS_JSONL"
|
||
|
|
done
|
||
|
|
done < "$CATALOG"
|
||
|
|
|
||
|
|
python3 "$ROOT_DIR/tools/mcp/summarize_project_benchmark_matrix.py" \
|
||
|
|
--results "$RESULTS_JSONL" \
|
||
|
|
--out "$OUT_DIR/summary.json"
|
||
|
|
|
||
|
|
echo "Benchmark matrix output: $OUT_DIR"
|
||
|
|
cat "$OUT_DIR/summary.json"
|