144 lines
4.2 KiB
Bash
Executable File
144 lines
4.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
OUT_DIR="${1:-$ROOT_DIR/datasets/run_specs}"
|
|
VARIANTS="${VARIANTS:-10}"
|
|
|
|
mkdir -p "$OUT_DIR"
|
|
INDEX_FILE="$OUT_DIR/index.jsonl"
|
|
: > "$INDEX_FILE"
|
|
|
|
cat > "$OUT_DIR/README.md" <<'DOC'
|
|
# Run Specs Dataset (Novelty-Oriented)
|
|
|
|
Synthetic data-collection corpus for MCP tool routing and execution quality.
|
|
This corpus is decoupled from editor sprint plans.
|
|
|
|
- `runspec_*.md`: per-case run specs
|
|
- `index.jsonl`: family + variant metadata
|
|
|
|
Families include both baseline and expansion categories to prevent local minima.
|
|
DOC
|
|
|
|
families=(
|
|
invalid_action_format
|
|
missing_required_args
|
|
wrong_tool_selection
|
|
dependency_order_violation
|
|
non_terminal_loop
|
|
ambiguous_scope_resolution
|
|
premature_finalize
|
|
overfetch_context
|
|
recovery_path_misroute
|
|
schema_boundary_violation
|
|
)
|
|
|
|
for family in "${families[@]}"; do
|
|
for i in $(seq 1 "$VARIANTS"); do
|
|
id="$(printf '%s_%02d' "$family" "$i")"
|
|
file="$OUT_DIR/runspec_${id}.md"
|
|
|
|
case "$family" in
|
|
invalid_action_format)
|
|
anti='Produce non-conforming action envelopes or markdown-wrapped payloads.'
|
|
good='Emit strict JSON action schema with valid keys and types.'
|
|
;;
|
|
missing_required_args)
|
|
anti='Call valid tools while omitting required args or using null placeholders.'
|
|
good='Provide required arguments with valid types and required collections.'
|
|
;;
|
|
wrong_tool_selection)
|
|
anti='Select semantically adjacent tools for the wrong workflow phase.'
|
|
good='Use phase-correct ordering: intake -> generate -> queue -> validate.'
|
|
;;
|
|
dependency_order_violation)
|
|
anti='Attempt queue/validate before generation artifacts exist.'
|
|
good='Respect dependency order and unblock in minimal corrective steps.'
|
|
;;
|
|
non_terminal_loop)
|
|
anti='Repeat successful low-value calls without converging to final action.'
|
|
good='Converge with explicit terminal decision and bounded retries.'
|
|
;;
|
|
ambiguous_scope_resolution)
|
|
anti='Use absolute or out-of-scope paths when workspace-relative is required.'
|
|
good='Keep all operations workspace-relative and scope-safe.'
|
|
;;
|
|
premature_finalize)
|
|
anti='Emit final action before collecting minimum required evidence.'
|
|
good='Finalize only after required tool evidence is present.'
|
|
;;
|
|
overfetch_context)
|
|
anti='Read excessive unrelated files to complete a narrow task.'
|
|
good='Use minimal context slices and avoid broad workspace sweeps.'
|
|
;;
|
|
recovery_path_misroute)
|
|
anti='On first failure, jump to unrelated tools instead of local recovery path.'
|
|
good='Apply deterministic nearest recovery path before escalation.'
|
|
;;
|
|
schema_boundary_violation)
|
|
anti='Cross schema boundaries (wrong field names/casing/nesting) in tool args.'
|
|
good='Use exact schema-compatible fields and shape in all tool arguments.'
|
|
;;
|
|
*)
|
|
anti='Unknown anti-pattern.'
|
|
good='Unknown expected behavior.'
|
|
;;
|
|
esac
|
|
|
|
cat > "$file" <<SPEC
|
|
# Run Spec: ${id}
|
|
|
|
## Context
|
|
|
|
Synthetic MCP data-collection case.
|
|
family = ${family}
|
|
variant = ${i}
|
|
|
|
## Objective
|
|
|
|
Exercise ${family} behavior and produce deterministic stage-labeled traces.
|
|
|
|
## Anti-Pattern To Detect
|
|
|
|
- ${anti}
|
|
|
|
## Target Behavior
|
|
|
|
- ${good}
|
|
- Keep tool-call chain bounded and auditable.
|
|
- Preserve deterministic output ordering.
|
|
|
|
## Requirements
|
|
|
|
- Workspace-relative paths only.
|
|
- Deterministic tool sequencing.
|
|
- Valid JSON arguments for each tool invocation.
|
|
|
|
## Acceptance Criteria
|
|
|
|
- At least one tool_selection event.
|
|
- At least one tool_execution event.
|
|
- No malformed MCP payloads.
|
|
- No unbounded loops.
|
|
|
|
## Data Capture Tags
|
|
|
|
- family: ${family}
|
|
- variant: ${i}
|
|
- capture_mode: run_spec
|
|
- expected_failure_signature: ${family}
|
|
SPEC
|
|
|
|
jq -nc \
|
|
--arg id "$id" \
|
|
--arg file "$(basename "$file")" \
|
|
--arg family "$family" \
|
|
--argjson variant "$i" \
|
|
'{id:$id,file:$file,family:$family,variant:$variant}' >> "$INDEX_FILE"
|
|
done
|
|
done
|
|
|
|
echo "Generated run specs in: $OUT_DIR"
|
|
echo "Total specs: $(find "$OUT_DIR" -maxdepth 1 -type f -name 'runspec_*.md' | wc -l | tr -d ' ')"
|