Integrate spec-readiness precheck into sprint pipeline
This commit is contained in:
Bill
@@ -51,3 +51,12 @@
|
||||
|
||||
- Current path-B repair closure is pattern-driven around recurring queue-shaped transpile failures.
|
||||
- Next required tranche: broaden to non-queue semantics and first-class spec-to-execution-ready planning.
|
||||
|
||||
## Added After Commit `55876d3` (Same Day Continuation)
|
||||
|
||||
- Wired first-class planning precheck into `tools/mcp/run_sprint_taskitem_pipeline.sh`.
|
||||
- Added environment controls:
|
||||
- `WSTONE_SPEC_READINESS_PRECHECK`
|
||||
- `WSTONE_SPEC_READINESS_HARD_GATE`
|
||||
- `WSTONE_SPEC_READINESS_MIN_SCORE`
|
||||
- Spec readiness is now available in pipeline `00_summary.json` as `planning_readiness`.
|
||||
|
||||
@@ -11,6 +11,12 @@ Planned and started:
|
||||
|
||||
- Added planning-readiness tool:
|
||||
- `tools/mcp/spec_planning_readiness.py`
|
||||
- Integrated planning precheck into taskitem pipeline:
|
||||
- `tools/mcp/run_sprint_taskitem_pipeline.sh`
|
||||
- new env controls:
|
||||
- `WSTONE_SPEC_READINESS_PRECHECK` (default `1`)
|
||||
- `WSTONE_SPEC_READINESS_HARD_GATE` (default `0`)
|
||||
- `WSTONE_SPEC_READINESS_MIN_SCORE` (default `65`)
|
||||
- Tool outputs:
|
||||
- readiness score (`section_score`, `keyword_score`, `total`)
|
||||
- verdict (`execution_ready` or `needs_spec_hardening`)
|
||||
@@ -28,13 +34,13 @@ Observed baseline:
|
||||
|
||||
## Explicit Completion Signal
|
||||
|
||||
- Sprint 228: `PARTIAL` (tool implemented and baseline executed)
|
||||
- Sprint 228: `PARTIAL` (tool implemented, baseline executed, pipeline precheck integrated)
|
||||
- Sprint 229: `PARTIAL` (constraint synthesis mapping implemented in tool)
|
||||
- Sprint 230: `PARTIAL` (acceptance readiness checks implemented in tool)
|
||||
- Sprint 231: `PARTIAL` (environment/projection checks implemented in tool)
|
||||
|
||||
## Next Closure Work
|
||||
|
||||
- Bind readiness tool into pre-taskitem pipeline gate.
|
||||
- Enable hard-gate mode for selected sprint ranges and capture pass/fail deltas.
|
||||
- Add structured spec hardening pass that applies template scaffolds to candidate specs.
|
||||
- Re-run benchmarks using hardened specs and measure downstream readiness deltas.
|
||||
|
||||
@@ -7,26 +7,69 @@ if ! command -v jq >/dev/null 2>&1; then
|
||||
fi
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
BIN="${WSTONE_MCP_BIN:-$ROOT_DIR/editor/build-native/whetstone_mcp_stable}"
|
||||
if [[ -z "${WSTONE_MCP_BIN:-}" ]]; then
|
||||
if [[ -x "$ROOT_DIR/editor/build-native/whetstone_mcp" ]]; then
|
||||
BIN="$ROOT_DIR/editor/build-native/whetstone_mcp"
|
||||
else
|
||||
BIN="$ROOT_DIR/editor/build-native/whetstone_mcp_stable"
|
||||
fi
|
||||
else
|
||||
BIN="$WSTONE_MCP_BIN"
|
||||
fi
|
||||
WORKSPACE="${WSTONE_WORKSPACE:-$ROOT_DIR}"
|
||||
LANGUAGE="${WSTONE_LANGUAGE:-cpp}"
|
||||
SPRINT_FILE_REL="${1:-sprint46_plan.md}"
|
||||
SPRINT_FILE="$ROOT_DIR/$SPRINT_FILE_REL"
|
||||
STRICT_EXECUTION_CONTRACT="${WSTONE_STRICT_EXECUTION_CONTRACT:-1}"
|
||||
CALIBRATE_AFTER_RUN="${WSTONE_CALIBRATE_AFTER_RUN:-0}"
|
||||
RUN_READINESS_SUITE="${WSTONE_RUN_READINESS_SUITE:-1}"
|
||||
SPEC_READINESS_PRECHECK="${WSTONE_SPEC_READINESS_PRECHECK:-1}"
|
||||
SPEC_READINESS_HARD_GATE="${WSTONE_SPEC_READINESS_HARD_GATE:-0}"
|
||||
SPEC_READINESS_MIN_SCORE="${WSTONE_SPEC_READINESS_MIN_SCORE:-65}"
|
||||
CAPABILITY_SIGNALS_JSON="${WSTONE_CAPABILITY_SIGNALS_JSON:-}"
|
||||
if [[ -z "$CAPABILITY_SIGNALS_JSON" ]]; then
|
||||
CAPABILITY_SIGNALS_JSON='{}'
|
||||
fi
|
||||
INPUT_FILE_ARG="${1:-sprint46_plan.md}"
|
||||
if [[ "$INPUT_FILE_ARG" = /* ]]; then
|
||||
INPUT_FILE="$INPUT_FILE_ARG"
|
||||
INPUT_FILE_LABEL="$INPUT_FILE_ARG"
|
||||
else
|
||||
INPUT_FILE="$ROOT_DIR/$INPUT_FILE_ARG"
|
||||
INPUT_FILE_LABEL="$INPUT_FILE_ARG"
|
||||
fi
|
||||
|
||||
if [[ ! -x "$BIN" ]]; then
|
||||
echo "error: MCP binary not executable: $BIN" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ ! -f "$SPRINT_FILE" ]]; then
|
||||
echo "error: sprint file not found: $SPRINT_FILE" >&2
|
||||
if [[ ! -f "$INPUT_FILE" ]]; then
|
||||
echo "error: input file not found: $INPUT_FILE" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! printf '%s' "$CAPABILITY_SIGNALS_JSON" | jq -e . >/dev/null 2>&1; then
|
||||
echo "error: WSTONE_CAPABILITY_SIGNALS_JSON is not valid JSON" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
STAMP="$(date +%Y%m%d_%H%M%S)"
|
||||
SPRINT_BASENAME="$(basename "$SPRINT_FILE_REL" .md)"
|
||||
OUT_DIR="$ROOT_DIR/logs/taskitem_runs/${SPRINT_BASENAME}_${STAMP}"
|
||||
INPUT_BASENAME="$(basename "$INPUT_FILE_LABEL" .md)"
|
||||
OUT_DIR="$ROOT_DIR/logs/taskitem_runs/${INPUT_BASENAME}_${STAMP}"
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
SPEC_READINESS_JSON='{}'
|
||||
if [[ "$SPEC_READINESS_PRECHECK" == "1" ]]; then
|
||||
python3 "$ROOT_DIR/tools/mcp/spec_planning_readiness.py" \
|
||||
--spec "$INPUT_FILE" \
|
||||
--out "$OUT_DIR/00a_spec_readiness.json" >/dev/null
|
||||
SPEC_READINESS_JSON="$(cat "$OUT_DIR/00a_spec_readiness.json")"
|
||||
SPEC_READINESS_SCORE="$(printf '%s' "$SPEC_READINESS_JSON" | jq -r '.scores.total // 0')"
|
||||
SPEC_READINESS_VERDICT="$(printf '%s' "$SPEC_READINESS_JSON" | jq -r '.verdict // "unknown"')"
|
||||
if [[ "$SPEC_READINESS_HARD_GATE" == "1" ]] && [[ "$SPEC_READINESS_SCORE" -lt "$SPEC_READINESS_MIN_SCORE" ]]; then
|
||||
echo "error: spec readiness hard gate failed ($SPEC_READINESS_SCORE < $SPEC_READINESS_MIN_SCORE, verdict=$SPEC_READINESS_VERDICT)" >&2
|
||||
echo "error: see $OUT_DIR/00a_spec_readiness.json for missing_actions and recommended_template" >&2
|
||||
exit 7
|
||||
fi
|
||||
fi
|
||||
|
||||
call_tool() {
|
||||
local tool_name="$1"
|
||||
local args_json="$2"
|
||||
@@ -60,9 +103,11 @@ extract_tool_text_json() {
|
||||
|
||||
build_fallback_intake_spec() {
|
||||
local source_file="$1"
|
||||
local tmp_steps tmp_constraints
|
||||
local tmp_steps tmp_constraints tmp_refs tmp_tools
|
||||
tmp_steps="$(mktemp)"
|
||||
tmp_constraints="$(mktemp)"
|
||||
tmp_refs="$(mktemp)"
|
||||
tmp_tools="$(mktemp)"
|
||||
|
||||
awk '/^### Step /{sub(/^### /, "", $0); print "- " $0}' "$source_file" > "$tmp_steps"
|
||||
awk '
|
||||
@@ -70,6 +115,26 @@ build_fallback_intake_spec() {
|
||||
/^## / && in_gate {in_gate=0}
|
||||
in_gate && /^- / {print}
|
||||
' "$source_file" > "$tmp_constraints"
|
||||
awk '
|
||||
{
|
||||
line=$0
|
||||
while (match(line, /`[^`]+`/)) {
|
||||
tok=substr(line, RSTART+1, RLENGTH-2)
|
||||
if (tok ~ /[\/.]/) print "- " tok
|
||||
line=substr(line, RSTART+RLENGTH)
|
||||
}
|
||||
}
|
||||
' "$source_file" | sort -u > "$tmp_refs"
|
||||
awk '
|
||||
{
|
||||
line=tolower($0)
|
||||
while (match(line, /whetstone_[a-z0-9_]+/)) {
|
||||
tok=substr(line, RSTART, RLENGTH)
|
||||
print "- " tok
|
||||
line=substr(line, RSTART+RLENGTH)
|
||||
}
|
||||
}
|
||||
' "$source_file" | sort -u > "$tmp_tools"
|
||||
|
||||
{
|
||||
echo "## Goals"
|
||||
@@ -86,32 +151,43 @@ build_fallback_intake_spec() {
|
||||
echo "## Dependencies"
|
||||
echo "- Existing editor/src modules and MCP toolchain"
|
||||
echo "- whetstone_mcp stable binary and workspace config"
|
||||
if [[ -s "$tmp_refs" ]]; then
|
||||
cat "$tmp_refs"
|
||||
fi
|
||||
if [[ -s "$tmp_tools" ]]; then
|
||||
cat "$tmp_tools"
|
||||
fi
|
||||
echo
|
||||
echo "## Acceptance Criteria"
|
||||
echo "- All sprint step tests pass"
|
||||
echo "- No regression in existing MCP tool behavior"
|
||||
echo "- Generated task queue is ready or blockers are explicit"
|
||||
if [[ -s "$tmp_steps" ]]; then
|
||||
awk '{print "- " $0}' "$tmp_steps"
|
||||
fi
|
||||
} | sed '/^[[:space:]]*$/N;/^\n$/D'
|
||||
|
||||
rm -f "$tmp_steps" "$tmp_constraints"
|
||||
rm -f "$tmp_steps" "$tmp_constraints" "$tmp_refs" "$tmp_tools"
|
||||
}
|
||||
|
||||
MARKDOWN_CONTENT="$(cat "$SPRINT_FILE")"
|
||||
MARKDOWN_CONTENT="$(cat "$INPUT_FILE")"
|
||||
INTAKE_ARGS="$(jq -nc --arg md "$MARKDOWN_CONTENT" '{markdown:$md}')"
|
||||
INTAKE_RESP_RAW="$(call_tool "whetstone_architect_intake" "$INTAKE_ARGS")"
|
||||
printf '%s\n' "$INTAKE_RESP_RAW" > "$OUT_DIR/01_intake_raw.ndjson.json"
|
||||
INTAKE_JSON="$(extract_tool_text_json "$INTAKE_RESP_RAW")"
|
||||
printf '%s\n' "$INTAKE_JSON" > "$OUT_DIR/01_intake.json"
|
||||
EFFECTIVE_INTAKE_JSON_PATH="$OUT_DIR/01_intake.json"
|
||||
|
||||
if [[ "$(printf '%s' "$INTAKE_JSON" | jq -r '.success // false')" != "true" ]] &&
|
||||
[[ "$(printf '%s' "$INTAKE_JSON" | jq -r '.error // ""')" == "no_requirements_found" ]]; then
|
||||
FALLBACK_SPEC="$(build_fallback_intake_spec "$SPRINT_FILE")"
|
||||
FALLBACK_SPEC="$(build_fallback_intake_spec "$INPUT_FILE")"
|
||||
printf '%s\n' "$FALLBACK_SPEC" > "$OUT_DIR/01a_fallback_intake_spec.md"
|
||||
INTAKE_ARGS="$(jq -nc --arg md "$FALLBACK_SPEC" '{markdown:$md}')"
|
||||
INTAKE_RESP_RAW="$(call_tool "whetstone_architect_intake" "$INTAKE_ARGS")"
|
||||
printf '%s\n' "$INTAKE_RESP_RAW" > "$OUT_DIR/01b_intake_retry_raw.ndjson.json"
|
||||
INTAKE_JSON="$(extract_tool_text_json "$INTAKE_RESP_RAW")"
|
||||
printf '%s\n' "$INTAKE_JSON" > "$OUT_DIR/01b_intake_retry.json"
|
||||
EFFECTIVE_INTAKE_JSON_PATH="$OUT_DIR/01b_intake_retry.json"
|
||||
fi
|
||||
|
||||
if [[ "$(printf '%s' "$INTAKE_JSON" | jq -r '.success // false')" != "true" ]]; then
|
||||
@@ -121,7 +197,8 @@ fi
|
||||
|
||||
NORMALIZED_REQS="$(printf '%s' "$INTAKE_JSON" | jq '.normalizedRequirements')"
|
||||
CONFLICTS="$(printf '%s' "$INTAKE_JSON" | jq '.conflicts // []')"
|
||||
GEN_ARGS="$(jq -nc --argjson nr "$NORMALIZED_REQS" --argjson cf "$CONFLICTS" '{normalizedRequirements:$nr,conflicts:$cf}')"
|
||||
GEN_ARGS="$(jq -nc --argjson nr "$NORMALIZED_REQS" --argjson cf "$CONFLICTS" --arg strict "$STRICT_EXECUTION_CONTRACT" \
|
||||
'{normalizedRequirements:$nr,conflicts:$cf,strictExecutionContract:($strict == "1")}')"
|
||||
GEN_RESP_RAW="$(call_tool "whetstone_generate_taskitems" "$GEN_ARGS")"
|
||||
printf '%s\n' "$GEN_RESP_RAW" > "$OUT_DIR/02_generate_taskitems_raw.ndjson.json"
|
||||
GEN_JSON="$(extract_tool_text_json "$GEN_RESP_RAW")"
|
||||
@@ -132,41 +209,49 @@ if [[ "$(printf '%s' "$GEN_JSON" | jq -r '.success // false')" != "true" ]]; the
|
||||
fi
|
||||
|
||||
TASKS="$(printf '%s' "$GEN_JSON" | jq '.tasks')"
|
||||
QUEUE_ARGS="$(jq -nc --argjson t "$TASKS" --argjson nr "$NORMALIZED_REQS" '{tasks:$t,normalizedRequirements:$nr}')"
|
||||
QUEUE_ARGS="$(jq -nc --argjson t "$TASKS" --argjson nr "$NORMALIZED_REQS" --arg strict "$STRICT_EXECUTION_CONTRACT" --argjson cs "$CAPABILITY_SIGNALS_JSON" \
|
||||
'{tasks:$t,normalizedRequirements:$nr,strictExecutionContract:($strict == "1"),capabilitySignals:$cs}')"
|
||||
QUEUE_RESP_RAW="$(call_tool "whetstone_queue_ready" "$QUEUE_ARGS")"
|
||||
printf '%s\n' "$QUEUE_RESP_RAW" > "$OUT_DIR/03_queue_ready_raw.ndjson.json"
|
||||
QUEUE_JSON="$(extract_tool_text_json "$QUEUE_RESP_RAW")"
|
||||
printf '%s\n' "$QUEUE_JSON" > "$OUT_DIR/03_queue_ready.json"
|
||||
|
||||
VALIDATE_TASKS="$(printf '%s' "$TASKS" | jq '[.[] | {
|
||||
VALIDATE_TASKS="$(printf '%s' "$TASKS" | jq --argjson cs "$CAPABILITY_SIGNALS_JSON" '[.[] | {
|
||||
task_id: .taskId,
|
||||
title: .title,
|
||||
prerequisite_ops: (.prerequisiteOps // []),
|
||||
reasons: (.reasons // []),
|
||||
confidence: (.confidence // 0),
|
||||
dependency_task_ids: (.dependencyTaskIds // [])
|
||||
dependency_task_ids: (.dependencyTaskIds // []),
|
||||
resource_locks: (.resourceLocks // []),
|
||||
execution_contract: (.executionContract // {}),
|
||||
capability_signals: $cs
|
||||
}]')"
|
||||
VALIDATE_ARGS="$(jq -nc --argjson items "$VALIDATE_TASKS" --arg ws "$WORKSPACE" '{taskitems:$items,workspace:$ws}')"
|
||||
VALIDATE_ARGS="$(jq -nc --argjson items "$VALIDATE_TASKS" --arg ws "$WORKSPACE" --arg strict "$STRICT_EXECUTION_CONTRACT" \
|
||||
'{taskitems:$items,workspace:$ws,strict_execution_contract:($strict == "1")}')"
|
||||
VALIDATE_RESP_RAW="$(call_tool "whetstone_validate_taskitem" "$VALIDATE_ARGS")"
|
||||
printf '%s\n' "$VALIDATE_RESP_RAW" > "$OUT_DIR/04_validate_taskitem_raw.ndjson.json"
|
||||
VALIDATE_JSON="$(extract_tool_text_json "$VALIDATE_RESP_RAW")"
|
||||
printf '%s\n' "$VALIDATE_JSON" > "$OUT_DIR/04_validate_taskitem.json"
|
||||
|
||||
SUMMARY_JSON="$(jq -nc \
|
||||
--arg sprint "$SPRINT_FILE_REL" \
|
||||
--arg input_file "$INPUT_FILE_LABEL" \
|
||||
--arg outDir "$OUT_DIR" \
|
||||
--arg bin "$BIN" \
|
||||
--arg workspace "$WORKSPACE" \
|
||||
--argjson planning_readiness "$SPEC_READINESS_JSON" \
|
||||
--argjson intake "$INTAKE_JSON" \
|
||||
--argjson generated "$GEN_JSON" \
|
||||
--argjson queue "$QUEUE_JSON" \
|
||||
--argjson validate "$VALIDATE_JSON" \
|
||||
'{
|
||||
sprint: $sprint,
|
||||
sprint: $input_file,
|
||||
input_file: $input_file,
|
||||
timestamp: now|todate,
|
||||
mcp_binary: $bin,
|
||||
workspace: $workspace,
|
||||
output_dir: $outDir,
|
||||
planning_readiness: $planning_readiness,
|
||||
intake: {
|
||||
success: ($intake.success // false),
|
||||
normalized_requirement_count: (($intake.normalizedRequirements // [])|length),
|
||||
@@ -176,22 +261,66 @@ SUMMARY_JSON="$(jq -nc \
|
||||
taskitems: {
|
||||
success: ($generated.success // false),
|
||||
task_count: (($generated.tasks // [])|length),
|
||||
escalate_count: ($generated.escalateCount // 0)
|
||||
escalate_count: ($generated.escalateCount // 0),
|
||||
strict_execution_contract: ($generated.strictExecutionContract // false),
|
||||
missing_execution_contract_count: ($generated.missingExecutionContractCount // 0)
|
||||
},
|
||||
queue_ready: {
|
||||
success: ($queue.success // false),
|
||||
ready: ($queue.ready // false),
|
||||
ready_count: ($queue.readyCount // 0),
|
||||
blocker_count: (($queue.blockers // [])|length)
|
||||
blocker_count: (($queue.blockers // [])|length),
|
||||
queue_warnings: ($queue.queueWarnings // []),
|
||||
resource_conflict_count: ($queue.resourceConflictCount // 0),
|
||||
strict_execution_contract: ($queue.strictExecutionContract // false),
|
||||
execution_contract_missing_count: ($queue.executionContractMissingCount // 0),
|
||||
execution_specificity_low_count: ($queue.executionSpecificityLowCount // 0),
|
||||
gap_class_counts: ($queue.gap_class_counts // {}),
|
||||
remediation_routing_hints: ($queue.remediationRoutingHints // {}),
|
||||
debug_loop_call_plan: ($queue.debugLoopCallPlan // {})
|
||||
},
|
||||
validation: {
|
||||
success: ($validate.success // false),
|
||||
total_taskitems: ($validate.report.total_taskitems // 0),
|
||||
average_score: ($validate.report.average_score // 0),
|
||||
average_execution_specificity_score: ($validate.report.average_execution_specificity_score // 0),
|
||||
self_contained_count: ($validate.report.self_contained_count // 0),
|
||||
failing_count: ($validate.report.failing_count // 0)
|
||||
failing_count: ($validate.report.failing_count // 0),
|
||||
gap_class_counts: ($validate.report.gap_class_counts // {}),
|
||||
top_recommended_tools: ($validate.report.top_recommended_tools // []),
|
||||
capability_gap_call_plan: ($validate.report.capability_gap_call_plan // {}),
|
||||
promotion_packet: ($validate.report.promotion_packet // {})
|
||||
}
|
||||
}')"
|
||||
|
||||
if [[ "$CALIBRATE_AFTER_RUN" == "1" ]]; then
|
||||
CALIBRATION_OUT_DIR="$OUT_DIR/calibration"
|
||||
CALIBRATION_JSON="$(python3 "$ROOT_DIR/tools/mcp/analyze_taskitem_calibration.py" \
|
||||
--runs-root "$ROOT_DIR/logs/taskitem_runs" \
|
||||
--out-dir "$CALIBRATION_OUT_DIR")"
|
||||
if [[ -n "$CALIBRATION_JSON" ]] && printf '%s' "$CALIBRATION_JSON" | jq -e . >/dev/null 2>&1; then
|
||||
SUMMARY_JSON="$(printf '%s' "$SUMMARY_JSON" | jq --argjson calibration "$CALIBRATION_JSON" '.calibration = $calibration')"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Persist current summary before optional post-check suites that read summary fields.
|
||||
printf '%s\n' "$SUMMARY_JSON" > "$OUT_DIR/00_summary.json"
|
||||
|
||||
if [[ "$RUN_READINESS_SUITE" == "1" ]]; then
|
||||
READINESS_JSON="$(python3 "$ROOT_DIR/tools/mcp/run_generator_readiness_suite.py" \
|
||||
--repo-root "$ROOT_DIR" \
|
||||
--out-dir "$OUT_DIR" \
|
||||
--summary-json "$OUT_DIR/00_summary.json" \
|
||||
--intake-json "$EFFECTIVE_INTAKE_JSON_PATH" \
|
||||
--generated-json "$OUT_DIR/02_generate_taskitems.json" \
|
||||
--queue-json "$OUT_DIR/03_queue_ready.json" \
|
||||
--validate-json "$OUT_DIR/04_validate_taskitem.json" \
|
||||
--input-file "$INPUT_FILE")"
|
||||
if [[ -n "$READINESS_JSON" ]] && printf '%s' "$READINESS_JSON" | jq -e . >/dev/null 2>&1; then
|
||||
SUMMARY_JSON="$(printf '%s' "$SUMMARY_JSON" | jq --argjson readiness "$READINESS_JSON" '.readiness = $readiness')"
|
||||
fi
|
||||
fi
|
||||
|
||||
printf '%s\n' "$SUMMARY_JSON" > "$OUT_DIR/00_summary.json"
|
||||
|
||||
echo "Taskitem pipeline complete."
|
||||
|
||||
Reference in New Issue
Block a user