From ccf8f7a817bb39498c36f183d14aae695cb7688b Mon Sep 17 00:00:00 2001 From: Bill Date: Thu, 26 Feb 2026 14:09:13 -0700 Subject: [PATCH] Integrate spec-readiness precheck into sprint pipeline --- docs/progress_log_2026-02-26.md | 9 + ...int228_231_execution_tracker_2026-02-26.md | 10 +- tools/mcp/run_sprint_taskitem_pipeline.sh | 171 +++++++++++++++--- 3 files changed, 167 insertions(+), 23 deletions(-) diff --git a/docs/progress_log_2026-02-26.md b/docs/progress_log_2026-02-26.md index 9f9b8ff..c0bad3b 100644 --- a/docs/progress_log_2026-02-26.md +++ b/docs/progress_log_2026-02-26.md @@ -51,3 +51,12 @@ - Current path-B repair closure is pattern-driven around recurring queue-shaped transpile failures. - Next required tranche: broaden to non-queue semantics and first-class spec-to-execution-ready planning. + +## Added After Commit `55876d3` (Same Day Continuation) + +- Wired first-class planning precheck into `tools/mcp/run_sprint_taskitem_pipeline.sh`. +- Added environment controls: + - `WSTONE_SPEC_READINESS_PRECHECK` + - `WSTONE_SPEC_READINESS_HARD_GATE` + - `WSTONE_SPEC_READINESS_MIN_SCORE` +- Spec readiness is now available in pipeline `00_summary.json` as `planning_readiness`. diff --git a/docs/sprint228_231_execution_tracker_2026-02-26.md b/docs/sprint228_231_execution_tracker_2026-02-26.md index 32d91b1..b2750e7 100644 --- a/docs/sprint228_231_execution_tracker_2026-02-26.md +++ b/docs/sprint228_231_execution_tracker_2026-02-26.md @@ -11,6 +11,12 @@ Planned and started: - Added planning-readiness tool: - `tools/mcp/spec_planning_readiness.py` +- Integrated planning precheck into taskitem pipeline: + - `tools/mcp/run_sprint_taskitem_pipeline.sh` + - new env controls: + - `WSTONE_SPEC_READINESS_PRECHECK` (default `1`) + - `WSTONE_SPEC_READINESS_HARD_GATE` (default `0`) + - `WSTONE_SPEC_READINESS_MIN_SCORE` (default `65`) - Tool outputs: - readiness score (`section_score`, `keyword_score`, `total`) - verdict (`execution_ready` or `needs_spec_hardening`) @@ -28,13 +34,13 @@ Observed baseline: ## Explicit Completion Signal -- Sprint 228: `PARTIAL` (tool implemented and baseline executed) +- Sprint 228: `PARTIAL` (tool implemented, baseline executed, pipeline precheck integrated) - Sprint 229: `PARTIAL` (constraint synthesis mapping implemented in tool) - Sprint 230: `PARTIAL` (acceptance readiness checks implemented in tool) - Sprint 231: `PARTIAL` (environment/projection checks implemented in tool) ## Next Closure Work -- Bind readiness tool into pre-taskitem pipeline gate. +- Enable hard-gate mode for selected sprint ranges and capture pass/fail deltas. - Add structured spec hardening pass that applies template scaffolds to candidate specs. - Re-run benchmarks using hardened specs and measure downstream readiness deltas. diff --git a/tools/mcp/run_sprint_taskitem_pipeline.sh b/tools/mcp/run_sprint_taskitem_pipeline.sh index 4c5a074..af3485e 100755 --- a/tools/mcp/run_sprint_taskitem_pipeline.sh +++ b/tools/mcp/run_sprint_taskitem_pipeline.sh @@ -7,26 +7,69 @@ if ! command -v jq >/dev/null 2>&1; then fi ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" -BIN="${WSTONE_MCP_BIN:-$ROOT_DIR/editor/build-native/whetstone_mcp_stable}" +if [[ -z "${WSTONE_MCP_BIN:-}" ]]; then + if [[ -x "$ROOT_DIR/editor/build-native/whetstone_mcp" ]]; then + BIN="$ROOT_DIR/editor/build-native/whetstone_mcp" + else + BIN="$ROOT_DIR/editor/build-native/whetstone_mcp_stable" + fi +else + BIN="$WSTONE_MCP_BIN" +fi WORKSPACE="${WSTONE_WORKSPACE:-$ROOT_DIR}" LANGUAGE="${WSTONE_LANGUAGE:-cpp}" -SPRINT_FILE_REL="${1:-sprint46_plan.md}" -SPRINT_FILE="$ROOT_DIR/$SPRINT_FILE_REL" +STRICT_EXECUTION_CONTRACT="${WSTONE_STRICT_EXECUTION_CONTRACT:-1}" +CALIBRATE_AFTER_RUN="${WSTONE_CALIBRATE_AFTER_RUN:-0}" +RUN_READINESS_SUITE="${WSTONE_RUN_READINESS_SUITE:-1}" +SPEC_READINESS_PRECHECK="${WSTONE_SPEC_READINESS_PRECHECK:-1}" +SPEC_READINESS_HARD_GATE="${WSTONE_SPEC_READINESS_HARD_GATE:-0}" +SPEC_READINESS_MIN_SCORE="${WSTONE_SPEC_READINESS_MIN_SCORE:-65}" +CAPABILITY_SIGNALS_JSON="${WSTONE_CAPABILITY_SIGNALS_JSON:-}" +if [[ -z "$CAPABILITY_SIGNALS_JSON" ]]; then + CAPABILITY_SIGNALS_JSON='{}' +fi +INPUT_FILE_ARG="${1:-sprint46_plan.md}" +if [[ "$INPUT_FILE_ARG" = /* ]]; then + INPUT_FILE="$INPUT_FILE_ARG" + INPUT_FILE_LABEL="$INPUT_FILE_ARG" +else + INPUT_FILE="$ROOT_DIR/$INPUT_FILE_ARG" + INPUT_FILE_LABEL="$INPUT_FILE_ARG" +fi if [[ ! -x "$BIN" ]]; then echo "error: MCP binary not executable: $BIN" >&2 exit 1 fi -if [[ ! -f "$SPRINT_FILE" ]]; then - echo "error: sprint file not found: $SPRINT_FILE" >&2 +if [[ ! -f "$INPUT_FILE" ]]; then + echo "error: input file not found: $INPUT_FILE" >&2 + exit 1 +fi +if ! printf '%s' "$CAPABILITY_SIGNALS_JSON" | jq -e . >/dev/null 2>&1; then + echo "error: WSTONE_CAPABILITY_SIGNALS_JSON is not valid JSON" >&2 exit 1 fi STAMP="$(date +%Y%m%d_%H%M%S)" -SPRINT_BASENAME="$(basename "$SPRINT_FILE_REL" .md)" -OUT_DIR="$ROOT_DIR/logs/taskitem_runs/${SPRINT_BASENAME}_${STAMP}" +INPUT_BASENAME="$(basename "$INPUT_FILE_LABEL" .md)" +OUT_DIR="$ROOT_DIR/logs/taskitem_runs/${INPUT_BASENAME}_${STAMP}" mkdir -p "$OUT_DIR" +SPEC_READINESS_JSON='{}' +if [[ "$SPEC_READINESS_PRECHECK" == "1" ]]; then + python3 "$ROOT_DIR/tools/mcp/spec_planning_readiness.py" \ + --spec "$INPUT_FILE" \ + --out "$OUT_DIR/00a_spec_readiness.json" >/dev/null + SPEC_READINESS_JSON="$(cat "$OUT_DIR/00a_spec_readiness.json")" + SPEC_READINESS_SCORE="$(printf '%s' "$SPEC_READINESS_JSON" | jq -r '.scores.total // 0')" + SPEC_READINESS_VERDICT="$(printf '%s' "$SPEC_READINESS_JSON" | jq -r '.verdict // "unknown"')" + if [[ "$SPEC_READINESS_HARD_GATE" == "1" ]] && [[ "$SPEC_READINESS_SCORE" -lt "$SPEC_READINESS_MIN_SCORE" ]]; then + echo "error: spec readiness hard gate failed ($SPEC_READINESS_SCORE < $SPEC_READINESS_MIN_SCORE, verdict=$SPEC_READINESS_VERDICT)" >&2 + echo "error: see $OUT_DIR/00a_spec_readiness.json for missing_actions and recommended_template" >&2 + exit 7 + fi +fi + call_tool() { local tool_name="$1" local args_json="$2" @@ -60,9 +103,11 @@ extract_tool_text_json() { build_fallback_intake_spec() { local source_file="$1" - local tmp_steps tmp_constraints + local tmp_steps tmp_constraints tmp_refs tmp_tools tmp_steps="$(mktemp)" tmp_constraints="$(mktemp)" + tmp_refs="$(mktemp)" + tmp_tools="$(mktemp)" awk '/^### Step /{sub(/^### /, "", $0); print "- " $0}' "$source_file" > "$tmp_steps" awk ' @@ -70,6 +115,26 @@ build_fallback_intake_spec() { /^## / && in_gate {in_gate=0} in_gate && /^- / {print} ' "$source_file" > "$tmp_constraints" + awk ' + { + line=$0 + while (match(line, /`[^`]+`/)) { + tok=substr(line, RSTART+1, RLENGTH-2) + if (tok ~ /[\/.]/) print "- " tok + line=substr(line, RSTART+RLENGTH) + } + } + ' "$source_file" | sort -u > "$tmp_refs" + awk ' + { + line=tolower($0) + while (match(line, /whetstone_[a-z0-9_]+/)) { + tok=substr(line, RSTART, RLENGTH) + print "- " tok + line=substr(line, RSTART+RLENGTH) + } + } + ' "$source_file" | sort -u > "$tmp_tools" { echo "## Goals" @@ -86,32 +151,43 @@ build_fallback_intake_spec() { echo "## Dependencies" echo "- Existing editor/src modules and MCP toolchain" echo "- whetstone_mcp stable binary and workspace config" + if [[ -s "$tmp_refs" ]]; then + cat "$tmp_refs" + fi + if [[ -s "$tmp_tools" ]]; then + cat "$tmp_tools" + fi echo echo "## Acceptance Criteria" echo "- All sprint step tests pass" echo "- No regression in existing MCP tool behavior" echo "- Generated task queue is ready or blockers are explicit" + if [[ -s "$tmp_steps" ]]; then + awk '{print "- " $0}' "$tmp_steps" + fi } | sed '/^[[:space:]]*$/N;/^\n$/D' - rm -f "$tmp_steps" "$tmp_constraints" + rm -f "$tmp_steps" "$tmp_constraints" "$tmp_refs" "$tmp_tools" } -MARKDOWN_CONTENT="$(cat "$SPRINT_FILE")" +MARKDOWN_CONTENT="$(cat "$INPUT_FILE")" INTAKE_ARGS="$(jq -nc --arg md "$MARKDOWN_CONTENT" '{markdown:$md}')" INTAKE_RESP_RAW="$(call_tool "whetstone_architect_intake" "$INTAKE_ARGS")" printf '%s\n' "$INTAKE_RESP_RAW" > "$OUT_DIR/01_intake_raw.ndjson.json" INTAKE_JSON="$(extract_tool_text_json "$INTAKE_RESP_RAW")" printf '%s\n' "$INTAKE_JSON" > "$OUT_DIR/01_intake.json" +EFFECTIVE_INTAKE_JSON_PATH="$OUT_DIR/01_intake.json" if [[ "$(printf '%s' "$INTAKE_JSON" | jq -r '.success // false')" != "true" ]] && [[ "$(printf '%s' "$INTAKE_JSON" | jq -r '.error // ""')" == "no_requirements_found" ]]; then - FALLBACK_SPEC="$(build_fallback_intake_spec "$SPRINT_FILE")" + FALLBACK_SPEC="$(build_fallback_intake_spec "$INPUT_FILE")" printf '%s\n' "$FALLBACK_SPEC" > "$OUT_DIR/01a_fallback_intake_spec.md" INTAKE_ARGS="$(jq -nc --arg md "$FALLBACK_SPEC" '{markdown:$md}')" INTAKE_RESP_RAW="$(call_tool "whetstone_architect_intake" "$INTAKE_ARGS")" printf '%s\n' "$INTAKE_RESP_RAW" > "$OUT_DIR/01b_intake_retry_raw.ndjson.json" INTAKE_JSON="$(extract_tool_text_json "$INTAKE_RESP_RAW")" printf '%s\n' "$INTAKE_JSON" > "$OUT_DIR/01b_intake_retry.json" + EFFECTIVE_INTAKE_JSON_PATH="$OUT_DIR/01b_intake_retry.json" fi if [[ "$(printf '%s' "$INTAKE_JSON" | jq -r '.success // false')" != "true" ]]; then @@ -121,7 +197,8 @@ fi NORMALIZED_REQS="$(printf '%s' "$INTAKE_JSON" | jq '.normalizedRequirements')" CONFLICTS="$(printf '%s' "$INTAKE_JSON" | jq '.conflicts // []')" -GEN_ARGS="$(jq -nc --argjson nr "$NORMALIZED_REQS" --argjson cf "$CONFLICTS" '{normalizedRequirements:$nr,conflicts:$cf}')" +GEN_ARGS="$(jq -nc --argjson nr "$NORMALIZED_REQS" --argjson cf "$CONFLICTS" --arg strict "$STRICT_EXECUTION_CONTRACT" \ + '{normalizedRequirements:$nr,conflicts:$cf,strictExecutionContract:($strict == "1")}')" GEN_RESP_RAW="$(call_tool "whetstone_generate_taskitems" "$GEN_ARGS")" printf '%s\n' "$GEN_RESP_RAW" > "$OUT_DIR/02_generate_taskitems_raw.ndjson.json" GEN_JSON="$(extract_tool_text_json "$GEN_RESP_RAW")" @@ -132,41 +209,49 @@ if [[ "$(printf '%s' "$GEN_JSON" | jq -r '.success // false')" != "true" ]]; the fi TASKS="$(printf '%s' "$GEN_JSON" | jq '.tasks')" -QUEUE_ARGS="$(jq -nc --argjson t "$TASKS" --argjson nr "$NORMALIZED_REQS" '{tasks:$t,normalizedRequirements:$nr}')" +QUEUE_ARGS="$(jq -nc --argjson t "$TASKS" --argjson nr "$NORMALIZED_REQS" --arg strict "$STRICT_EXECUTION_CONTRACT" --argjson cs "$CAPABILITY_SIGNALS_JSON" \ + '{tasks:$t,normalizedRequirements:$nr,strictExecutionContract:($strict == "1"),capabilitySignals:$cs}')" QUEUE_RESP_RAW="$(call_tool "whetstone_queue_ready" "$QUEUE_ARGS")" printf '%s\n' "$QUEUE_RESP_RAW" > "$OUT_DIR/03_queue_ready_raw.ndjson.json" QUEUE_JSON="$(extract_tool_text_json "$QUEUE_RESP_RAW")" printf '%s\n' "$QUEUE_JSON" > "$OUT_DIR/03_queue_ready.json" -VALIDATE_TASKS="$(printf '%s' "$TASKS" | jq '[.[] | { +VALIDATE_TASKS="$(printf '%s' "$TASKS" | jq --argjson cs "$CAPABILITY_SIGNALS_JSON" '[.[] | { task_id: .taskId, title: .title, prerequisite_ops: (.prerequisiteOps // []), reasons: (.reasons // []), confidence: (.confidence // 0), - dependency_task_ids: (.dependencyTaskIds // []) + dependency_task_ids: (.dependencyTaskIds // []), + resource_locks: (.resourceLocks // []), + execution_contract: (.executionContract // {}), + capability_signals: $cs }]')" -VALIDATE_ARGS="$(jq -nc --argjson items "$VALIDATE_TASKS" --arg ws "$WORKSPACE" '{taskitems:$items,workspace:$ws}')" +VALIDATE_ARGS="$(jq -nc --argjson items "$VALIDATE_TASKS" --arg ws "$WORKSPACE" --arg strict "$STRICT_EXECUTION_CONTRACT" \ + '{taskitems:$items,workspace:$ws,strict_execution_contract:($strict == "1")}')" VALIDATE_RESP_RAW="$(call_tool "whetstone_validate_taskitem" "$VALIDATE_ARGS")" printf '%s\n' "$VALIDATE_RESP_RAW" > "$OUT_DIR/04_validate_taskitem_raw.ndjson.json" VALIDATE_JSON="$(extract_tool_text_json "$VALIDATE_RESP_RAW")" printf '%s\n' "$VALIDATE_JSON" > "$OUT_DIR/04_validate_taskitem.json" SUMMARY_JSON="$(jq -nc \ - --arg sprint "$SPRINT_FILE_REL" \ + --arg input_file "$INPUT_FILE_LABEL" \ --arg outDir "$OUT_DIR" \ --arg bin "$BIN" \ --arg workspace "$WORKSPACE" \ + --argjson planning_readiness "$SPEC_READINESS_JSON" \ --argjson intake "$INTAKE_JSON" \ --argjson generated "$GEN_JSON" \ --argjson queue "$QUEUE_JSON" \ --argjson validate "$VALIDATE_JSON" \ '{ - sprint: $sprint, + sprint: $input_file, + input_file: $input_file, timestamp: now|todate, mcp_binary: $bin, workspace: $workspace, output_dir: $outDir, + planning_readiness: $planning_readiness, intake: { success: ($intake.success // false), normalized_requirement_count: (($intake.normalizedRequirements // [])|length), @@ -176,22 +261,66 @@ SUMMARY_JSON="$(jq -nc \ taskitems: { success: ($generated.success // false), task_count: (($generated.tasks // [])|length), - escalate_count: ($generated.escalateCount // 0) + escalate_count: ($generated.escalateCount // 0), + strict_execution_contract: ($generated.strictExecutionContract // false), + missing_execution_contract_count: ($generated.missingExecutionContractCount // 0) }, queue_ready: { success: ($queue.success // false), ready: ($queue.ready // false), ready_count: ($queue.readyCount // 0), - blocker_count: (($queue.blockers // [])|length) + blocker_count: (($queue.blockers // [])|length), + queue_warnings: ($queue.queueWarnings // []), + resource_conflict_count: ($queue.resourceConflictCount // 0), + strict_execution_contract: ($queue.strictExecutionContract // false), + execution_contract_missing_count: ($queue.executionContractMissingCount // 0), + execution_specificity_low_count: ($queue.executionSpecificityLowCount // 0), + gap_class_counts: ($queue.gap_class_counts // {}), + remediation_routing_hints: ($queue.remediationRoutingHints // {}), + debug_loop_call_plan: ($queue.debugLoopCallPlan // {}) }, validation: { success: ($validate.success // false), total_taskitems: ($validate.report.total_taskitems // 0), average_score: ($validate.report.average_score // 0), + average_execution_specificity_score: ($validate.report.average_execution_specificity_score // 0), self_contained_count: ($validate.report.self_contained_count // 0), - failing_count: ($validate.report.failing_count // 0) + failing_count: ($validate.report.failing_count // 0), + gap_class_counts: ($validate.report.gap_class_counts // {}), + top_recommended_tools: ($validate.report.top_recommended_tools // []), + capability_gap_call_plan: ($validate.report.capability_gap_call_plan // {}), + promotion_packet: ($validate.report.promotion_packet // {}) } }')" + +if [[ "$CALIBRATE_AFTER_RUN" == "1" ]]; then + CALIBRATION_OUT_DIR="$OUT_DIR/calibration" + CALIBRATION_JSON="$(python3 "$ROOT_DIR/tools/mcp/analyze_taskitem_calibration.py" \ + --runs-root "$ROOT_DIR/logs/taskitem_runs" \ + --out-dir "$CALIBRATION_OUT_DIR")" + if [[ -n "$CALIBRATION_JSON" ]] && printf '%s' "$CALIBRATION_JSON" | jq -e . >/dev/null 2>&1; then + SUMMARY_JSON="$(printf '%s' "$SUMMARY_JSON" | jq --argjson calibration "$CALIBRATION_JSON" '.calibration = $calibration')" + fi +fi + +# Persist current summary before optional post-check suites that read summary fields. +printf '%s\n' "$SUMMARY_JSON" > "$OUT_DIR/00_summary.json" + +if [[ "$RUN_READINESS_SUITE" == "1" ]]; then + READINESS_JSON="$(python3 "$ROOT_DIR/tools/mcp/run_generator_readiness_suite.py" \ + --repo-root "$ROOT_DIR" \ + --out-dir "$OUT_DIR" \ + --summary-json "$OUT_DIR/00_summary.json" \ + --intake-json "$EFFECTIVE_INTAKE_JSON_PATH" \ + --generated-json "$OUT_DIR/02_generate_taskitems.json" \ + --queue-json "$OUT_DIR/03_queue_ready.json" \ + --validate-json "$OUT_DIR/04_validate_taskitem.json" \ + --input-file "$INPUT_FILE")" + if [[ -n "$READINESS_JSON" ]] && printf '%s' "$READINESS_JSON" | jq -e . >/dev/null 2>&1; then + SUMMARY_JSON="$(printf '%s' "$SUMMARY_JSON" | jq --argjson readiness "$READINESS_JSON" '.readiness = $readiness')" + fi +fi + printf '%s\n' "$SUMMARY_JSON" > "$OUT_DIR/00_summary.json" echo "Taskitem pipeline complete."