From 461a702e114bda759d9abef2bb44606453ad1a17 Mon Sep 17 00:00:00 2001 From: Bill Date: Mon, 16 Feb 2026 12:50:32 -0700 Subject: [PATCH] Step 383: add Sprint 15a integration tests --- editor/CMakeLists.txt | 9 ++ editor/tests/step383_test.cpp | 222 ++++++++++++++++++++++++++++++++++ progress.md | 44 +++++++ 3 files changed, 275 insertions(+) create mode 100644 editor/tests/step383_test.cpp diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 8387e6b..3bb4372 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -2326,4 +2326,13 @@ target_link_libraries(step382_test PRIVATE tree_sitter_javascript tree_sitter_typescript tree_sitter_java tree_sitter_rust tree_sitter_go) +add_executable(step383_test tests/step383_test.cpp) +target_include_directories(step383_test PRIVATE src) +target_link_libraries(step383_test PRIVATE + nlohmann_json::nlohmann_json + unofficial::tree-sitter::tree-sitter + tree_sitter_python tree_sitter_cpp tree_sitter_elisp + tree_sitter_javascript tree_sitter_typescript + tree_sitter_java tree_sitter_rust tree_sitter_go) + # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies) diff --git a/editor/tests/step383_test.cpp b/editor/tests/step383_test.cpp new file mode 100644 index 0000000..0297b49 --- /dev/null +++ b/editor/tests/step383_test.cpp @@ -0,0 +1,222 @@ +// Step 383: Phase 15a integration tests (8 tests) + +#include +#include +#include +#include "HeadlessEditorState.h" +#include "HeadlessAgentRPCHandler.h" +#include "MCPServer.h" +#include "WorkflowOrchestrator.h" + +static WorkItem makeItem(const std::string& id, + const std::string& name, + const std::string& workerType = "", + const std::string& contextWidth = "local", + bool reviewRequired = false) { + WorkItem item; + item.id = id; + item.nodeId = id + "_node"; + item.nodeName = name; + item.nodeType = "Function"; + item.bufferId = "main.py"; + item.workerType = workerType; + item.contextWidth = contextWidth; + item.reviewRequired = reviewRequired; + item.priority = "medium"; + item.status = WI_PENDING; + item.createdAt = workItemTimestamp(); + return item; +} + +static HeadlessEditorState makeState() { + HeadlessEditorState state; + state.defaultLanguage = "python"; + state.openBuffer("main.py", "def a():\n return 1\n", "python"); + state.setAgentRole("test-session", AgentRole::Generator); + state.workflow = WorkflowState("phase15a"); + return state; +} + +static json rpc(HeadlessEditorState& state, const std::string& method, + const json& params = json::object()) { + json request = {{"jsonrpc", "2.0"}, {"id", 1}, {"method", method}, + {"params", params}}; + return handleHeadlessAgentRequest(state, request, "test-session"); +} + +int main() { + int passed = 0; + + // Test 1: mixed 5-item workflow reaches partial deterministic completion + { + auto state = makeState(); + state.workflow->queue.enqueue(makeItem("d1", "getName", "template")); + state.workflow->queue.enqueue(makeItem("d2", "setValue", "template")); + state.workflow->queue.enqueue(makeItem("a1", "planGraph", "", "project")); + state.workflow->queue.enqueue(makeItem("a2", "buildIndex", "", "project")); + state.workflow->queue.enqueue(makeItem("h1", "manualAudit", "human")); + + auto resp = rpc(state, "orchestrateRunDeterministic"); + assert(resp.contains("result")); + auto stats = state.workflow->getStats(); + assert(stats.complete == 2); + assert(stats.inProgress >= 2); // prepared agent/human tasks + std::cout << "Test 1 PASSED: mixed workflow partial completion profile\n"; + passed++; + } + + // Test 2: dependency chain respected + { + auto state = makeState(); + WorkItem a = makeItem("a", "getA", "template"); + WorkItem b = makeItem("b", "getB", "template"); + b.dependencies.push_back("a"); + state.workflow->queue.enqueue(a); + state.workflow->queue.enqueue(b); + + auto first = rpc(state, "orchestrateAdvance"); + assert(first["result"]["itemsAdvanced"].get() == 1); + auto second = rpc(state, "orchestrateAdvance"); + assert(second["result"]["itemsAdvanced"].get() == 1); + std::cout << "Test 2 PASSED: dependency gating respected\n"; + passed++; + } + + // Test 3: rejection escalation deterministic -> slm -> llm + { + auto state = makeState(); + state.workflow->queue.enqueue(makeItem("r1", "getRisky", "template", "local", true)); + WorkflowOrchestrator orchestrator(*state.workflow, state.routingEngine, + state.workerRegistry, state.contextAssembler, + state.reviewGate); + orchestrator.setBuffers({}); + orchestrator.setReviewPolicy(state.reviewPolicy); + orchestrator.step(); // sent to review + assert(orchestrator.rejectAndRequeue("r1", "attempt1")); + orchestrator.step(); // slm blocked + assert(orchestrator.rejectAndRequeue("r1", "attempt2")); + orchestrator.step(); // llm blocked + auto item = state.workflow->queue.getItem("r1"); + assert(item.has_value()); + assert(item->workerType == "llm"); + std::cout << "Test 3 PASSED: rejection escalation reaches llm\n"; + passed++; + } + + // Test 4: shared project-context optimization yields token savings + { + auto state = makeState(); + state.workflow->queue.enqueue(makeItem("p1", "task1", "", "project")); + state.workflow->queue.enqueue(makeItem("p2", "task2", "", "project")); + state.workflow->queue.enqueue(makeItem("p3", "task3", "", "project")); + auto resp = rpc(state, "orchestrateAdvance"); + assert(resp["result"]["contextTokensSaved"].get() > 0); + std::cout << "Test 4 PASSED: shared-context optimization active\n"; + passed++; + } + + // Test 5: progress snapshot reflects orchestrator events + { + auto state = makeState(); + state.workflow->queue.enqueue(makeItem("d1", "getFast", "template")); + state.workflow->queue.enqueue(makeItem("d2", "setFast", "template")); + rpc(state, "orchestrateRunDeterministic"); + auto progress = rpc(state, "getProgress"); + assert(progress.contains("result")); + assert(progress["result"]["completedItems"].get() == 2); + assert(progress["result"]["completionPercent"].get() > 0.0f); + std::cout << "Test 5 PASSED: progress tracking updates with orchestration\n"; + passed++; + } + + // Test 6: submitExternalResult can complete an agent-prepared task + { + auto state = makeState(); + state.reviewPolicy.defaultAction = "auto-approve"; + state.workflow->queue.enqueue(makeItem("x1", "buildGraph", "", "project")); + rpc(state, "orchestrateStep"); // prepare llm task + auto submit = rpc(state, "submitExternalResult", { + {"itemId", "x1"}, + {"result", { + {"generatedCode", "def buildGraph(d):\n return d"}, + {"confidence", 0.9}, + {"reasoning", "model output"}, + {"tokensGenerated", 20} + }} + }); + assert(submit.contains("result")); + auto item = state.workflow->queue.getItem("x1"); + assert(item.has_value()); + assert(item->status == WI_COMPLETE); + std::cout << "Test 6 PASSED: external submission closes agent task\n"; + passed++; + } + + // Test 7: blockers include both external-model and human categories + { + auto state = makeState(); + state.workflow->queue.enqueue(makeItem("a1", "projectTask", "", "project")); + state.workflow->queue.enqueue(makeItem("h1", "manualTask", "human")); + rpc(state, "orchestrateRunDeterministic"); + auto blockers = rpc(state, "getBlockers"); + assert(blockers.contains("result")); + bool hasHuman = false; + bool hasExternal = false; + for (const auto& b : blockers["result"]["blockers"]) { + if (b["type"] == "needs-human") hasHuman = true; + if (b["type"] == "needs-external-model") hasExternal = true; + } + assert(hasHuman && hasExternal); + std::cout << "Test 7 PASSED: blocker categories surfaced\n"; + passed++; + } + + // Test 8: MCP orchestration loop integration (run -> submit -> progress) + { + auto state = makeState(); + state.reviewPolicy.defaultAction = "auto-approve"; + state.workflow->queue.enqueue(makeItem("d1", "getOne", "template")); + state.workflow->queue.enqueue(makeItem("x1", "genTwo", "", "project")); + + MCPServer server; + server.setRpcCallback([&state](const json& request) { + return handleHeadlessAgentRequest(state, request, "test-session"); + }); + + auto runResp = server.handleRequest({ + {"jsonrpc", "2.0"}, + {"id", 1}, + {"method", "tools/call"}, + {"params", {{"name", "whetstone_orchestrate_run_deterministic"}, + {"arguments", json::object()}}} + }); + assert(runResp["result"]["isError"].get() == false); + + server.handleRequest({ + {"jsonrpc", "2.0"}, + {"id", 2}, + {"method", "tools/call"}, + {"params", {{"name", "whetstone_submit_result"}, + {"arguments", {{"itemId", "x1"}, + {"result", {{"generatedCode", "def genTwo():\n return 2"}, + {"confidence", 0.93}, + {"reasoning", "llm"}, + {"tokensGenerated", 16}}}}}}} + }); + + auto progResp = server.handleRequest({ + {"jsonrpc", "2.0"}, + {"id", 3}, + {"method", "tools/call"}, + {"params", {{"name", "whetstone_get_progress"}, + {"arguments", json::object()}}} + }); + assert(progResp["result"]["isError"].get() == false); + std::cout << "Test 8 PASSED: MCP orchestrator end-to-end path\n"; + passed++; + } + + std::cout << "\nResults: " << passed << "/8\n"; + assert(passed == 8); + return 0; +} diff --git a/progress.md b/progress.md index 3f41991..11215e0 100644 --- a/progress.md +++ b/progress.md @@ -2978,6 +2978,50 @@ submit external model output back into the workflow loop. - `editor/src/HeadlessAgentRPCHandler.h` (`2623` > `600`) - `editor/src/MCPServer.h` (`1652` > `600`) +### Step 383: Phase 15a Integration Tests +**Status:** PASS (8/8 tests) + +Added end-to-end integration validation for Sprint 15a orchestration behavior: +mixed deterministic/agent/human flows, dependency gating, rejection escalation, +shared-context optimization, progress tracking, external-result submission, and +MCP orchestration tool loop closure. + +**Files created:** +- `editor/tests/step383_test.cpp` — 8 integration tests covering: + 1. mixed 5-item workflow partial deterministic completion profile + 2. dependency-chain progression constraints + 3. rejection escalation through deterministic -> slm -> llm + 4. shared project-context token-savings path + 5. progress snapshot updates during orchestration + 6. `submitExternalResult` completion path for agent-prepared task + 7. blocker-category surfacing (`needs-human`, `needs-external-model`) + 8. MCP end-to-end loop (`run_deterministic` -> `submit_result` -> `get_progress`) + +**Files modified:** +- `editor/CMakeLists.txt` — `step383_test` target + +**Verification run:** +- `step383_test` — PASS (8/8) new integration coverage +- `step382_test` — PASS (12/12) regression coverage +- `step381_test` — PASS (12/12) regression coverage +- `step380_test` — PASS (12/12) regression coverage + +**Sprint 15a completion snapshot (Steps 378-383):** +- Step 378: orchestrator core loop +- Step 379: batching + shared-context optimization +- Step 380: rejection rerouting feedback loop +- Step 381: progress/ETA tracking +- Step 382: orchestrator RPC + MCP tools +- Step 383: phase integration coverage + +**Architecture gate check (end of Sprint 15a):** +- `editor/tests/step383_test.cpp` within test-file size guidance (`222` lines) +- `editor/src/HeadlessOrchestratorRPC.h` within header-size limit (`287` <= `600`) +- `editor/src/WorkflowProgress.h` within header-size limit (`303` <= `600`) +- Legacy oversized headers persist and need planned split before Sprint 15b+: + - `editor/src/HeadlessAgentRPCHandler.h` (`2623` > `600`) + - `editor/src/MCPServer.h` (`1652` > `600`) + # Roadmap Planning — Sprints 12-25+ ## Status: Planning Complete (Sprints 12-19 detailed, 20-25 in roadmap.md)