From 6784281489b419221a5e4b14a9b082357e5a4713 Mon Sep 17 00:00:00 2001 From: Bill Date: Sun, 15 Feb 2026 19:27:29 -0700 Subject: [PATCH] =?UTF-8?q?Step=20331:=20Phase=2012b=20Integration=20Tests?= =?UTF-8?q?=20=E2=80=94=20full=20routing=20pipeline=20(8/8=20tests)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 8 integration tests covering skeleton→workflow→route→execute→review lifecycle: mixed worker routing, getter auto-approve, LLM context, human routing, dependency chains, rejection flow, review policy, 5-function mixed skeleton. Co-Authored-By: Claude Opus 4.6 --- editor/CMakeLists.txt | 10 ++ editor/tests/step331_test.cpp | 303 ++++++++++++++++++++++++++++++++++ progress.md | 21 +++ 3 files changed, 334 insertions(+) create mode 100644 editor/tests/step331_test.cpp diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 2cfd2a1..0b93a6c 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -1986,4 +1986,14 @@ add_executable(step330_test tests/step330_test.cpp) target_include_directories(step330_test PRIVATE src) target_link_libraries(step330_test PRIVATE nlohmann_json::nlohmann_json) +# Step 331: Phase 12b Integration Tests +add_executable(step331_test tests/step331_test.cpp) +target_include_directories(step331_test PRIVATE src) +target_link_libraries(step331_test PRIVATE + nlohmann_json::nlohmann_json + unofficial::tree-sitter::tree-sitter + tree_sitter_python tree_sitter_cpp tree_sitter_elisp + tree_sitter_javascript tree_sitter_typescript + tree_sitter_java tree_sitter_rust tree_sitter_go) + # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies) diff --git a/editor/tests/step331_test.cpp b/editor/tests/step331_test.cpp new file mode 100644 index 0000000..9743578 --- /dev/null +++ b/editor/tests/step331_test.cpp @@ -0,0 +1,303 @@ +// Step 331: Phase 12b Integration Tests (8 tests) +// Full routing pipeline from skeleton through execution and review. + +#include "HeadlessEditorState.h" +#include "HeadlessAgentRPCHandler.h" +#include "ReviewGate.h" +#include +#include +#include + +using json = nlohmann::json; + +static int passed = 0, failed = 0; +#define TEST(name) { std::cout << " " << #name << "... "; } +#define PASS() { std::cout << "PASS\n"; ++passed; } +#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; } +#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {} + +static json rpcCall(HeadlessEditorState& state, const std::string& method, + const json& params = json::object()) { + json req = {{"jsonrpc", "2.0"}, {"id", 1}, {"method", method}}; + if (!params.empty()) req["params"] = params; + return handleHeadlessAgentRequest(state, req, "test-session"); +} + +static HeadlessEditorState makeState() { + HeadlessEditorState state; + state.defaultLanguage = "python"; + state.setAgentRole("test-session", AgentRole::Generator); + return state; +} + +static void setupSkeleton(HeadlessEditorState& state, Module* mod) { + state.openBuffer("test", "# skeleton", "python"); + state.activeBuffer->sync.setAST(std::unique_ptr(mod)); +} + +static void advanceToInProgress(HeadlessEditorState& state, const std::string& itemId) { + rpcCall(state, "assignTask", {{"itemId", itemId}, {"assignee", "worker"}}); + auto item = *state.workflow->queue.getItem(itemId); + transitionWorkItem(item, WI_IN_PROGRESS); + state.workflow->queue.updateItem(itemId, item); +} + +// 1. Skeleton → workflow → route all → verify worker types +void test_skeleton_route_all() { + TEST(skeleton_route_all); + auto state = makeState(); + auto* mod = createSkeletonModule("proj", "python"); + addSkeletonFunction(mod, "getName", {}, "str"); + addSkeletonFunction(mod, "processData", {"x"}, "int"); + addSkeletonFunction(mod, "validate", {"data"}, "bool"); + setupSkeleton(state, mod); + + rpcCall(state, "createWorkflow", {{"projectName", "proj"}}); + auto resp = rpcCall(state, "routeAllReady"); + CHECK(resp["result"]["routed"].get() == 3, "3 routed"); + + // Check that worker types were assigned + auto items = rpcCall(state, "getReadyTasks")["result"]["items"]; + bool hasTemplate = false, hasSlm = false; + for (const auto& item : items) { + std::string wt = item["workerType"].get(); + if (wt == "template") hasTemplate = true; + if (wt == "slm") hasSlm = true; + } + CHECK(hasTemplate, "has template-routed item (getter)"); + CHECK(hasSlm, "has slm-routed item"); + PASS(); +} + +// 2. Getter → route → template → execute → auto-approve +void test_getter_full_lifecycle() { + TEST(getter_full_lifecycle); + auto state = makeState(); + auto* mod = createSkeletonModule("proj", "python"); + addSkeletonFunction(mod, "getValue", {}, "int"); + setupSkeleton(state, mod); + + rpcCall(state, "createWorkflow", {{"projectName", "proj"}}); + rpcCall(state, "routeAllReady"); + + auto items = rpcCall(state, "getReadyTasks")["result"]["items"]; + std::string itemId = items[0]["id"].get(); + + auto execResp = rpcCall(state, "executeTask", {{"itemId", itemId}}); + CHECK(execResp["result"]["workerType"] == "template", "template worker"); + CHECK(execResp["result"]["autoApproved"].get(), "auto-approved"); + + // Check result has getter code + auto code = execResp["result"]["result"]["generatedCode"].get(); + CHECK(code.find("self.value") != std::string::npos, "getter code: " + code); + PASS(); +} + +// 3. Complex function → route → LLM → prepared context +void test_complex_llm_context() { + TEST(complex_llm_context); + auto state = makeState(); + auto* mod = createSkeletonModule("proj", "python"); + auto* fn = addSkeletonFunction(mod, "buildGraph", {"data"}, "Graph"); + // Set project context width + auto* cwAnno = new ContextWidthAnnotation(); + cwAnno->id = "cw1"; cwAnno->width = "project"; + fn->addChild("annotations", cwAnno); + setupSkeleton(state, mod); + + rpcCall(state, "createWorkflow", {{"projectName", "proj"}}); + rpcCall(state, "routeAllReady"); + + auto items = rpcCall(state, "getReadyTasks")["result"]["items"]; + std::string itemId = items[0]["id"].get(); + CHECK(items[0]["workerType"] == "llm", "routed to llm"); + + auto execResp = rpcCall(state, "executeTask", {{"itemId", itemId}}); + CHECK(execResp["result"]["result"]["confidence"].get() == 0.0f, + "deferred confidence"); + PASS(); +} + +// 4. High ambiguity → human → marked for review +void test_high_ambiguity_human() { + TEST(high_ambiguity_human); + auto state = makeState(); + auto* mod = createSkeletonModule("proj", "python"); + auto* fn = addSkeletonFunction(mod, "handleEdgeCase", {"input"}, "Result"); + // Set automatability to human + auto* autoAnno = new AutomatabilityAnnotation(); + autoAnno->id = "aa1"; autoAnno->strategy = "human"; + fn->addChild("annotations", autoAnno); + setupSkeleton(state, mod); + + rpcCall(state, "createWorkflow", {{"projectName", "proj"}}); + rpcCall(state, "routeAllReady"); + + auto items = rpcCall(state, "getReadyTasks")["result"]["items"]; + CHECK(items[0]["workerType"] == "human", "routed to human"); + + auto execResp = rpcCall(state, "executeTask", {{"itemId", items[0]["id"].get()}}); + CHECK(execResp["result"]["autoApproved"].get() == false, "not auto-approved"); + PASS(); +} + +// 5. Dependency chain with mixed routing +void test_dependency_mixed_routing() { + TEST(dependency_mixed_routing); + auto state = makeState(); + state.openBuffer("test", "# deps", "python"); + auto* mod = createSkeletonModule("deps", "python"); + addSkeletonFunction(mod, "getName", {}, "str"); // template + state.activeBuffer->sync.setAST(std::unique_ptr(mod)); + + state.workflow = WorkflowState("deps"); + auto makeWI = [](const std::string& id, const std::string& name, + const std::string& wtype, const std::vector& deps) { + WorkItem wi; + wi.id = id; wi.nodeId = "n_" + id; wi.nodeName = name; + wi.nodeType = "Function"; wi.workerType = wtype; + wi.priority = "high"; wi.status = WI_PENDING; + wi.contextWidth = "local"; wi.dependencies = deps; + wi.createdAt = workItemTimestamp(); + return wi; + }; + + state.workflow->queue.enqueue(makeWI("A", "getName", "template", {})); + state.workflow->queue.enqueue(makeWI("B", "process", "slm", {"A"})); + state.workflow->queue.enqueue(makeWI("C", "review", "human", {"B"})); + + CHECK(state.workflow->queue.readyCount() == 1, "only A ready"); + + // Execute A (template) + auto execA = rpcCall(state, "executeTask", {{"itemId", "A"}}); + CHECK(execA["result"]["workerType"] == "template", "A=template"); + + // Complete A → B ready + advanceToInProgress(state, "A"); + rpcCall(state, "completeTask", {{"itemId", "A"}}); + CHECK(state.workflow->queue.readyCount() == 1, "B ready"); + + // Execute B (slm) + auto execB = rpcCall(state, "executeTask", {{"itemId", "B"}}); + CHECK(execB["result"]["workerType"] == "slm", "B=slm"); + + // Complete B → C ready + advanceToInProgress(state, "B"); + rpcCall(state, "completeTask", {{"itemId", "B"}}); + CHECK(state.workflow->queue.readyCount() == 1, "C ready"); + PASS(); +} + +// 6. Rejection flow with feedback +void test_rejection_with_feedback() { + TEST(rejection_with_feedback); + auto state = makeState(); + auto* mod = createSkeletonModule("rej", "python"); + addSkeletonFunction(mod, "transform", {"data"}, "Result"); + setupSkeleton(state, mod); + + rpcCall(state, "createWorkflow", {{"projectName", "rej"}}); + rpcCall(state, "routeAllReady"); + + auto items = rpcCall(state, "getReadyTasks")["result"]["items"]; + std::string itemId = items[0]["id"].get(); + + // Execute → assign → in-progress → review → reject + rpcCall(state, "assignTask", {{"itemId", itemId}}); + auto item = *state.workflow->queue.getItem(itemId); + transitionWorkItem(item, WI_IN_PROGRESS); + state.workflow->queue.updateItem(itemId, item); + item = *state.workflow->queue.getItem(itemId); + transitionWorkItem(item, WI_REVIEW); + state.workflow->queue.updateItem(itemId, item); + + rpcCall(state, "rejectTask", {{"itemId", itemId}, {"reason", "missing edge cases"}}); + + // Item back in ready queue + auto ready = rpcCall(state, "getReadyTasks")["result"]["items"]; + CHECK(ready.size() == 1, "back in queue"); + CHECK(ready[0]["id"] == itemId, "same item"); + PASS(); +} + +// 7. Review policy: strict → permissive +void test_review_policy_switch() { + TEST(review_policy_switch); + ReviewGate gate; + + WorkItem wi; + wi.id = "t1"; wi.workerType = "deterministic"; wi.reviewRequired = false; + WorkItemResult result; + result.confidence = 0.95f; + + // Strict: no rules → require review + ReviewPolicy strict; + strict.defaultAction = "require-review"; + auto d1 = gate.shouldAutoApprove(wi, result, strict); + CHECK(!d1.approved, "strict: not approved"); + + // Permissive: default auto-approve + ReviewPolicy permissive; + permissive.defaultAction = "auto-approve"; + auto d2 = gate.shouldAutoApprove(wi, result, permissive); + CHECK(d2.approved, "permissive: approved"); + PASS(); +} + +// 8. Full lifecycle: 5-function mixed skeleton +void test_full_lifecycle_mixed() { + TEST(full_lifecycle_mixed); + auto state = makeState(); + auto* mod = createSkeletonModule("full", "python"); + addSkeletonFunction(mod, "getName", {}, "str"); // template + addSkeletonFunction(mod, "setValue", {"v"}, "void"); // template + addSkeletonFunction(mod, "processData", {"x"}, "int"); // slm + addSkeletonFunction(mod, "validate", {"d"}, "bool"); // slm + addSkeletonFunction(mod, "isActive", {}, "bool"); // template + setupSkeleton(state, mod); + + rpcCall(state, "createWorkflow", {{"projectName", "full"}}); + rpcCall(state, "routeAllReady"); + + // Count by worker type + auto items = rpcCall(state, "getReadyTasks")["result"]["items"]; + int templates = 0, slms = 0; + for (const auto& item : items) { + std::string wt = item["workerType"].get(); + if (wt == "template") templates++; + if (wt == "slm") slms++; + } + CHECK(templates == 3, "3 template items (getName, setValue, isActive), got " + + std::to_string(templates)); + CHECK(slms == 2, "2 slm items, got " + std::to_string(slms)); + + // Execute all template items (auto-approved) + for (const auto& item : items) { + if (item["workerType"] == "template") { + rpcCall(state, "executeTask", {{"itemId", item["id"].get()}}); + } + } + + auto stats = state.workflow->getStats(); + CHECK(stats.total == 5, "5 total"); + PASS(); +} + +int main() { + std::cout << "=== Step 331: Phase 12b Integration Tests ===\n"; + try { + test_skeleton_route_all(); + test_getter_full_lifecycle(); + test_complex_llm_context(); + test_high_ambiguity_human(); + test_dependency_mixed_routing(); + test_rejection_with_feedback(); + test_review_policy_switch(); + test_full_lifecycle_mixed(); + } catch (const std::exception& e) { + std::cout << "EXCEPTION: " << e.what() << "\n"; + ++failed; + } + std::cout << "\nResults: " << passed << "/" << (passed + failed) << " passed\n"; + return failed > 0 ? 1 : 0; +} diff --git a/progress.md b/progress.md index 27f5bbe..077b2a5 100644 --- a/progress.md +++ b/progress.md @@ -1446,6 +1446,27 @@ review. Explicit @Review(required) always overrides auto-approve. **Tool count:** 56+ (54 existing + 2 review tools) +### Step 331: Phase 12b Integration Tests +**Status:** PASS (8/8 tests) + +Full routing pipeline integration tests covering skeleton→workflow→route→execute→review +lifecycle. Tests verify mixed worker type routing, getter template auto-approve, +LLM context preparation, human routing for high-ambiguity, dependency chains with +mixed routing, rejection with feedback flow, review policy switching, and 5-function +mixed skeleton lifecycle. + +**Files created:** +- `editor/tests/step331_test.cpp` — 8 integration tests: skeleton_route_all, + getter_full_lifecycle, complex_llm_context, high_ambiguity_human, + dependency_mixed_routing, rejection_with_feedback, review_policy_switch, + full_lifecycle_mixed + +**Files modified:** +- `editor/CMakeLists.txt` — step331_test target with tree-sitter libraries + +**Phase 12b complete:** Steps 326-331 (RoutingEngine, WorkerRegistry, +ContextAssembler, Routing RPC+MCP, ReviewGate, Integration Tests) + --- # Roadmap Planning — Sprints 12-25+