From 67e637b9e8133b347ed4403200eb8655cde2b105 Mon Sep 17 00:00:00 2001 From: Bill Date: Mon, 16 Feb 2026 12:57:58 -0700 Subject: [PATCH] Step 386: add result acceptance protocol --- editor/CMakeLists.txt | 9 + editor/src/HeadlessOrchestratorRPC.h | 58 ++++-- editor/src/ResultAcceptance.h | 125 +++++++++++++ editor/tests/step386_test.cpp | 269 +++++++++++++++++++++++++++ progress.md | 44 +++++ 5 files changed, 487 insertions(+), 18 deletions(-) create mode 100644 editor/src/ResultAcceptance.h create mode 100644 editor/tests/step386_test.cpp diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 5fb9f70..3a2b8aa 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -2353,4 +2353,13 @@ target_link_libraries(step385_test PRIVATE tree_sitter_javascript tree_sitter_typescript tree_sitter_java tree_sitter_rust tree_sitter_go) +add_executable(step386_test tests/step386_test.cpp) +target_include_directories(step386_test PRIVATE src) +target_link_libraries(step386_test PRIVATE + nlohmann_json::nlohmann_json + unofficial::tree-sitter::tree-sitter + tree_sitter_python tree_sitter_cpp tree_sitter_elisp + tree_sitter_javascript tree_sitter_typescript + tree_sitter_java tree_sitter_rust tree_sitter_go) + # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies) diff --git a/editor/src/HeadlessOrchestratorRPC.h b/editor/src/HeadlessOrchestratorRPC.h index 4089a77..3aba931 100644 --- a/editor/src/HeadlessOrchestratorRPC.h +++ b/editor/src/HeadlessOrchestratorRPC.h @@ -2,6 +2,7 @@ // Step 382: Orchestrator RPC surface for headless agent requests. #include "HeadlessEditorState.h" +#include "ResultAcceptance.h" #include static inline json orchestratorRpcError(const json& id, int code, @@ -216,25 +217,34 @@ inline std::optional tryHandleHeadlessOrchestratorRPC( return orchestratorRpcError(id, -32602, "Work item not found"); if (!(item->workerType == "slm" || item->workerType == "llm")) return orchestratorRpcError(id, -32000, "submitExternalResult only supports slm/llm items"); - if (item->status != WI_IN_PROGRESS && item->status != WI_ASSIGNED) + if (item->status != WI_IN_PROGRESS && + item->status != WI_ASSIGNED && + item->status != WI_REVIEW) return orchestratorRpcError(id, -32000, - "Item must be assigned or in-progress for external submission"); + "Item must be assigned, in-progress, or review for external submission"); WorkItem updated = *item; if (updated.status == WI_ASSIGNED) transitionWorkItem(updated, WI_IN_PROGRESS); - - const auto& result = params["result"]; - updated.result.generatedCode = result.value("generatedCode", ""); - updated.result.confidence = result.value("confidence", 0.0f); - updated.result.reasoning = result.value("reasoning", ""); - updated.result.tokensGenerated = result.value("tokensGenerated", 0); - updated.result.tokensBudget = result.value("tokensBudget", - estimateContextBudget(updated.contextWidth)); - if (result.contains("astJson")) updated.result.astJson = result["astJson"]; - if (result.contains("diagnostics") && result["diagnostics"].is_array()) { - updated.result.diagnostics = result["diagnostics"].get>(); + if (updated.status == WI_REVIEW) { + updated.status = WI_IN_PROGRESS; } + ResultSubmission submission; + submission.itemId = itemId; + submission.generatedCode = params["result"].value("generatedCode", ""); + submission.confidence = params["result"].value("confidence", 0.0f); + submission.reasoning = params["result"].value("reasoning", ""); + if (params["result"].contains("suggestedAnnotations")) { + submission.suggestedAnnotations = + params["result"]["suggestedAnnotations"].get>(); + } + + std::string language = state.activeBuffer ? state.activeBuffer->language + : state.defaultLanguage; + WorkItemResult evaluatedResult; + ResultAcceptance acceptance = evaluateResultSubmission( + submission, updated, language, state.reviewGate, state.reviewPolicy, evaluatedResult); + updated.result = evaluatedResult; state.workflow->queue.updateItem(itemId, updated); std::vector events; @@ -247,12 +257,23 @@ inline std::optional tryHandleHeadlessOrchestratorRPC( workItemTimestamp() }); - ReviewDecision reviewDecision = - state.reviewGate.shouldAutoApprove(updated, updated.result, state.reviewPolicy); - if (reviewDecision.approved) { + if (!acceptance.validationPassed) { + WorkflowOrchestrator orchestrator(*state.workflow, state.routingEngine, + state.workerRegistry, state.contextAssembler, + state.reviewGate); + orchestrator.setReviewPolicy(state.reviewPolicy); + orchestrator.setBuffers(collectOrchestratorBufferInfos(state)); + std::string feedback = acceptance.validationErrors.empty() + ? "validation failed" + : acceptance.validationErrors.front(); + orchestrator.rejectAndRequeue(itemId, feedback, "validator"); + events.push_back({"rejected", itemId, + {{"reason", feedback}}, + workItemTimestamp()}); + } else if (acceptance.autoApproved) { state.workflow->queue.complete(itemId); events.push_back({"auto-approved", itemId, - {{"rule", reviewDecision.ruleMatched}}, + {{"rule", "acceptance-auto-approve"}}, workItemTimestamp()}); events.push_back({"completed", itemId, json::object(), workItemTimestamp()}); } else { @@ -264,7 +285,7 @@ inline std::optional tryHandleHeadlessOrchestratorRPC( transitionWorkItem(reviewItem, WI_REVIEW); state.workflow->queue.updateItem(itemId, reviewItem); events.push_back({"sent-to-review", itemId, - {{"reason", reviewDecision.reasoning}}, + {{"reason", "validation passed, awaiting review"}}, workItemTimestamp()}); } @@ -278,6 +299,7 @@ inline std::optional tryHandleHeadlessOrchestratorRPC( auto latest = state.workflow->queue.getItem(itemId); return orchestratorRpcResult(id, { {"success", true}, + {"acceptance", acceptance.toJson()}, {"events", orchestratorEventsToJson(events)}, {"item", latest ? workItemToJson(*latest) : json::object()} }); diff --git a/editor/src/ResultAcceptance.h b/editor/src/ResultAcceptance.h new file mode 100644 index 0000000..d7f69be --- /dev/null +++ b/editor/src/ResultAcceptance.h @@ -0,0 +1,125 @@ +#pragma once +// Step 386: External result acceptance protocol for workflow orchestration. + +#include "Pipeline.h" +#include "ReviewGate.h" +#include +#include + +struct ResultSubmission { + std::string itemId; + std::string generatedCode; + float confidence = 0.0f; + std::string reasoning; + std::vector suggestedAnnotations; + + static ResultSubmission fromJson(const json& j) { + ResultSubmission s; + s.itemId = j.value("itemId", ""); + s.generatedCode = j.value("generatedCode", ""); + s.confidence = j.value("confidence", 0.0f); + s.reasoning = j.value("reasoning", ""); + if (j.contains("suggestedAnnotations") && j["suggestedAnnotations"].is_array()) { + s.suggestedAnnotations = j["suggestedAnnotations"].get>(); + } + return s; + } +}; + +struct ResultAcceptance { + bool accepted = false; + bool validationPassed = false; + int diagnosticCount = 0; + bool autoApproved = false; + bool reviewRequired = true; + std::vector validationErrors; + + json toJson() const { + return { + {"accepted", accepted}, + {"validationPassed", validationPassed}, + {"diagnosticCount", diagnosticCount}, + {"autoApproved", autoApproved}, + {"reviewRequired", reviewRequired}, + {"validationErrors", validationErrors} + }; + } +}; + +inline std::vector validateSuggestedAnnotations( + const std::vector& suggestedAnnotations) { + std::vector errors; + for (size_t i = 0; i < suggestedAnnotations.size(); ++i) { + const auto& a = suggestedAnnotations[i]; + if (!a.is_object()) { + errors.push_back("Suggested annotation #" + std::to_string(i) + " must be an object"); + continue; + } + if (!a.contains("nodeId") || !a.contains("annotationType")) { + errors.push_back("Suggested annotation #" + std::to_string(i) + + " requires nodeId and annotationType"); + } + } + return errors; +} + +inline ResultAcceptance evaluateResultSubmission(const ResultSubmission& submission, + const WorkItem& item, + const std::string& language, + const ReviewGate& reviewGate, + const ReviewPolicy& reviewPolicy, + WorkItemResult& outResult) { + ResultAcceptance acceptance; + outResult = WorkItemResult{}; + outResult.generatedCode = submission.generatedCode; + outResult.confidence = submission.confidence; + outResult.reasoning = submission.reasoning; + outResult.tokensGenerated = static_cast(submission.generatedCode.size() / 4); + outResult.tokensBudget = estimateContextBudget(item.contextWidth); + + if (submission.generatedCode.empty()) { + acceptance.validationErrors.push_back("Generated code is empty"); + acceptance.validationPassed = false; + return acceptance; + } + + Pipeline pipeline; + std::vector parseDiags; + auto parsed = pipeline.parse(submission.generatedCode, + language.empty() ? "python" : language, + parseDiags); + for (const auto& d : parseDiags) { + acceptance.validationErrors.push_back( + "Parse error line " + std::to_string(d.line) + ": " + d.message); + } + acceptance.diagnosticCount = static_cast(parseDiags.size()); + + auto annotationErrors = validateSuggestedAnnotations(submission.suggestedAnnotations); + for (const auto& err : annotationErrors) { + acceptance.validationErrors.push_back(err); + } + acceptance.diagnosticCount += static_cast(annotationErrors.size()); + + if (!submission.suggestedAnnotations.empty()) { + outResult.astJson["suggestedAnnotations"] = submission.suggestedAnnotations; + } + if (parsed) { + outResult.astJson["parsed"] = true; + outResult.astJson["language"] = language.empty() ? "python" : language; + } + + if (!acceptance.validationErrors.empty() || !parsed) { + acceptance.validationPassed = false; + acceptance.accepted = false; + acceptance.reviewRequired = true; + return acceptance; + } + + acceptance.validationPassed = true; + ReviewDecision reviewDecision = + reviewGate.shouldAutoApprove(item, outResult, reviewPolicy); + acceptance.autoApproved = reviewDecision.approved; + acceptance.reviewRequired = !reviewDecision.approved; + acceptance.accepted = true; + return acceptance; +} diff --git a/editor/tests/step386_test.cpp b/editor/tests/step386_test.cpp new file mode 100644 index 0000000..5a1f803 --- /dev/null +++ b/editor/tests/step386_test.cpp @@ -0,0 +1,269 @@ +// Step 386: Result acceptance protocol (12 tests) + +#include +#include +#include +#include "HeadlessEditorState.h" +#include "HeadlessAgentRPCHandler.h" +#include "ResultAcceptance.h" + +static WorkItem makeAgentItem(const std::string& id, + const std::string& workerType = "llm") { + WorkItem item; + item.id = id; + item.nodeId = id + "_node"; + item.nodeName = "buildTask"; + item.nodeType = "Function"; + item.bufferId = "main.py"; + item.contextWidth = "project"; + item.workerType = workerType; + item.priority = "high"; + item.status = WI_IN_PROGRESS; + item.createdAt = workItemTimestamp(); + return item; +} + +static json rpc(HeadlessEditorState& state, const std::string& method, + const json& params = json::object()) { + json request = {{"jsonrpc", "2.0"}, {"id", 1}, {"method", method}, + {"params", params}}; + return handleHeadlessAgentRequest(state, request, "test-session"); +} + +static HeadlessEditorState makeState() { + HeadlessEditorState state; + state.defaultLanguage = "python"; + state.openBuffer("main.py", "def seed():\n return 1\n", "python"); + state.setAgentRole("test-session", AgentRole::Generator); + state.workflow = WorkflowState("accept"); + return state; +} + +int main() { + int passed = 0; + + // Test 1: valid code submission accepted + { + WorkItem item = makeAgentItem("a1"); + ReviewGate gate; + ReviewPolicy policy; + policy.defaultAction = "auto-approve"; + ResultSubmission sub; + sub.itemId = "a1"; + sub.generatedCode = "def buildTask(x):\n return x\n"; + sub.confidence = 0.9f; + WorkItemResult out; + auto acceptance = evaluateResultSubmission(sub, item, "python", gate, policy, out); + assert(acceptance.accepted); + assert(acceptance.validationPassed); + std::cout << "Test 1 PASSED: valid submission accepted\n"; + passed++; + } + + // Test 2: syntax error rejected with validation feedback + { + WorkItem item = makeAgentItem("a2"); + ReviewGate gate; + ReviewPolicy policy; + ResultSubmission sub; + sub.itemId = "a2"; + sub.generatedCode = "def broken(:\n"; + WorkItemResult out; + auto acceptance = evaluateResultSubmission(sub, item, "python", gate, policy, out); + assert(!acceptance.validationPassed); + assert(!acceptance.validationErrors.empty()); + std::cout << "Test 2 PASSED: syntax errors rejected\n"; + passed++; + } + + // Test 3: suggested annotation validation flags malformed entries + { + WorkItem item = makeAgentItem("a3"); + ReviewGate gate; + ReviewPolicy policy; + ResultSubmission sub; + sub.itemId = "a3"; + sub.generatedCode = "def ok():\n return 1\n"; + sub.suggestedAnnotations.push_back(json{{"foo", "bar"}}); + WorkItemResult out; + auto acceptance = evaluateResultSubmission(sub, item, "python", gate, policy, out); + assert(!acceptance.validationPassed); + assert(acceptance.diagnosticCount >= 1); + std::cout << "Test 3 PASSED: malformed suggested annotations flagged\n"; + passed++; + } + + // Test 4: diagnostics prevent auto-approval + { + WorkItem item = makeAgentItem("a4"); + ReviewGate gate; + ReviewPolicy policy; + policy.defaultAction = "auto-approve"; + ResultSubmission sub; + sub.itemId = "a4"; + sub.generatedCode = "def bad(:\n"; + WorkItemResult out; + auto acceptance = evaluateResultSubmission(sub, item, "python", gate, policy, out); + assert(!acceptance.autoApproved); + assert(!acceptance.accepted); + std::cout << "Test 4 PASSED: diagnostics block auto-approval\n"; + passed++; + } + + // Test 5: low-confidence valid result goes to review under strict policy + { + WorkItem item = makeAgentItem("a5"); + ReviewGate gate; + ReviewPolicy policy; + policy.defaultAction = "require-review"; + ResultSubmission sub; + sub.itemId = "a5"; + sub.generatedCode = "def ok(v):\n return v\n"; + sub.confidence = 0.1f; + WorkItemResult out; + auto acceptance = evaluateResultSubmission(sub, item, "python", gate, policy, out); + assert(acceptance.accepted); + assert(acceptance.reviewRequired); + std::cout << "Test 5 PASSED: low-confidence result routed to review\n"; + passed++; + } + + // Test 6: suggested annotations attached to output result payload + { + WorkItem item = makeAgentItem("a6"); + ReviewGate gate; + ReviewPolicy policy; + ResultSubmission sub; + sub.itemId = "a6"; + sub.generatedCode = "def ok():\n return 1\n"; + sub.suggestedAnnotations.push_back( + json{{"nodeId", "n1"}, {"annotationType", "Intent"}}); + WorkItemResult out; + auto acceptance = evaluateResultSubmission(sub, item, "python", gate, policy, out); + assert(acceptance.validationPassed); + assert(out.astJson.contains("suggestedAnnotations")); + std::cout << "Test 6 PASSED: suggested annotations preserved\n"; + passed++; + } + + // Test 7: acceptance pipeline returns combined diagnostic counts + { + WorkItem item = makeAgentItem("a7"); + ReviewGate gate; + ReviewPolicy policy; + ResultSubmission sub; + sub.itemId = "a7"; + sub.generatedCode = "def bad(:\n"; + sub.suggestedAnnotations.push_back(json{{"foo", "bar"}}); + WorkItemResult out; + auto acceptance = evaluateResultSubmission(sub, item, "python", gate, policy, out); + assert(acceptance.diagnosticCount >= 2); + std::cout << "Test 7 PASSED: validation stages aggregate diagnostics\n"; + passed++; + } + + // Test 8: partial acceptance (valid code but review required) + { + auto state = makeState(); + state.workflow->queue.enqueue(makeAgentItem("a8")); + auto resp = rpc(state, "submitExternalResult", { + {"itemId", "a8"}, + {"result", { + {"generatedCode", "def buildTask(x):\n return x\n"}, + {"confidence", 0.2}, + {"reasoning", "uncertain"} + }} + }); + assert(resp.contains("result")); + assert(resp["result"]["acceptance"]["accepted"].get()); + assert(resp["result"]["acceptance"]["reviewRequired"].get()); + std::cout << "Test 8 PASSED: partial acceptance path works\n"; + passed++; + } + + // Test 9: multiple submissions for same item keep latest result + { + auto state = makeState(); + state.workflow->queue.enqueue(makeAgentItem("a9")); + rpc(state, "submitExternalResult", { + {"itemId", "a9"}, + {"result", { + {"generatedCode", "def buildTask(x):\n return x + 1\n"}, + {"confidence", 0.3} + }} + }); + auto second = rpc(state, "submitExternalResult", { + {"itemId", "a9"}, + {"result", { + {"generatedCode", "def buildTask(x):\n return x + 2\n"}, + {"confidence", 0.4} + }} + }); + assert(second.contains("result")); + auto item = state.workflow->queue.getItem("a9"); + assert(item.has_value()); + assert(item->result.generatedCode.find("+ 2") != std::string::npos); + std::cout << "Test 9 PASSED: latest submission wins\n"; + passed++; + } + + // Test 10: acceptance completion cascades dependencies + { + auto state = makeState(); + state.reviewPolicy.defaultAction = "auto-approve"; + WorkItem a = makeAgentItem("a10"); + WorkItem b = makeAgentItem("b10"); + b.status = WI_PENDING; + b.dependencies.push_back("a10"); + state.workflow->queue.enqueue(a); + state.workflow->queue.enqueue(b); + auto resp = rpc(state, "submitExternalResult", { + {"itemId", "a10"}, + {"result", { + {"generatedCode", "def buildTask(x):\n return x\n"}, + {"confidence", 0.95} + }} + }); + assert(resp["result"]["acceptance"]["autoApproved"].get()); + auto dep = state.workflow->queue.getItem("b10"); + assert(dep.has_value()); + assert(dep->status == WI_READY); + std::cout << "Test 10 PASSED: dependency cascade on acceptance\n"; + passed++; + } + + // Test 11: submitExternalResult response includes acceptance payload + { + auto state = makeState(); + state.workflow->queue.enqueue(makeAgentItem("a11")); + auto resp = rpc(state, "submitExternalResult", { + {"itemId", "a11"}, + {"result", { + {"generatedCode", "def buildTask(x):\n return x\n"}, + {"confidence", 0.3} + }} + }); + assert(resp.contains("result")); + assert(resp["result"].contains("acceptance")); + assert(resp["result"]["acceptance"].contains("validationPassed")); + std::cout << "Test 11 PASSED: acceptance payload exposed in RPC response\n"; + passed++; + } + + // Test 12: non-agent worker submissions are rejected + { + auto state = makeState(); + state.workflow->queue.enqueue(makeAgentItem("a12", "template")); + auto resp = rpc(state, "submitExternalResult", { + {"itemId", "a12"}, + {"result", {{"generatedCode", "return 1"}}} + }); + assert(resp.contains("error")); + std::cout << "Test 12 PASSED: non-agent worker guard preserved\n"; + passed++; + } + + std::cout << "\nResults: " << passed << "/12\n"; + assert(passed == 12); + return 0; +} diff --git a/progress.md b/progress.md index 0635bd7..e98f128 100644 --- a/progress.md +++ b/progress.md @@ -3102,6 +3102,50 @@ prompt rendering, JSON transport format, and token estimation utilities. - `editor/src/ContextBundle.h` within header-size limit (`168` <= `600`) - `editor/tests/step385_test.cpp` within test-file size guidance (`192` lines) +### Step 386: Result Acceptance Protocol +**Status:** PASS (12/12 tests) + +Added a standardized acceptance pipeline for external model submissions: +syntax/annotation validation, diagnostic accounting, review-gate decisioning, +structured acceptance output, dependency cascade on acceptance, and requeue on +validation failure. + +**Files created:** +- `editor/src/ResultAcceptance.h` — acceptance primitives: + - `ResultSubmission` + - `ResultAcceptance` + - `validateSuggestedAnnotations(...)` + - `evaluateResultSubmission(...)` +- `editor/tests/step386_test.cpp` — 12 tests covering: + 1. valid code acceptance + 2. syntax rejection with feedback + 3. malformed suggested-annotation rejection + 4. diagnostics preventing auto-approval + 5. low-confidence review path + 6. suggested-annotation payload carry-through + 7. aggregated diagnostic counts across validation stages + 8. partial acceptance (valid + review required) + 9. multiple-submission latest-wins behavior + 10. dependency cascade on acceptance + 11. acceptance payload in RPC response + 12. non-agent worker submission guard + +**Files modified:** +- `editor/src/HeadlessOrchestratorRPC.h` — `submitExternalResult` now uses + `evaluateResultSubmission(...)`, emits acceptance metadata, supports review + resubmission, and requeues invalid submissions with validator feedback +- `editor/CMakeLists.txt` — `step386_test` target + +**Verification run:** +- `step386_test` — PASS (12/12) new step coverage +- `step385_test` — PASS (12/12) regression coverage +- `step384_test` — PASS (12/12) regression coverage + +**Architecture gate check:** +- `editor/src/ResultAcceptance.h` within header-size limit (`125` <= `600`) +- `editor/src/HeadlessOrchestratorRPC.h` within header-size limit (`309` <= `600`) +- `editor/tests/step386_test.cpp` within test-file size guidance (`269` lines) + # Roadmap Planning — Sprints 12-25+ ## Status: Planning Complete (Sprints 12-19 detailed, 20-25 in roadmap.md)