bill/whetstone_DSL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Step 386: add result acceptance protocol

Browse Source
This commit is contained in:
Bill
2026-02-16 12:57:58 -07:00
parent 1f15e82401
commit 67e637b9e8
5 changed files with 487 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
editor/CMakeLists.txt
View File

@@ -2353,4 +2353,13 @@ target_link_libraries(step385_test PRIVATE
tree_sitter_javascript tree_sitter_typescript
tree_sitter_java tree_sitter_rust tree_sitter_go)
add_executable(step386_test tests/step386_test.cpp)
target_include_directories(step386_test PRIVATE src)
target_link_libraries(step386_test PRIVATE
nlohmann_json::nlohmann_json
unofficial::tree-sitter::tree-sitter
tree_sitter_python tree_sitter_cpp tree_sitter_elisp
tree_sitter_javascript tree_sitter_typescript
tree_sitter_java tree_sitter_rust tree_sitter_go)
# Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies)

58
editor/src/HeadlessOrchestratorRPC.h
View File

@@ -2,6 +2,7 @@
// Step 382: Orchestrator RPC surface for headless agent requests.
#include "HeadlessEditorState.h"
#include "ResultAcceptance.h"
#include <optional>
static inline json orchestratorRpcError(const json& id, int code,
@@ -216,25 +217,34 @@ inline std::optional<json> tryHandleHeadlessOrchestratorRPC(
return orchestratorRpcError(id, -32602, "Work item not found");
if (!(item->workerType == "slm" || item->workerType == "llm"))
return orchestratorRpcError(id, -32000, "submitExternalResult only supports slm/llm items");
if (item->status != WI_IN_PROGRESS && item->status != WI_ASSIGNED)
if (item->status != WI_IN_PROGRESS &&
item->status != WI_ASSIGNED &&
item->status != WI_REVIEW)
return orchestratorRpcError(id, -32000,
"Item must be assigned or in-progress for external submission");
"Item must be assigned, in-progress, or review for external submission");
WorkItem updated = *item;
if (updated.status == WI_ASSIGNED) transitionWorkItem(updated, WI_IN_PROGRESS);
const auto& result = params["result"];
updated.result.generatedCode = result.value("generatedCode", "");
updated.result.confidence = result.value("confidence", 0.0f);
updated.result.reasoning = result.value("reasoning", "");
updated.result.tokensGenerated = result.value("tokensGenerated", 0);
updated.result.tokensBudget = result.value("tokensBudget",
estimateContextBudget(updated.contextWidth));
if (result.contains("astJson")) updated.result.astJson = result["astJson"];
if (result.contains("diagnostics") && result["diagnostics"].is_array()) {
updated.result.diagnostics = result["diagnostics"].get<std::vector<json>>();
if (updated.status == WI_REVIEW) {
updated.status = WI_IN_PROGRESS;
}
ResultSubmission submission;
submission.itemId = itemId;
submission.generatedCode = params["result"].value("generatedCode", "");
submission.confidence = params["result"].value("confidence", 0.0f);
submission.reasoning = params["result"].value("reasoning", "");
if (params["result"].contains("suggestedAnnotations")) {
submission.suggestedAnnotations =
params["result"]["suggestedAnnotations"].get<std::vector<json>>();
}
std::string language = state.activeBuffer ? state.activeBuffer->language
: state.defaultLanguage;
WorkItemResult evaluatedResult;
ResultAcceptance acceptance = evaluateResultSubmission(
submission, updated, language, state.reviewGate, state.reviewPolicy, evaluatedResult);
updated.result = evaluatedResult;
state.workflow->queue.updateItem(itemId, updated);
std::vector<OrchestratorEvent> events;
@@ -247,12 +257,23 @@ inline std::optional<json> tryHandleHeadlessOrchestratorRPC(
workItemTimestamp()
});
ReviewDecision reviewDecision =
state.reviewGate.shouldAutoApprove(updated, updated.result, state.reviewPolicy);
if (reviewDecision.approved) {
if (!acceptance.validationPassed) {
WorkflowOrchestrator orchestrator(*state.workflow, state.routingEngine,
state.workerRegistry, state.contextAssembler,
state.reviewGate);
orchestrator.setReviewPolicy(state.reviewPolicy);
orchestrator.setBuffers(collectOrchestratorBufferInfos(state));
std::string feedback = acceptance.validationErrors.empty()
? "validation failed"
: acceptance.validationErrors.front();
orchestrator.rejectAndRequeue(itemId, feedback, "validator");
events.push_back({"rejected", itemId,
{{"reason", feedback}},
workItemTimestamp()});
} else if (acceptance.autoApproved) {
state.workflow->queue.complete(itemId);
events.push_back({"auto-approved", itemId,
{{"rule", reviewDecision.ruleMatched}},
{{"rule", "acceptance-auto-approve"}},
workItemTimestamp()});
events.push_back({"completed", itemId, json::object(), workItemTimestamp()});
} else {
@@ -264,7 +285,7 @@ inline std::optional<json> tryHandleHeadlessOrchestratorRPC(
transitionWorkItem(reviewItem, WI_REVIEW);
state.workflow->queue.updateItem(itemId, reviewItem);
events.push_back({"sent-to-review", itemId,
{{"reason", reviewDecision.reasoning}},
{{"reason", "validation passed, awaiting review"}},
workItemTimestamp()});
}
@@ -278,6 +299,7 @@ inline std::optional<json> tryHandleHeadlessOrchestratorRPC(
auto latest = state.workflow->queue.getItem(itemId);
return orchestratorRpcResult(id, {
{"success", true},
{"acceptance", acceptance.toJson()},
{"events", orchestratorEventsToJson(events)},
{"item", latest ? workItemToJson(*latest) : json::object()}
});

125
editor/src/ResultAcceptance.h Normal file
View File

@@ -0,0 +1,125 @@
#pragma once
// Step 386: External result acceptance protocol for workflow orchestration.
#include "Pipeline.h"
#include "ReviewGate.h"
#include <string>
#include <vector>
struct ResultSubmission {
std::string itemId;
std::string generatedCode;
float confidence = 0.0f;
std::string reasoning;
std::vector<json> suggestedAnnotations;
static ResultSubmission fromJson(const json& j) {
ResultSubmission s;
s.itemId = j.value("itemId", "");
s.generatedCode = j.value("generatedCode", "");
s.confidence = j.value("confidence", 0.0f);
s.reasoning = j.value("reasoning", "");
if (j.contains("suggestedAnnotations") && j["suggestedAnnotations"].is_array()) {
s.suggestedAnnotations = j["suggestedAnnotations"].get<std::vector<json>>();
}
return s;
}
};
struct ResultAcceptance {
bool accepted = false;
bool validationPassed = false;
int diagnosticCount = 0;
bool autoApproved = false;
bool reviewRequired = true;
std::vector<std::string> validationErrors;
json toJson() const {
return {
{"accepted", accepted},
{"validationPassed", validationPassed},
{"diagnosticCount", diagnosticCount},
{"autoApproved", autoApproved},
{"reviewRequired", reviewRequired},
{"validationErrors", validationErrors}
};
}
};
inline std::vector<std::string> validateSuggestedAnnotations(
const std::vector<json>& suggestedAnnotations) {
std::vector<std::string> errors;
for (size_t i = 0; i < suggestedAnnotations.size(); ++i) {
const auto& a = suggestedAnnotations[i];
if (!a.is_object()) {
errors.push_back("Suggested annotation #" + std::to_string(i) + " must be an object");
continue;
}
if (!a.contains("nodeId") || !a.contains("annotationType")) {
errors.push_back("Suggested annotation #" + std::to_string(i) +
" requires nodeId and annotationType");
}
}
return errors;
}
inline ResultAcceptance evaluateResultSubmission(const ResultSubmission& submission,
const WorkItem& item,
const std::string& language,
const ReviewGate& reviewGate,
const ReviewPolicy& reviewPolicy,
WorkItemResult& outResult) {
ResultAcceptance acceptance;
outResult = WorkItemResult{};
outResult.generatedCode = submission.generatedCode;
outResult.confidence = submission.confidence;
outResult.reasoning = submission.reasoning;
outResult.tokensGenerated = static_cast<int>(submission.generatedCode.size() / 4);
outResult.tokensBudget = estimateContextBudget(item.contextWidth);
if (submission.generatedCode.empty()) {
acceptance.validationErrors.push_back("Generated code is empty");
acceptance.validationPassed = false;
return acceptance;
}
Pipeline pipeline;
std::vector<ParseDiagnostic> parseDiags;
auto parsed = pipeline.parse(submission.generatedCode,
language.empty() ? "python" : language,
parseDiags);
for (const auto& d : parseDiags) {
acceptance.validationErrors.push_back(
"Parse error line " + std::to_string(d.line) + ": " + d.message);
}
acceptance.diagnosticCount = static_cast<int>(parseDiags.size());
auto annotationErrors = validateSuggestedAnnotations(submission.suggestedAnnotations);
for (const auto& err : annotationErrors) {
acceptance.validationErrors.push_back(err);
}
acceptance.diagnosticCount += static_cast<int>(annotationErrors.size());
if (!submission.suggestedAnnotations.empty()) {
outResult.astJson["suggestedAnnotations"] = submission.suggestedAnnotations;
}
if (parsed) {
outResult.astJson["parsed"] = true;
outResult.astJson["language"] = language.empty() ? "python" : language;
}
if (!acceptance.validationErrors.empty() || !parsed) {
acceptance.validationPassed = false;
acceptance.accepted = false;
acceptance.reviewRequired = true;
return acceptance;
}
acceptance.validationPassed = true;
ReviewDecision reviewDecision =
reviewGate.shouldAutoApprove(item, outResult, reviewPolicy);
acceptance.autoApproved = reviewDecision.approved;
acceptance.reviewRequired = !reviewDecision.approved;
acceptance.accepted = true;
return acceptance;
}

269
editor/tests/step386_test.cpp Normal file
View File

@@ -0,0 +1,269 @@
// Step 386: Result acceptance protocol (12 tests)
#include <cassert>
#include <iostream>
#include <string>
#include "HeadlessEditorState.h"
#include "HeadlessAgentRPCHandler.h"
#include "ResultAcceptance.h"
static WorkItem makeAgentItem(const std::string& id,
const std::string& workerType = "llm") {
WorkItem item;
item.id = id;
item.nodeId = id + "_node";
item.nodeName = "buildTask";
item.nodeType = "Function";
item.bufferId = "main.py";
item.contextWidth = "project";
item.workerType = workerType;
item.priority = "high";
item.status = WI_IN_PROGRESS;
item.createdAt = workItemTimestamp();
return item;
}
static json rpc(HeadlessEditorState& state, const std::string& method,
const json& params = json::object()) {
json request = {{"jsonrpc", "2.0"}, {"id", 1}, {"method", method},
{"params", params}};
return handleHeadlessAgentRequest(state, request, "test-session");
}
static HeadlessEditorState makeState() {
HeadlessEditorState state;
state.defaultLanguage = "python";
state.openBuffer("main.py", "def seed():\n return 1\n", "python");
state.setAgentRole("test-session", AgentRole::Generator);
state.workflow = WorkflowState("accept");
return state;
}
int main() {
int passed = 0;
// Test 1: valid code submission accepted
{
WorkItem item = makeAgentItem("a1");
ReviewGate gate;
ReviewPolicy policy;
policy.defaultAction = "auto-approve";
ResultSubmission sub;
sub.itemId = "a1";
sub.generatedCode = "def buildTask(x):\n return x\n";
sub.confidence = 0.9f;
WorkItemResult out;
auto acceptance = evaluateResultSubmission(sub, item, "python", gate, policy, out);
assert(acceptance.accepted);
assert(acceptance.validationPassed);
std::cout << "Test 1 PASSED: valid submission accepted\n";
passed++;
}
// Test 2: syntax error rejected with validation feedback
{
WorkItem item = makeAgentItem("a2");
ReviewGate gate;
ReviewPolicy policy;
ResultSubmission sub;
sub.itemId = "a2";
sub.generatedCode = "def broken(:\n";
WorkItemResult out;
auto acceptance = evaluateResultSubmission(sub, item, "python", gate, policy, out);
assert(!acceptance.validationPassed);
assert(!acceptance.validationErrors.empty());
std::cout << "Test 2 PASSED: syntax errors rejected\n";
passed++;
}
// Test 3: suggested annotation validation flags malformed entries
{
WorkItem item = makeAgentItem("a3");
ReviewGate gate;
ReviewPolicy policy;
ResultSubmission sub;
sub.itemId = "a3";
sub.generatedCode = "def ok():\n return 1\n";
sub.suggestedAnnotations.push_back(json{{"foo", "bar"}});
WorkItemResult out;
auto acceptance = evaluateResultSubmission(sub, item, "python", gate, policy, out);
assert(!acceptance.validationPassed);
assert(acceptance.diagnosticCount >= 1);
std::cout << "Test 3 PASSED: malformed suggested annotations flagged\n";
passed++;
}
// Test 4: diagnostics prevent auto-approval
{
WorkItem item = makeAgentItem("a4");
ReviewGate gate;
ReviewPolicy policy;
policy.defaultAction = "auto-approve";
ResultSubmission sub;
sub.itemId = "a4";
sub.generatedCode = "def bad(:\n";
WorkItemResult out;
auto acceptance = evaluateResultSubmission(sub, item, "python", gate, policy, out);
assert(!acceptance.autoApproved);
assert(!acceptance.accepted);
std::cout << "Test 4 PASSED: diagnostics block auto-approval\n";
passed++;
}
// Test 5: low-confidence valid result goes to review under strict policy
{
WorkItem item = makeAgentItem("a5");
ReviewGate gate;
ReviewPolicy policy;
policy.defaultAction = "require-review";
ResultSubmission sub;
sub.itemId = "a5";
sub.generatedCode = "def ok(v):\n return v\n";
sub.confidence = 0.1f;
WorkItemResult out;
auto acceptance = evaluateResultSubmission(sub, item, "python", gate, policy, out);
assert(acceptance.accepted);
assert(acceptance.reviewRequired);
std::cout << "Test 5 PASSED: low-confidence result routed to review\n";
passed++;
}
// Test 6: suggested annotations attached to output result payload
{
WorkItem item = makeAgentItem("a6");
ReviewGate gate;
ReviewPolicy policy;
ResultSubmission sub;
sub.itemId = "a6";
sub.generatedCode = "def ok():\n return 1\n";
sub.suggestedAnnotations.push_back(
json{{"nodeId", "n1"}, {"annotationType", "Intent"}});
WorkItemResult out;
auto acceptance = evaluateResultSubmission(sub, item, "python", gate, policy, out);
assert(acceptance.validationPassed);
assert(out.astJson.contains("suggestedAnnotations"));
std::cout << "Test 6 PASSED: suggested annotations preserved\n";
passed++;
}
// Test 7: acceptance pipeline returns combined diagnostic counts
{
WorkItem item = makeAgentItem("a7");
ReviewGate gate;
ReviewPolicy policy;
ResultSubmission sub;
sub.itemId = "a7";
sub.generatedCode = "def bad(:\n";
sub.suggestedAnnotations.push_back(json{{"foo", "bar"}});
WorkItemResult out;
auto acceptance = evaluateResultSubmission(sub, item, "python", gate, policy, out);
assert(acceptance.diagnosticCount >= 2);
std::cout << "Test 7 PASSED: validation stages aggregate diagnostics\n";
passed++;
}
// Test 8: partial acceptance (valid code but review required)
{
auto state = makeState();
state.workflow->queue.enqueue(makeAgentItem("a8"));
auto resp = rpc(state, "submitExternalResult", {
{"itemId", "a8"},
{"result", {
{"generatedCode", "def buildTask(x):\n return x\n"},
{"confidence", 0.2},
{"reasoning", "uncertain"}
}}
});
assert(resp.contains("result"));
assert(resp["result"]["acceptance"]["accepted"].get<bool>());
assert(resp["result"]["acceptance"]["reviewRequired"].get<bool>());
std::cout << "Test 8 PASSED: partial acceptance path works\n";
passed++;
}
// Test 9: multiple submissions for same item keep latest result
{
auto state = makeState();
state.workflow->queue.enqueue(makeAgentItem("a9"));
rpc(state, "submitExternalResult", {
{"itemId", "a9"},
{"result", {
{"generatedCode", "def buildTask(x):\n return x + 1\n"},
{"confidence", 0.3}
}}
});
auto second = rpc(state, "submitExternalResult", {
{"itemId", "a9"},
{"result", {
{"generatedCode", "def buildTask(x):\n return x + 2\n"},
{"confidence", 0.4}
}}
});
assert(second.contains("result"));
auto item = state.workflow->queue.getItem("a9");
assert(item.has_value());
assert(item->result.generatedCode.find("+ 2") != std::string::npos);
std::cout << "Test 9 PASSED: latest submission wins\n";
passed++;
}
// Test 10: acceptance completion cascades dependencies
{
auto state = makeState();
state.reviewPolicy.defaultAction = "auto-approve";
WorkItem a = makeAgentItem("a10");
WorkItem b = makeAgentItem("b10");
b.status = WI_PENDING;
b.dependencies.push_back("a10");
state.workflow->queue.enqueue(a);
state.workflow->queue.enqueue(b);
auto resp = rpc(state, "submitExternalResult", {
{"itemId", "a10"},
{"result", {
{"generatedCode", "def buildTask(x):\n return x\n"},
{"confidence", 0.95}
}}
});
assert(resp["result"]["acceptance"]["autoApproved"].get<bool>());
auto dep = state.workflow->queue.getItem("b10");
assert(dep.has_value());
assert(dep->status == WI_READY);
std::cout << "Test 10 PASSED: dependency cascade on acceptance\n";
passed++;
}
// Test 11: submitExternalResult response includes acceptance payload
{
auto state = makeState();
state.workflow->queue.enqueue(makeAgentItem("a11"));
auto resp = rpc(state, "submitExternalResult", {
{"itemId", "a11"},
{"result", {
{"generatedCode", "def buildTask(x):\n return x\n"},
{"confidence", 0.3}
}}
});
assert(resp.contains("result"));
assert(resp["result"].contains("acceptance"));
assert(resp["result"]["acceptance"].contains("validationPassed"));
std::cout << "Test 11 PASSED: acceptance payload exposed in RPC response\n";
passed++;
}
// Test 12: non-agent worker submissions are rejected
{
auto state = makeState();
state.workflow->queue.enqueue(makeAgentItem("a12", "template"));
auto resp = rpc(state, "submitExternalResult", {
{"itemId", "a12"},
{"result", {{"generatedCode", "return 1"}}}
});
assert(resp.contains("error"));
std::cout << "Test 12 PASSED: non-agent worker guard preserved\n";
passed++;
}
std::cout << "\nResults: " << passed << "/12\n";
assert(passed == 12);
return 0;
}

44
progress.md
View File

@@ -3102,6 +3102,50 @@ prompt rendering, JSON transport format, and token estimation utilities.
- `editor/src/ContextBundle.h` within header-size limit (`168` <= `600`)
- `editor/tests/step385_test.cpp` within test-file size guidance (`192` lines)
### Step 386: Result Acceptance Protocol
**Status:** PASS (12/12 tests)
Added a standardized acceptance pipeline for external model submissions:
syntax/annotation validation, diagnostic accounting, review-gate decisioning,
structured acceptance output, dependency cascade on acceptance, and requeue on
validation failure.
**Files created:**
- `editor/src/ResultAcceptance.h` — acceptance primitives:
- `ResultSubmission`
- `ResultAcceptance`
- `validateSuggestedAnnotations(...)`
- `evaluateResultSubmission(...)`
- `editor/tests/step386_test.cpp` — 12 tests covering:
1. valid code acceptance
2. syntax rejection with feedback
3. malformed suggested-annotation rejection
4. diagnostics preventing auto-approval
5. low-confidence review path
6. suggested-annotation payload carry-through
7. aggregated diagnostic counts across validation stages
8. partial acceptance (valid + review required)
9. multiple-submission latest-wins behavior
10. dependency cascade on acceptance
11. acceptance payload in RPC response
12. non-agent worker submission guard
**Files modified:**
- `editor/src/HeadlessOrchestratorRPC.h` — `submitExternalResult` now uses
`evaluateResultSubmission(...)`, emits acceptance metadata, supports review
resubmission, and requeues invalid submissions with validator feedback
- `editor/CMakeLists.txt` — `step386_test` target
**Verification run:**
- `step386_test` — PASS (12/12) new step coverage
- `step385_test` — PASS (12/12) regression coverage
- `step384_test` — PASS (12/12) regression coverage
**Architecture gate check:**
- `editor/src/ResultAcceptance.h` within header-size limit (`125` <= `600`)
- `editor/src/HeadlessOrchestratorRPC.h` within header-size limit (`309` <= `600`)
- `editor/tests/step386_test.cpp` within test-file size guidance (`269` lines)
# Roadmap Planning — Sprints 12-25+
## Status: Planning Complete (Sprints 12-19 detailed, 20-25 in roadmap.md)