bill/whetstone_DSL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Step 386: add result acceptance protocol

Browse Source
This commit is contained in:
Bill
2026-02-16 12:57:58 -07:00
parent 1f15e82401
commit 67e637b9e8
5 changed files with 487 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
editor/src/HeadlessOrchestratorRPC.h
View File

@@ -2,6 +2,7 @@
// Step 382: Orchestrator RPC surface for headless agent requests.
#include "HeadlessEditorState.h"
#include "ResultAcceptance.h"
#include <optional>
static inline json orchestratorRpcError(const json& id, int code,
@@ -216,25 +217,34 @@ inline std::optional<json> tryHandleHeadlessOrchestratorRPC(
return orchestratorRpcError(id, -32602, "Work item not found");
if (!(item->workerType == "slm" || item->workerType == "llm"))
return orchestratorRpcError(id, -32000, "submitExternalResult only supports slm/llm items");
if (item->status != WI_IN_PROGRESS && item->status != WI_ASSIGNED)
if (item->status != WI_IN_PROGRESS &&
item->status != WI_ASSIGNED &&
item->status != WI_REVIEW)
return orchestratorRpcError(id, -32000,
"Item must be assigned or in-progress for external submission");
"Item must be assigned, in-progress, or review for external submission");
WorkItem updated = *item;
if (updated.status == WI_ASSIGNED) transitionWorkItem(updated, WI_IN_PROGRESS);
const auto& result = params["result"];
updated.result.generatedCode = result.value("generatedCode", "");
updated.result.confidence = result.value("confidence", 0.0f);
updated.result.reasoning = result.value("reasoning", "");
updated.result.tokensGenerated = result.value("tokensGenerated", 0);
updated.result.tokensBudget = result.value("tokensBudget",
estimateContextBudget(updated.contextWidth));
if (result.contains("astJson")) updated.result.astJson = result["astJson"];
if (result.contains("diagnostics") && result["diagnostics"].is_array()) {
updated.result.diagnostics = result["diagnostics"].get<std::vector<json>>();
if (updated.status == WI_REVIEW) {
updated.status = WI_IN_PROGRESS;
}
ResultSubmission submission;
submission.itemId = itemId;
submission.generatedCode = params["result"].value("generatedCode", "");
submission.confidence = params["result"].value("confidence", 0.0f);
submission.reasoning = params["result"].value("reasoning", "");
if (params["result"].contains("suggestedAnnotations")) {
submission.suggestedAnnotations =
params["result"]["suggestedAnnotations"].get<std::vector<json>>();
}
std::string language = state.activeBuffer ? state.activeBuffer->language
: state.defaultLanguage;
WorkItemResult evaluatedResult;
ResultAcceptance acceptance = evaluateResultSubmission(
submission, updated, language, state.reviewGate, state.reviewPolicy, evaluatedResult);
updated.result = evaluatedResult;
state.workflow->queue.updateItem(itemId, updated);
std::vector<OrchestratorEvent> events;
@@ -247,12 +257,23 @@ inline std::optional<json> tryHandleHeadlessOrchestratorRPC(
workItemTimestamp()
});
ReviewDecision reviewDecision =
state.reviewGate.shouldAutoApprove(updated, updated.result, state.reviewPolicy);
if (reviewDecision.approved) {
if (!acceptance.validationPassed) {
WorkflowOrchestrator orchestrator(*state.workflow, state.routingEngine,
state.workerRegistry, state.contextAssembler,
state.reviewGate);
orchestrator.setReviewPolicy(state.reviewPolicy);
orchestrator.setBuffers(collectOrchestratorBufferInfos(state));
std::string feedback = acceptance.validationErrors.empty()
? "validation failed"
: acceptance.validationErrors.front();
orchestrator.rejectAndRequeue(itemId, feedback, "validator");
events.push_back({"rejected", itemId,
{{"reason", feedback}},
workItemTimestamp()});
} else if (acceptance.autoApproved) {
state.workflow->queue.complete(itemId);
events.push_back({"auto-approved", itemId,
{{"rule", reviewDecision.ruleMatched}},
{{"rule", "acceptance-auto-approve"}},
workItemTimestamp()});
events.push_back({"completed", itemId, json::object(), workItemTimestamp()});
} else {
@@ -264,7 +285,7 @@ inline std::optional<json> tryHandleHeadlessOrchestratorRPC(
transitionWorkItem(reviewItem, WI_REVIEW);
state.workflow->queue.updateItem(itemId, reviewItem);
events.push_back({"sent-to-review", itemId,
{{"reason", reviewDecision.reasoning}},
{{"reason", "validation passed, awaiting review"}},
workItemTimestamp()});
}
@@ -278,6 +299,7 @@ inline std::optional<json> tryHandleHeadlessOrchestratorRPC(
auto latest = state.workflow->queue.getItem(itemId);
return orchestratorRpcResult(id, {
{"success", true},
{"acceptance", acceptance.toJson()},
{"events", orchestratorEventsToJson(events)},
{"item", latest ? workItemToJson(*latest) : json::object()}
});

125
editor/src/ResultAcceptance.h Normal file
View File

@@ -0,0 +1,125 @@
#pragma once
// Step 386: External result acceptance protocol for workflow orchestration.
#include "Pipeline.h"
#include "ReviewGate.h"
#include <string>
#include <vector>
struct ResultSubmission {
std::string itemId;
std::string generatedCode;
float confidence = 0.0f;
std::string reasoning;
std::vector<json> suggestedAnnotations;
static ResultSubmission fromJson(const json& j) {
ResultSubmission s;
s.itemId = j.value("itemId", "");
s.generatedCode = j.value("generatedCode", "");
s.confidence = j.value("confidence", 0.0f);
s.reasoning = j.value("reasoning", "");
if (j.contains("suggestedAnnotations") && j["suggestedAnnotations"].is_array()) {
s.suggestedAnnotations = j["suggestedAnnotations"].get<std::vector<json>>();
}
return s;
}
};
struct ResultAcceptance {
bool accepted = false;
bool validationPassed = false;
int diagnosticCount = 0;
bool autoApproved = false;
bool reviewRequired = true;
std::vector<std::string> validationErrors;
json toJson() const {
return {
{"accepted", accepted},
{"validationPassed", validationPassed},
{"diagnosticCount", diagnosticCount},
{"autoApproved", autoApproved},
{"reviewRequired", reviewRequired},
{"validationErrors", validationErrors}
};
}
};
inline std::vector<std::string> validateSuggestedAnnotations(
const std::vector<json>& suggestedAnnotations) {
std::vector<std::string> errors;
for (size_t i = 0; i < suggestedAnnotations.size(); ++i) {
const auto& a = suggestedAnnotations[i];
if (!a.is_object()) {
errors.push_back("Suggested annotation #" + std::to_string(i) + " must be an object");
continue;
}
if (!a.contains("nodeId") || !a.contains("annotationType")) {
errors.push_back("Suggested annotation #" + std::to_string(i) +
" requires nodeId and annotationType");
}
}
return errors;
}
inline ResultAcceptance evaluateResultSubmission(const ResultSubmission& submission,
const WorkItem& item,
const std::string& language,
const ReviewGate& reviewGate,
const ReviewPolicy& reviewPolicy,
WorkItemResult& outResult) {
ResultAcceptance acceptance;
outResult = WorkItemResult{};
outResult.generatedCode = submission.generatedCode;
outResult.confidence = submission.confidence;
outResult.reasoning = submission.reasoning;
outResult.tokensGenerated = static_cast<int>(submission.generatedCode.size() / 4);
outResult.tokensBudget = estimateContextBudget(item.contextWidth);
if (submission.generatedCode.empty()) {
acceptance.validationErrors.push_back("Generated code is empty");
acceptance.validationPassed = false;
return acceptance;
}
Pipeline pipeline;
std::vector<ParseDiagnostic> parseDiags;
auto parsed = pipeline.parse(submission.generatedCode,
language.empty() ? "python" : language,
parseDiags);
for (const auto& d : parseDiags) {
acceptance.validationErrors.push_back(
"Parse error line " + std::to_string(d.line) + ": " + d.message);
}
acceptance.diagnosticCount = static_cast<int>(parseDiags.size());
auto annotationErrors = validateSuggestedAnnotations(submission.suggestedAnnotations);
for (const auto& err : annotationErrors) {
acceptance.validationErrors.push_back(err);
}
acceptance.diagnosticCount += static_cast<int>(annotationErrors.size());
if (!submission.suggestedAnnotations.empty()) {
outResult.astJson["suggestedAnnotations"] = submission.suggestedAnnotations;
}
if (parsed) {
outResult.astJson["parsed"] = true;
outResult.astJson["language"] = language.empty() ? "python" : language;
}
if (!acceptance.validationErrors.empty() || !parsed) {
acceptance.validationPassed = false;
acceptance.accepted = false;
acceptance.reviewRequired = true;
return acceptance;
}
acceptance.validationPassed = true;
ReviewDecision reviewDecision =
reviewGate.shouldAutoApprove(item, outResult, reviewPolicy);
acceptance.autoApproved = reviewDecision.approved;
acceptance.reviewRequired = !reviewDecision.approved;
acceptance.accepted = true;
return acceptance;
}