Steps 269-271: Phase 10b — annotation agent interface (32/32 tests)
- Step 269: RPC annotation methods (set/get/remove/unannotated) with permission enforcement (12 tests) - Step 270: MCP prompt templates for annotation workflows (12 tests) - Step 271: Integration tests for full agent annotation pipeline (8 tests) - Fixed sideEffectHint heuristic to check descendants of body statements Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Bill
@@ -1651,4 +1651,15 @@ target_link_libraries(step268_test PRIVATE
|
||||
tree_sitter_java tree_sitter_rust tree_sitter_go
|
||||
tree_sitter_org)
|
||||
|
||||
# Step 271: Phase 10b integration tests
|
||||
add_executable(step271_test tests/step271_test.cpp)
|
||||
target_include_directories(step271_test PRIVATE src)
|
||||
target_link_libraries(step271_test PRIVATE
|
||||
nlohmann_json::nlohmann_json
|
||||
unofficial::tree-sitter::tree-sitter
|
||||
tree_sitter_python tree_sitter_cpp tree_sitter_elisp
|
||||
tree_sitter_javascript tree_sitter_typescript
|
||||
tree_sitter_java tree_sitter_rust tree_sitter_go
|
||||
tree_sitter_org)
|
||||
|
||||
# Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies)
|
||||
|
||||
@@ -1866,10 +1866,17 @@ inline json handleHeadlessAgentRequest(HeadlessEditorState& state,
|
||||
hints["bodyStatements"] = (int)body.size();
|
||||
// Side effect heuristic: check for io/network calls
|
||||
bool hasSideEffects = false;
|
||||
// Check body statements and their descendants
|
||||
std::vector<const ASTNode*> toCheck;
|
||||
for (auto* stmt : body) {
|
||||
if (stmt->conceptType == "FunctionCall" ||
|
||||
stmt->conceptType == "MethodCall") {
|
||||
std::string callee = getNodeName(stmt);
|
||||
toCheck.push_back(stmt);
|
||||
for (auto* desc : stmt->allChildren())
|
||||
toCheck.push_back(desc);
|
||||
}
|
||||
for (auto* node : toCheck) {
|
||||
if (node->conceptType == "FunctionCall" ||
|
||||
node->conceptType == "MethodCall") {
|
||||
std::string callee = getNodeName(node);
|
||||
if (callee.find("print") != std::string::npos ||
|
||||
callee.find("write") != std::string::npos ||
|
||||
callee.find("send") != std::string::npos ||
|
||||
|
||||
530
editor/tests/step269_test.cpp
Normal file
530
editor/tests/step269_test.cpp
Normal file
@@ -0,0 +1,530 @@
|
||||
// Step 269 TDD Test: Annotation RPC Methods
|
||||
//
|
||||
// High-level RPC methods for agents to set/get/remove semantic annotations
|
||||
// without needing low-level applyMutation(insertNode) calls.
|
||||
#include "HeadlessEditorState.h"
|
||||
#include "MCPServer.h"
|
||||
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
|
||||
static void expect(bool cond, const std::string& name,
|
||||
int& passed, int& failed) {
|
||||
if (cond) {
|
||||
std::cout << "Test " << (passed + failed + 1)
|
||||
<< " PASS: " << name << "\n";
|
||||
++passed;
|
||||
} else {
|
||||
std::cout << "Test " << (passed + failed + 1)
|
||||
<< " FAIL: " << name << "\n";
|
||||
++failed;
|
||||
}
|
||||
}
|
||||
|
||||
static json rpc(HeadlessEditorState& state, const std::string& session,
|
||||
const std::string& method,
|
||||
json params = json::object()) {
|
||||
json request = {{"jsonrpc", "2.0"}, {"id", 1},
|
||||
{"method", method}, {"params", params}};
|
||||
return state.processAgentRequest(request, session);
|
||||
}
|
||||
|
||||
int main() {
|
||||
int passed = 0;
|
||||
int failed = 0;
|
||||
|
||||
std::string src =
|
||||
"def greet(name):\n"
|
||||
" return \"hello \" + name\n"
|
||||
"\n"
|
||||
"def farewell(name):\n"
|
||||
" return \"goodbye \" + name\n";
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 1: setSemanticAnnotation adds intent annotation
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.setAgentRole("s1", AgentRole::Refactor);
|
||||
|
||||
rpc(state, "s1", "openFile",
|
||||
{{"path", "t.py"}, {"content", src}});
|
||||
|
||||
std::string fnId;
|
||||
for (auto* c : state.activeAST()->allChildren()) {
|
||||
if (c->conceptType == "Function" && getNodeName(c) == "greet") {
|
||||
fnId = c->id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
json resp = rpc(state, "s1", "setSemanticAnnotation",
|
||||
{{"nodeId", fnId}, {"type", "intent"},
|
||||
{"fields", {{"summary", "greets user"},
|
||||
{"category", "io"}}}});
|
||||
|
||||
bool success = resp.contains("result") &&
|
||||
resp["result"].value("success", false);
|
||||
|
||||
// Verify annotation exists
|
||||
bool found = false;
|
||||
ASTNode* fn = findNodeById(state.activeAST(), fnId);
|
||||
if (fn) {
|
||||
for (auto* a : fn->getChildren("annotations")) {
|
||||
if (a->conceptType == "IntentAnnotation") {
|
||||
auto* ia = static_cast<IntentAnnotation*>(a);
|
||||
if (ia->summary == "greets user") found = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(success && found,
|
||||
"setSemanticAnnotation adds IntentAnnotation",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 2: setSemanticAnnotation updates existing (no duplicates)
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.setAgentRole("s1", AgentRole::Refactor);
|
||||
|
||||
rpc(state, "s1", "openFile",
|
||||
{{"path", "t.py"}, {"content", src}});
|
||||
|
||||
std::string fnId;
|
||||
for (auto* c : state.activeAST()->allChildren()) {
|
||||
if (c->conceptType == "Function" && getNodeName(c) == "greet") {
|
||||
fnId = c->id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Set intent twice with different values
|
||||
rpc(state, "s1", "setSemanticAnnotation",
|
||||
{{"nodeId", fnId}, {"type", "intent"},
|
||||
{"fields", {{"summary", "greets v1"}, {"category", "io"}}}});
|
||||
rpc(state, "s1", "setSemanticAnnotation",
|
||||
{{"nodeId", fnId}, {"type", "intent"},
|
||||
{"fields", {{"summary", "greets v2"}, {"category", "io"}}}});
|
||||
|
||||
// Should have exactly 1 IntentAnnotation with "greets v2"
|
||||
int count = 0;
|
||||
std::string lastSummary;
|
||||
ASTNode* fn = findNodeById(state.activeAST(), fnId);
|
||||
if (fn) {
|
||||
for (auto* a : fn->getChildren("annotations")) {
|
||||
if (a->conceptType == "IntentAnnotation") {
|
||||
++count;
|
||||
lastSummary = static_cast<IntentAnnotation*>(a)->summary;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(count == 1 && lastSummary == "greets v2",
|
||||
"setSemanticAnnotation updates existing, no duplicates",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 3: setSemanticAnnotation for all 5 types
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.setAgentRole("s1", AgentRole::Refactor);
|
||||
|
||||
rpc(state, "s1", "openFile",
|
||||
{{"path", "t.py"}, {"content", src}});
|
||||
|
||||
std::string fnId;
|
||||
for (auto* c : state.activeAST()->allChildren()) {
|
||||
if (c->conceptType == "Function" && getNodeName(c) == "greet") {
|
||||
fnId = c->id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
rpc(state, "s1", "setSemanticAnnotation",
|
||||
{{"nodeId", fnId}, {"type", "intent"},
|
||||
{"fields", {{"summary", "greets"}, {"category", "io"}}}});
|
||||
rpc(state, "s1", "setSemanticAnnotation",
|
||||
{{"nodeId", fnId}, {"type", "complexity"},
|
||||
{"fields", {{"timeComplexity", "O(1)"},
|
||||
{"cognitiveComplexity", 1},
|
||||
{"linesOfLogic", 2}}}});
|
||||
rpc(state, "s1", "setSemanticAnnotation",
|
||||
{{"nodeId", fnId}, {"type", "risk"},
|
||||
{"fields", {{"level", "low"}, {"reason", "simple"},
|
||||
{"dependentCount", 2}}}});
|
||||
rpc(state, "s1", "setSemanticAnnotation",
|
||||
{{"nodeId", fnId}, {"type", "contract"},
|
||||
{"fields", {{"preconditions", "name is str"},
|
||||
{"postconditions", "returns greeting"},
|
||||
{"returnShape", "str"},
|
||||
{"sideEffects", "none"}}}});
|
||||
rpc(state, "s1", "setSemanticAnnotation",
|
||||
{{"nodeId", fnId}, {"type", "tags"},
|
||||
{"fields", {{"tags", json::array({"@io", "@greeting"})}}}});
|
||||
|
||||
int count = 0;
|
||||
ASTNode* fn = findNodeById(state.activeAST(), fnId);
|
||||
if (fn) {
|
||||
for (auto* a : fn->getChildren("annotations")) {
|
||||
if (isSemanticAnnotation(a->conceptType))
|
||||
++count;
|
||||
}
|
||||
}
|
||||
|
||||
expect(count == 5,
|
||||
"setSemanticAnnotation works for all 5 types (" +
|
||||
std::to_string(count) + "/5)",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 4: getSemanticAnnotations on a specific node
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.setAgentRole("s1", AgentRole::Refactor);
|
||||
|
||||
rpc(state, "s1", "openFile",
|
||||
{{"path", "t.py"}, {"content", src}});
|
||||
|
||||
std::string fnId;
|
||||
for (auto* c : state.activeAST()->allChildren()) {
|
||||
if (c->conceptType == "Function" && getNodeName(c) == "greet") {
|
||||
fnId = c->id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
rpc(state, "s1", "setSemanticAnnotation",
|
||||
{{"nodeId", fnId}, {"type", "intent"},
|
||||
{"fields", {{"summary", "greets user"}, {"category", "io"}}}});
|
||||
rpc(state, "s1", "setSemanticAnnotation",
|
||||
{{"nodeId", fnId}, {"type", "risk"},
|
||||
{"fields", {{"level", "low"}, {"reason", "trivial"},
|
||||
{"dependentCount", 0}}}});
|
||||
|
||||
json resp = rpc(state, "s1", "getSemanticAnnotations",
|
||||
{{"nodeId", fnId}});
|
||||
bool hasResult = resp.contains("result");
|
||||
int annoCount = 0;
|
||||
bool hasIntent = false, hasRisk = false;
|
||||
if (hasResult && resp["result"].contains("annotations")) {
|
||||
auto annos = resp["result"]["annotations"];
|
||||
annoCount = (int)annos.size();
|
||||
for (const auto& a : annos) {
|
||||
if (a.value("type", "") == "intent") hasIntent = true;
|
||||
if (a.value("type", "") == "risk") hasRisk = true;
|
||||
}
|
||||
}
|
||||
|
||||
expect(hasResult && annoCount == 2 && hasIntent && hasRisk,
|
||||
"getSemanticAnnotations returns 2 annotations on specific node",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 5: getSemanticAnnotations with no nodeId returns all
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.setAgentRole("s1", AgentRole::Refactor);
|
||||
|
||||
rpc(state, "s1", "openFile",
|
||||
{{"path", "t.py"}, {"content", src}});
|
||||
|
||||
std::string greetId, farewellId;
|
||||
for (auto* c : state.activeAST()->allChildren()) {
|
||||
if (c->conceptType == "Function") {
|
||||
if (getNodeName(c) == "greet") greetId = c->id;
|
||||
if (getNodeName(c) == "farewell") farewellId = c->id;
|
||||
}
|
||||
}
|
||||
|
||||
rpc(state, "s1", "setSemanticAnnotation",
|
||||
{{"nodeId", greetId}, {"type", "intent"},
|
||||
{"fields", {{"summary", "greets"}, {"category", "io"}}}});
|
||||
rpc(state, "s1", "setSemanticAnnotation",
|
||||
{{"nodeId", farewellId}, {"type", "intent"},
|
||||
{"fields", {{"summary", "says goodbye"}, {"category", "io"}}}});
|
||||
|
||||
json resp = rpc(state, "s1", "getSemanticAnnotations");
|
||||
bool hasResult = resp.contains("result");
|
||||
int nodeCount = 0;
|
||||
if (hasResult && resp["result"].contains("nodes"))
|
||||
nodeCount = (int)resp["result"]["nodes"].size();
|
||||
|
||||
expect(hasResult && nodeCount >= 2,
|
||||
"getSemanticAnnotations (no nodeId) returns " +
|
||||
std::to_string(nodeCount) + " annotated nodes",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 6: removeSemanticAnnotation removes specific type
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.setAgentRole("s1", AgentRole::Refactor);
|
||||
|
||||
rpc(state, "s1", "openFile",
|
||||
{{"path", "t.py"}, {"content", src}});
|
||||
|
||||
std::string fnId;
|
||||
for (auto* c : state.activeAST()->allChildren()) {
|
||||
if (c->conceptType == "Function" && getNodeName(c) == "greet") {
|
||||
fnId = c->id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Add two annotations
|
||||
rpc(state, "s1", "setSemanticAnnotation",
|
||||
{{"nodeId", fnId}, {"type", "intent"},
|
||||
{"fields", {{"summary", "greets"}, {"category", "io"}}}});
|
||||
rpc(state, "s1", "setSemanticAnnotation",
|
||||
{{"nodeId", fnId}, {"type", "risk"},
|
||||
{"fields", {{"level", "low"}, {"reason", "simple"},
|
||||
{"dependentCount", 0}}}});
|
||||
|
||||
// Remove intent
|
||||
json resp = rpc(state, "s1", "removeSemanticAnnotation",
|
||||
{{"nodeId", fnId}, {"type", "intent"}});
|
||||
bool success = resp.contains("result") &&
|
||||
resp["result"].value("removed", false);
|
||||
|
||||
// Check only risk remains
|
||||
int intentCount = 0, riskCount = 0;
|
||||
ASTNode* fn = findNodeById(state.activeAST(), fnId);
|
||||
if (fn) {
|
||||
for (auto* a : fn->getChildren("annotations")) {
|
||||
if (a->conceptType == "IntentAnnotation") ++intentCount;
|
||||
if (a->conceptType == "RiskAnnotation") ++riskCount;
|
||||
}
|
||||
}
|
||||
|
||||
expect(success && intentCount == 0 && riskCount == 1,
|
||||
"removeSemanticAnnotation removes intent, keeps risk",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 7: getUnannotatedNodes returns functions without annotations
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.setAgentRole("s1", AgentRole::Refactor);
|
||||
|
||||
rpc(state, "s1", "openFile",
|
||||
{{"path", "t.py"}, {"content", src}});
|
||||
|
||||
// Before annotations: both should be unannotated
|
||||
json resp = rpc(state, "s1", "getUnannotatedNodes");
|
||||
int countBefore = 0;
|
||||
if (resp.contains("result") && resp["result"].contains("nodes"))
|
||||
countBefore = (int)resp["result"]["nodes"].size();
|
||||
|
||||
// Annotate greet
|
||||
std::string greetId;
|
||||
for (auto* c : state.activeAST()->allChildren()) {
|
||||
if (c->conceptType == "Function" && getNodeName(c) == "greet") {
|
||||
greetId = c->id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
rpc(state, "s1", "setSemanticAnnotation",
|
||||
{{"nodeId", greetId}, {"type", "intent"},
|
||||
{"fields", {{"summary", "greets"}, {"category", "io"}}}});
|
||||
|
||||
// After: only farewell should be unannotated
|
||||
json resp2 = rpc(state, "s1", "getUnannotatedNodes");
|
||||
int countAfter = 0;
|
||||
if (resp2.contains("result") && resp2["result"].contains("nodes"))
|
||||
countAfter = (int)resp2["result"]["nodes"].size();
|
||||
|
||||
expect(countBefore >= 2 && countAfter == countBefore - 1,
|
||||
"getUnannotatedNodes: " + std::to_string(countBefore) +
|
||||
" before, " + std::to_string(countAfter) + " after annotating greet",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 8: getUnannotatedNodes with type filter
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.setAgentRole("s1", AgentRole::Refactor);
|
||||
|
||||
rpc(state, "s1", "openFile",
|
||||
{{"path", "t.py"}, {"content", src}});
|
||||
|
||||
std::string greetId;
|
||||
for (auto* c : state.activeAST()->allChildren()) {
|
||||
if (c->conceptType == "Function" && getNodeName(c) == "greet") {
|
||||
greetId = c->id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Add only intent to greet
|
||||
rpc(state, "s1", "setSemanticAnnotation",
|
||||
{{"nodeId", greetId}, {"type", "intent"},
|
||||
{"fields", {{"summary", "greets"}, {"category", "io"}}}});
|
||||
|
||||
// getUnannotatedNodes for "risk" should include greet
|
||||
json resp = rpc(state, "s1", "getUnannotatedNodes",
|
||||
{{"type", "risk"}});
|
||||
int count = 0;
|
||||
bool greetIncluded = false;
|
||||
if (resp.contains("result") && resp["result"].contains("nodes")) {
|
||||
count = (int)resp["result"]["nodes"].size();
|
||||
for (const auto& n : resp["result"]["nodes"]) {
|
||||
if (n.value("name", "") == "greet") greetIncluded = true;
|
||||
}
|
||||
}
|
||||
|
||||
expect(count >= 2 && greetIncluded,
|
||||
"getUnannotatedNodes(type=risk) includes greet (has intent but not risk)",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 9: Linter cannot set/remove annotations
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.setAgentRole("admin", AgentRole::Refactor);
|
||||
state.setAgentRole("linter", AgentRole::Linter);
|
||||
|
||||
rpc(state, "admin", "openFile",
|
||||
{{"path", "t.py"}, {"content", src}});
|
||||
|
||||
std::string fnId;
|
||||
for (auto* c : state.activeAST()->allChildren()) {
|
||||
if (c->conceptType == "Function" && getNodeName(c) == "greet") {
|
||||
fnId = c->id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Linter cannot set
|
||||
json resp1 = rpc(state, "linter", "setSemanticAnnotation",
|
||||
{{"nodeId", fnId}, {"type", "intent"},
|
||||
{"fields", {{"summary", "test"}, {"category", "io"}}}});
|
||||
bool setDenied = resp1.contains("error");
|
||||
|
||||
// Linter CAN get
|
||||
json resp2 = rpc(state, "linter", "getSemanticAnnotations",
|
||||
{{"nodeId", fnId}});
|
||||
bool getOk = resp2.contains("result");
|
||||
|
||||
// Linter CAN get unannotated
|
||||
json resp3 = rpc(state, "linter", "getUnannotatedNodes");
|
||||
bool getUnOk = resp3.contains("result");
|
||||
|
||||
expect(setDenied && getOk && getUnOk,
|
||||
"Linter denied set, allowed get/getUnannotated",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 10: 4 MCP tools registered for annotation operations
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
MCPServer mcp;
|
||||
bool foundSet = false, foundGet = false,
|
||||
foundRemove = false, foundUnannotated = false;
|
||||
for (const auto& t : mcp.getTools()) {
|
||||
if (t.name == "whetstone_set_semantic_annotation") foundSet = true;
|
||||
if (t.name == "whetstone_get_semantic_annotations") foundGet = true;
|
||||
if (t.name == "whetstone_remove_semantic_annotation") foundRemove = true;
|
||||
if (t.name == "whetstone_get_unannotated_nodes") foundUnannotated = true;
|
||||
}
|
||||
expect(foundSet && foundGet && foundRemove && foundUnannotated,
|
||||
"4 annotation MCP tools registered",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 11: setSemanticAnnotation on invalid node returns error
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.setAgentRole("s1", AgentRole::Refactor);
|
||||
|
||||
rpc(state, "s1", "openFile",
|
||||
{{"path", "t.py"}, {"content", src}});
|
||||
|
||||
json resp = rpc(state, "s1", "setSemanticAnnotation",
|
||||
{{"nodeId", "nonexistent_node"}, {"type", "intent"},
|
||||
{"fields", {{"summary", "test"}, {"category", "io"}}}});
|
||||
bool hasError = resp.contains("error");
|
||||
|
||||
expect(hasError,
|
||||
"setSemanticAnnotation on invalid nodeId returns error",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 12: getSemanticAnnotations response format
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.setAgentRole("s1", AgentRole::Refactor);
|
||||
|
||||
rpc(state, "s1", "openFile",
|
||||
{{"path", "t.py"}, {"content", src}});
|
||||
|
||||
std::string fnId;
|
||||
for (auto* c : state.activeAST()->allChildren()) {
|
||||
if (c->conceptType == "Function" && getNodeName(c) == "greet") {
|
||||
fnId = c->id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
rpc(state, "s1", "setSemanticAnnotation",
|
||||
{{"nodeId", fnId}, {"type", "intent"},
|
||||
{"fields", {{"summary", "greets user"}, {"category", "io"}}}});
|
||||
|
||||
json resp = rpc(state, "s1", "getSemanticAnnotations",
|
||||
{{"nodeId", fnId}});
|
||||
bool hasAnnotations = resp.contains("result") &&
|
||||
resp["result"].contains("annotations");
|
||||
bool hasCorrectFields = false;
|
||||
if (hasAnnotations) {
|
||||
for (const auto& a : resp["result"]["annotations"]) {
|
||||
if (a.value("type", "") == "intent") {
|
||||
hasCorrectFields =
|
||||
a.contains("summary") &&
|
||||
a.value("summary", "") == "greets user" &&
|
||||
a.contains("category") &&
|
||||
a.value("category", "") == "io";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(hasAnnotations && hasCorrectFields,
|
||||
"getSemanticAnnotations returns structured annotation fields",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
std::cout << "\n=== Step 269 Results: " << passed << " passed, "
|
||||
<< failed << " failed ===\n";
|
||||
return failed == 0 ? 0 : 1;
|
||||
}
|
||||
283
editor/tests/step270_test.cpp
Normal file
283
editor/tests/step270_test.cpp
Normal file
@@ -0,0 +1,283 @@
|
||||
// Step 270 TDD Test: Annotation Prompt Templates
|
||||
//
|
||||
// MCP prompt templates that guide a frontier model through annotating
|
||||
// a codebase. Tests prompt registration, static analysis hints in
|
||||
// getUnannotatedNodes, and prompt message content.
|
||||
#include "HeadlessEditorState.h"
|
||||
#include "MCPServer.h"
|
||||
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
|
||||
static void expect(bool cond, const std::string& name,
|
||||
int& passed, int& failed) {
|
||||
if (cond) {
|
||||
std::cout << "Test " << (passed + failed + 1)
|
||||
<< " PASS: " << name << "\n";
|
||||
++passed;
|
||||
} else {
|
||||
std::cout << "Test " << (passed + failed + 1)
|
||||
<< " FAIL: " << name << "\n";
|
||||
++failed;
|
||||
}
|
||||
}
|
||||
|
||||
static json rpc(HeadlessEditorState& state, const std::string& session,
|
||||
const std::string& method,
|
||||
json params = json::object()) {
|
||||
json request = {{"jsonrpc", "2.0"}, {"id", 1},
|
||||
{"method", method}, {"params", params}};
|
||||
return state.processAgentRequest(request, session);
|
||||
}
|
||||
|
||||
int main() {
|
||||
int passed = 0;
|
||||
int failed = 0;
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 1: annotate_intent prompt registered
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
MCPServer mcp;
|
||||
bool found = false;
|
||||
for (const auto& p : mcp.getPrompts()) {
|
||||
if (p.name == "annotate_intent") found = true;
|
||||
}
|
||||
expect(found, "annotate_intent prompt registered", passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 2: annotate_complexity prompt registered
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
MCPServer mcp;
|
||||
bool found = false;
|
||||
for (const auto& p : mcp.getPrompts()) {
|
||||
if (p.name == "annotate_complexity") found = true;
|
||||
}
|
||||
expect(found, "annotate_complexity prompt registered", passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 3: annotate_risk prompt registered
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
MCPServer mcp;
|
||||
bool found = false;
|
||||
for (const auto& p : mcp.getPrompts()) {
|
||||
if (p.name == "annotate_risk") found = true;
|
||||
}
|
||||
expect(found, "annotate_risk prompt registered", passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 4: annotate_contracts prompt registered
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
MCPServer mcp;
|
||||
bool found = false;
|
||||
for (const auto& p : mcp.getPrompts()) {
|
||||
if (p.name == "annotate_contracts") found = true;
|
||||
}
|
||||
expect(found, "annotate_contracts prompt registered", passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 5: annotate_full prompt registered
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
MCPServer mcp;
|
||||
bool found = false;
|
||||
for (const auto& p : mcp.getPrompts()) {
|
||||
if (p.name == "annotate_full") found = true;
|
||||
}
|
||||
expect(found, "annotate_full prompt registered", passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 6: annotate_intent prompt returns message with instructions
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
MCPServer mcp;
|
||||
json req = {{"jsonrpc", "2.0"}, {"id", 1},
|
||||
{"method", "prompts/get"},
|
||||
{"params", {{"name", "annotate_intent"},
|
||||
{"arguments", json::object()}}}};
|
||||
json resp = mcp.handleRequest(req);
|
||||
bool hasMessages = resp.contains("result") &&
|
||||
resp["result"].contains("messages");
|
||||
bool mentionsSetAnnotation = false;
|
||||
bool mentionsGetUnannotated = false;
|
||||
if (hasMessages) {
|
||||
std::string text = resp["result"]["messages"][0]
|
||||
["content"].value("text", "");
|
||||
mentionsSetAnnotation =
|
||||
text.find("setSemanticAnnotation") != std::string::npos ||
|
||||
text.find("set_semantic_annotation") != std::string::npos;
|
||||
mentionsGetUnannotated =
|
||||
text.find("getUnannotatedNodes") != std::string::npos ||
|
||||
text.find("get_unannotated") != std::string::npos;
|
||||
}
|
||||
expect(hasMessages && mentionsSetAnnotation && mentionsGetUnannotated,
|
||||
"annotate_intent prompt mentions setAnnotation and getUnannotated",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 7: annotate_full prompt mentions save
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
MCPServer mcp;
|
||||
json req = {{"jsonrpc", "2.0"}, {"id", 1},
|
||||
{"method", "prompts/get"},
|
||||
{"params", {{"name", "annotate_full"},
|
||||
{"arguments", json::object()}}}};
|
||||
json resp = mcp.handleRequest(req);
|
||||
bool mentionsSave = false;
|
||||
if (resp.contains("result") &&
|
||||
resp["result"].contains("messages")) {
|
||||
std::string text = resp["result"]["messages"][0]
|
||||
["content"].value("text", "");
|
||||
mentionsSave =
|
||||
text.find("save") != std::string::npos ||
|
||||
text.find("Save") != std::string::npos;
|
||||
}
|
||||
expect(mentionsSave,
|
||||
"annotate_full prompt mentions saving annotated AST",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 8: getUnannotatedNodes includes static analysis hints
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.setAgentRole("s1", AgentRole::Refactor);
|
||||
|
||||
std::string src =
|
||||
"def greet(name):\n"
|
||||
" return \"hello \" + name\n"
|
||||
"\n"
|
||||
"def process(data):\n"
|
||||
" x = data + 1\n"
|
||||
" y = x * 2\n"
|
||||
" return y\n";
|
||||
|
||||
rpc(state, "s1", "openFile",
|
||||
{{"path", "t.py"}, {"content", src}});
|
||||
|
||||
json resp = rpc(state, "s1", "getUnannotatedNodes",
|
||||
{{"includeHints", true}});
|
||||
bool hasHints = false;
|
||||
if (resp.contains("result") && resp["result"].contains("nodes")) {
|
||||
for (const auto& n : resp["result"]["nodes"]) {
|
||||
if (n.contains("hints")) {
|
||||
hasHints = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(hasHints,
|
||||
"getUnannotatedNodes with includeHints returns static hints",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 9: Hints include bodyStatements count
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.setAgentRole("s1", AgentRole::Refactor);
|
||||
|
||||
std::string src =
|
||||
"def process(data):\n"
|
||||
" x = data + 1\n"
|
||||
" y = x * 2\n"
|
||||
" return y\n";
|
||||
|
||||
rpc(state, "s1", "openFile",
|
||||
{{"path", "t.py"}, {"content", src}});
|
||||
|
||||
json resp = rpc(state, "s1", "getUnannotatedNodes",
|
||||
{{"includeHints", true}});
|
||||
int bodyStmts = -1;
|
||||
if (resp.contains("result") && resp["result"].contains("nodes")) {
|
||||
for (const auto& n : resp["result"]["nodes"]) {
|
||||
if (n.value("name", "") == "process" &&
|
||||
n.contains("hints")) {
|
||||
bodyStmts = n["hints"].value("bodyStatements", -1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
expect(bodyStmts >= 2,
|
||||
"Hints include bodyStatements=" + std::to_string(bodyStmts) +
|
||||
" for process()",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 10: All 5 prompts available via prompts/list
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
MCPServer mcp;
|
||||
json req = {{"jsonrpc", "2.0"}, {"id", 1},
|
||||
{"method", "prompts/list"}};
|
||||
json resp = mcp.handleRequest(req);
|
||||
int annotatePrompts = 0;
|
||||
if (resp.contains("result") && resp["result"].contains("prompts")) {
|
||||
for (const auto& p : resp["result"]["prompts"]) {
|
||||
std::string name = p.value("name", "");
|
||||
if (name.find("annotate_") == 0)
|
||||
++annotatePrompts;
|
||||
}
|
||||
}
|
||||
|
||||
expect(annotatePrompts >= 5,
|
||||
std::to_string(annotatePrompts) +
|
||||
" annotate_* prompts in prompts/list",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 11: annotate_risk prompt mentions callers/dependents
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
MCPServer mcp;
|
||||
json req = {{"jsonrpc", "2.0"}, {"id", 1},
|
||||
{"method", "prompts/get"},
|
||||
{"params", {{"name", "annotate_risk"},
|
||||
{"arguments", json::object()}}}};
|
||||
json resp = mcp.handleRequest(req);
|
||||
bool mentionsCallers = false;
|
||||
if (resp.contains("result") && resp["result"].contains("messages")) {
|
||||
std::string text = resp["result"]["messages"][0]
|
||||
["content"].value("text", "");
|
||||
mentionsCallers =
|
||||
text.find("caller") != std::string::npos ||
|
||||
text.find("dependent") != std::string::npos ||
|
||||
text.find("Caller") != std::string::npos;
|
||||
}
|
||||
expect(mentionsCallers,
|
||||
"annotate_risk prompt mentions callers/dependents",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 12: Total prompt count after Sprint 10
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
MCPServer mcp;
|
||||
int total = (int)mcp.getPrompts().size();
|
||||
expect(total >= 9,
|
||||
"Total prompts: " + std::to_string(total) + " (>= 9 expected)",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
std::cout << "\n=== Step 270 Results: " << passed << " passed, "
|
||||
<< failed << " failed ===\n";
|
||||
return failed == 0 ? 0 : 1;
|
||||
}
|
||||
492
editor/tests/step271_test.cpp
Normal file
492
editor/tests/step271_test.cpp
Normal file
@@ -0,0 +1,492 @@
|
||||
// Step 271 TDD Test: Phase 10b Integration Tests
|
||||
//
|
||||
// Full agent annotation session: discover unannotated → set annotations →
|
||||
// query → remove → re-query → save sidecar. Tests multi-function workflow,
|
||||
// prompt template content, permission enforcement, and hint accuracy.
|
||||
#include "HeadlessEditorState.h"
|
||||
#include "MCPServer.h"
|
||||
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <filesystem>
|
||||
|
||||
static void expect(bool cond, const std::string& name,
|
||||
int& passed, int& failed) {
|
||||
if (cond) {
|
||||
std::cout << "Test " << (passed + failed + 1)
|
||||
<< " PASS: " << name << "\n";
|
||||
++passed;
|
||||
} else {
|
||||
std::cout << "Test " << (passed + failed + 1)
|
||||
<< " FAIL: " << name << "\n";
|
||||
++failed;
|
||||
}
|
||||
}
|
||||
|
||||
static json rpc(HeadlessEditorState& state, const std::string& session,
|
||||
const std::string& method,
|
||||
json params = json::object()) {
|
||||
json request = {{"jsonrpc", "2.0"}, {"id", 1},
|
||||
{"method", method}, {"params", params}};
|
||||
return state.processAgentRequest(request, session);
|
||||
}
|
||||
|
||||
int main() {
|
||||
int passed = 0;
|
||||
int failed = 0;
|
||||
|
||||
auto tmpDir = std::filesystem::temp_directory_path() /
|
||||
"whetstone_step271_test";
|
||||
std::filesystem::remove_all(tmpDir);
|
||||
std::filesystem::create_directories(tmpDir);
|
||||
|
||||
std::string src =
|
||||
"def validate(data):\n"
|
||||
" if not data:\n"
|
||||
" return False\n"
|
||||
" return True\n"
|
||||
"\n"
|
||||
"def transform(items):\n"
|
||||
" result = []\n"
|
||||
" for item in items:\n"
|
||||
" result.append(item * 2)\n"
|
||||
" return result\n"
|
||||
"\n"
|
||||
"def notify(user):\n"
|
||||
" print(\"hello \" + user)\n";
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 1: Full annotation workflow — discover → annotate → query
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.workspaceRoot = tmpDir.string();
|
||||
state.setAgentRole("agent", AgentRole::Refactor);
|
||||
|
||||
rpc(state, "agent", "openFile",
|
||||
{{"path", "app.py"}, {"content", src}});
|
||||
|
||||
// Step 1: Discover unannotated functions
|
||||
json unannotated = rpc(state, "agent", "getUnannotatedNodes",
|
||||
{{"includeHints", true}});
|
||||
int initialCount = unannotated["result"].value("count", 0);
|
||||
|
||||
// Step 2: Get function IDs
|
||||
std::string validateId, transformId, notifyId;
|
||||
for (auto* c : state.activeAST()->allChildren()) {
|
||||
if (c->conceptType == "Function") {
|
||||
std::string n = getNodeName(c);
|
||||
if (n == "validate") validateId = c->id;
|
||||
if (n == "transform") transformId = c->id;
|
||||
if (n == "notify") notifyId = c->id;
|
||||
}
|
||||
}
|
||||
|
||||
// Step 3: Annotate all 3 functions with intent
|
||||
rpc(state, "agent", "setSemanticAnnotation",
|
||||
{{"nodeId", validateId}, {"type", "intent"},
|
||||
{"fields", {{"summary", "validates input data"},
|
||||
{"category", "validation"}}}});
|
||||
rpc(state, "agent", "setSemanticAnnotation",
|
||||
{{"nodeId", transformId}, {"type", "intent"},
|
||||
{"fields", {{"summary", "doubles each item"},
|
||||
{"category", "transformation"}}}});
|
||||
rpc(state, "agent", "setSemanticAnnotation",
|
||||
{{"nodeId", notifyId}, {"type", "intent"},
|
||||
{"fields", {{"summary", "prints greeting"},
|
||||
{"category", "io"}}}});
|
||||
|
||||
// Step 4: Query — all should be annotated now
|
||||
json afterAnnotate = rpc(state, "agent", "getUnannotatedNodes");
|
||||
int afterCount = afterAnnotate["result"].value("count", 0);
|
||||
|
||||
// Step 5: Query all annotations project-wide
|
||||
json allAnnotations = rpc(state, "agent", "getSemanticAnnotations");
|
||||
int nodeCount = (int)allAnnotations["result"]["nodes"].size();
|
||||
|
||||
expect(initialCount == 3 && afterCount == 0 && nodeCount == 3,
|
||||
"Full workflow: discover " + std::to_string(initialCount) +
|
||||
" → annotate → " + std::to_string(afterCount) +
|
||||
" unannotated, " + std::to_string(nodeCount) + " annotated nodes",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 2: Remove and re-query workflow
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.workspaceRoot = tmpDir.string();
|
||||
state.setAgentRole("agent", AgentRole::Refactor);
|
||||
|
||||
rpc(state, "agent", "openFile",
|
||||
{{"path", "app.py"}, {"content", src}});
|
||||
|
||||
std::string validateId;
|
||||
for (auto* c : state.activeAST()->allChildren()) {
|
||||
if (c->conceptType == "Function" && getNodeName(c) == "validate") {
|
||||
validateId = c->id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Add intent + risk
|
||||
rpc(state, "agent", "setSemanticAnnotation",
|
||||
{{"nodeId", validateId}, {"type", "intent"},
|
||||
{"fields", {{"summary", "validates"}, {"category", "validation"}}}});
|
||||
rpc(state, "agent", "setSemanticAnnotation",
|
||||
{{"nodeId", validateId}, {"type", "risk"},
|
||||
{"fields", {{"level", "medium"}, {"reason", "multiple callers"},
|
||||
{"dependentCount", 5}}}});
|
||||
|
||||
// Verify 2 annotations
|
||||
json before = rpc(state, "agent", "getSemanticAnnotations",
|
||||
{{"nodeId", validateId}});
|
||||
int beforeCount = (int)before["result"]["annotations"].size();
|
||||
|
||||
// Remove risk
|
||||
rpc(state, "agent", "removeSemanticAnnotation",
|
||||
{{"nodeId", validateId}, {"type", "risk"}});
|
||||
|
||||
// Re-query: should have 1
|
||||
json after = rpc(state, "agent", "getSemanticAnnotations",
|
||||
{{"nodeId", validateId}});
|
||||
int afterCount = (int)after["result"]["annotations"].size();
|
||||
|
||||
// getUnannotatedNodes for risk should include validate now
|
||||
json unanRisk = rpc(state, "agent", "getUnannotatedNodes",
|
||||
{{"type", "risk"}});
|
||||
bool validateMissingRisk = false;
|
||||
for (const auto& n : unanRisk["result"]["nodes"]) {
|
||||
if (n.value("name", "") == "validate")
|
||||
validateMissingRisk = true;
|
||||
}
|
||||
|
||||
expect(beforeCount == 2 && afterCount == 1 && validateMissingRisk,
|
||||
"Remove risk → re-query shows 2→1, validate needs risk again",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 3: Annotation session with sidecar save/load roundtrip
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.workspaceRoot = tmpDir.string();
|
||||
state.setAgentRole("agent", AgentRole::Refactor);
|
||||
|
||||
rpc(state, "agent", "openFile",
|
||||
{{"path", "roundtrip.py"}, {"content", src}});
|
||||
|
||||
// Annotate via RPC
|
||||
for (auto* c : state.activeAST()->allChildren()) {
|
||||
if (c->conceptType != "Function") continue;
|
||||
std::string name = getNodeName(c);
|
||||
rpc(state, "agent", "setSemanticAnnotation",
|
||||
{{"nodeId", c->id}, {"type", "intent"},
|
||||
{"fields", {{"summary", name + " function"},
|
||||
{"category", "general"}}}});
|
||||
rpc(state, "agent", "setSemanticAnnotation",
|
||||
{{"nodeId", c->id}, {"type", "complexity"},
|
||||
{"fields", {{"timeComplexity", "O(n)"},
|
||||
{"cognitiveComplexity", 3},
|
||||
{"linesOfLogic", 4}}}});
|
||||
}
|
||||
|
||||
// Save sidecar
|
||||
json saveResp = rpc(state, "agent", "saveAnnotatedAST",
|
||||
{{"path", "roundtrip.py"}});
|
||||
int savedCount = saveResp["result"].value("annotationCount", 0);
|
||||
|
||||
// Load into fresh state
|
||||
HeadlessEditorState state2;
|
||||
state2.defaultLanguage = "python";
|
||||
state2.workspaceRoot = tmpDir.string();
|
||||
state2.setAgentRole("agent", AgentRole::Refactor);
|
||||
rpc(state2, "agent", "openFile",
|
||||
{{"path", "roundtrip.py"}, {"content", src}});
|
||||
rpc(state2, "agent", "loadAnnotatedAST", {{"path", "roundtrip.py"}});
|
||||
|
||||
// Query via RPC in new state
|
||||
json allAnnotations = rpc(state2, "agent", "getSemanticAnnotations");
|
||||
int restoredNodes = (int)allAnnotations["result"]["nodes"].size();
|
||||
|
||||
expect(savedCount >= 6 && restoredNodes == 3,
|
||||
"Sidecar roundtrip: saved " + std::to_string(savedCount) +
|
||||
" annotations, restored " + std::to_string(restoredNodes) +
|
||||
" annotated nodes",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 4: Permission enforcement across the pipeline
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.workspaceRoot = tmpDir.string();
|
||||
state.setAgentRole("admin", AgentRole::Refactor);
|
||||
state.setAgentRole("reader", AgentRole::Linter);
|
||||
|
||||
rpc(state, "admin", "openFile",
|
||||
{{"path", "perms.py"}, {"content", src}});
|
||||
|
||||
std::string fnId;
|
||||
for (auto* c : state.activeAST()->allChildren()) {
|
||||
if (c->conceptType == "Function") {
|
||||
fnId = c->id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Admin can set annotation
|
||||
json setResp = rpc(state, "admin", "setSemanticAnnotation",
|
||||
{{"nodeId", fnId}, {"type", "intent"},
|
||||
{"fields", {{"summary", "test"}, {"category", "io"}}}});
|
||||
bool adminSetOk = setResp.contains("result");
|
||||
|
||||
// Reader cannot set
|
||||
json readerSet = rpc(state, "reader", "setSemanticAnnotation",
|
||||
{{"nodeId", fnId}, {"type", "risk"},
|
||||
{"fields", {{"level", "low"}, {"reason", "test"},
|
||||
{"dependentCount", 0}}}});
|
||||
bool readerSetDenied = readerSet.contains("error");
|
||||
|
||||
// Reader cannot remove
|
||||
json readerRemove = rpc(state, "reader", "removeSemanticAnnotation",
|
||||
{{"nodeId", fnId}, {"type", "intent"}});
|
||||
bool readerRemoveDenied = readerRemove.contains("error");
|
||||
|
||||
// Reader CAN query
|
||||
json readerGet = rpc(state, "reader", "getSemanticAnnotations",
|
||||
{{"nodeId", fnId}});
|
||||
bool readerGetOk = readerGet.contains("result");
|
||||
|
||||
// Reader CAN get unannotated
|
||||
json readerUnan = rpc(state, "reader", "getUnannotatedNodes");
|
||||
bool readerUnanOk = readerUnan.contains("result");
|
||||
|
||||
expect(adminSetOk && readerSetDenied && readerRemoveDenied &&
|
||||
readerGetOk && readerUnanOk,
|
||||
"Linter: denied set/remove, allowed get/getUnannotated",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 5: Hint accuracy — sideEffectHint for io function
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.setAgentRole("agent", AgentRole::Refactor);
|
||||
|
||||
rpc(state, "agent", "openFile",
|
||||
{{"path", "hints.py"}, {"content", src}});
|
||||
|
||||
json resp = rpc(state, "agent", "getUnannotatedNodes",
|
||||
{{"includeHints", true}});
|
||||
|
||||
// notify() calls print(), so sideEffectHint should be "possible"
|
||||
std::string notifyHint;
|
||||
// validate() is pure, so sideEffectHint should be "none"
|
||||
std::string validateHint;
|
||||
for (const auto& n : resp["result"]["nodes"]) {
|
||||
if (n.value("name", "") == "notify" && n.contains("hints"))
|
||||
notifyHint = n["hints"].value("sideEffectHint", "");
|
||||
if (n.value("name", "") == "validate" && n.contains("hints"))
|
||||
validateHint = n["hints"].value("sideEffectHint", "");
|
||||
}
|
||||
|
||||
expect(notifyHint == "possible" && validateHint == "none",
|
||||
"Hint accuracy: notify='" + notifyHint +
|
||||
"' validate='" + validateHint + "'",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 6: Prompt template content validates workflow steps
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
MCPServer mcp;
|
||||
|
||||
// Each annotation prompt should have non-empty messages
|
||||
std::vector<std::string> promptNames = {
|
||||
"annotate_intent", "annotate_complexity",
|
||||
"annotate_risk", "annotate_contracts", "annotate_full"
|
||||
};
|
||||
|
||||
int validPrompts = 0;
|
||||
for (const auto& pname : promptNames) {
|
||||
json req = {{"jsonrpc", "2.0"}, {"id", 1},
|
||||
{"method", "prompts/get"},
|
||||
{"params", {{"name", pname},
|
||||
{"arguments", json::object()}}}};
|
||||
json resp = mcp.handleRequest(req);
|
||||
if (resp.contains("result") &&
|
||||
resp["result"].contains("messages") &&
|
||||
!resp["result"]["messages"].empty()) {
|
||||
std::string text = resp["result"]["messages"][0]
|
||||
["content"].value("text", "");
|
||||
if (text.size() > 50) ++validPrompts;
|
||||
}
|
||||
}
|
||||
|
||||
expect(validPrompts == 5,
|
||||
"All 5 prompt templates return substantive instructions (" +
|
||||
std::to_string(validPrompts) + "/5)",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 7: Multi-type annotation then compact AST verification
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.setAgentRole("agent", AgentRole::Refactor);
|
||||
|
||||
rpc(state, "agent", "openFile",
|
||||
{{"path", "compact.py"}, {"content", src}});
|
||||
|
||||
std::string transformId;
|
||||
for (auto* c : state.activeAST()->allChildren()) {
|
||||
if (c->conceptType == "Function" &&
|
||||
getNodeName(c) == "transform") {
|
||||
transformId = c->id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Set 3 annotation types via RPC
|
||||
rpc(state, "agent", "setSemanticAnnotation",
|
||||
{{"nodeId", transformId}, {"type", "intent"},
|
||||
{"fields", {{"summary", "doubles items"},
|
||||
{"category", "transformation"}}}});
|
||||
rpc(state, "agent", "setSemanticAnnotation",
|
||||
{{"nodeId", transformId}, {"type", "complexity"},
|
||||
{"fields", {{"timeComplexity", "O(n)"},
|
||||
{"cognitiveComplexity", 2},
|
||||
{"linesOfLogic", 4}}}});
|
||||
rpc(state, "agent", "setSemanticAnnotation",
|
||||
{{"nodeId", transformId}, {"type", "tags"},
|
||||
{"fields", {{"tags", json::array({"@collection", "@pure"})}}}});
|
||||
|
||||
// Verify compact AST shows all 3 in semantic field
|
||||
json resp = rpc(state, "agent", "getAST", {{"compact", true}});
|
||||
json nodes = resp["result"]["nodes"];
|
||||
|
||||
bool allPresent = false;
|
||||
for (const auto& n : nodes) {
|
||||
if (n.value("name", "") == "transform" &&
|
||||
n.contains("semantic")) {
|
||||
auto sem = n["semantic"];
|
||||
allPresent = sem.contains("intent") &&
|
||||
sem.contains("complexity") &&
|
||||
sem.contains("tags");
|
||||
}
|
||||
}
|
||||
|
||||
expect(allPresent,
|
||||
"Compact AST reflects 3 annotation types set via RPC",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Test 8: End-to-end: RPC annotate → save → load → RPC query
|
||||
// ---------------------------------------------------------------
|
||||
{
|
||||
HeadlessEditorState state;
|
||||
state.defaultLanguage = "python";
|
||||
state.workspaceRoot = tmpDir.string();
|
||||
state.setAgentRole("agent", AgentRole::Refactor);
|
||||
|
||||
rpc(state, "agent", "openFile",
|
||||
{{"path", "e2e.py"}, {"content", src}});
|
||||
|
||||
// Annotate validate with all 5 types via RPC
|
||||
std::string validateId;
|
||||
for (auto* c : state.activeAST()->allChildren()) {
|
||||
if (c->conceptType == "Function" &&
|
||||
getNodeName(c) == "validate") {
|
||||
validateId = c->id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
rpc(state, "agent", "setSemanticAnnotation",
|
||||
{{"nodeId", validateId}, {"type", "intent"},
|
||||
{"fields", {{"summary", "validates input"},
|
||||
{"category", "validation"}}}});
|
||||
rpc(state, "agent", "setSemanticAnnotation",
|
||||
{{"nodeId", validateId}, {"type", "complexity"},
|
||||
{"fields", {{"timeComplexity", "O(1)"},
|
||||
{"cognitiveComplexity", 2},
|
||||
{"linesOfLogic", 3}}}});
|
||||
rpc(state, "agent", "setSemanticAnnotation",
|
||||
{{"nodeId", validateId}, {"type", "risk"},
|
||||
{"fields", {{"level", "high"}, {"reason", "12 callers"},
|
||||
{"dependentCount", 12}}}});
|
||||
rpc(state, "agent", "setSemanticAnnotation",
|
||||
{{"nodeId", validateId}, {"type", "contract"},
|
||||
{"fields", {{"preconditions", "data not null"},
|
||||
{"postconditions", "returns bool"},
|
||||
{"returnShape", "bool"},
|
||||
{"sideEffects", "none"}}}});
|
||||
rpc(state, "agent", "setSemanticAnnotation",
|
||||
{{"nodeId", validateId}, {"type", "tags"},
|
||||
{"fields", {{"tags", json::array({"@validation", "@guard"})}}}});
|
||||
|
||||
// Save sidecar
|
||||
rpc(state, "agent", "saveAnnotatedAST", {{"path", "e2e.py"}});
|
||||
|
||||
// Load into fresh state and query via RPC
|
||||
HeadlessEditorState state2;
|
||||
state2.defaultLanguage = "python";
|
||||
state2.workspaceRoot = tmpDir.string();
|
||||
state2.setAgentRole("agent", AgentRole::Refactor);
|
||||
rpc(state2, "agent", "openFile",
|
||||
{{"path", "e2e.py"}, {"content", src}});
|
||||
rpc(state2, "agent", "loadAnnotatedAST", {{"path", "e2e.py"}});
|
||||
|
||||
// Find validate in new state
|
||||
std::string newValidateId;
|
||||
for (auto* c : state2.activeAST()->allChildren()) {
|
||||
if (c->conceptType == "Function" &&
|
||||
getNodeName(c) == "validate") {
|
||||
newValidateId = c->id;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
json resp = rpc(state2, "agent", "getSemanticAnnotations",
|
||||
{{"nodeId", newValidateId}});
|
||||
int annoCount = (int)resp["result"]["annotations"].size();
|
||||
|
||||
// Check all 5 types present
|
||||
bool hasI = false, hasCx = false, hasR = false,
|
||||
hasCt = false, hasT = false;
|
||||
for (const auto& a : resp["result"]["annotations"]) {
|
||||
std::string t = a.value("type", "");
|
||||
if (t == "intent") hasI = true;
|
||||
if (t == "complexity") hasCx = true;
|
||||
if (t == "risk") hasR = true;
|
||||
if (t == "contract") hasCt = true;
|
||||
if (t == "tags") hasT = true;
|
||||
}
|
||||
|
||||
expect(annoCount == 5 && hasI && hasCx && hasR && hasCt && hasT,
|
||||
"E2E: RPC annotate 5 types → save → load → RPC query (" +
|
||||
std::to_string(annoCount) + "/5)",
|
||||
passed, failed);
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
std::filesystem::remove_all(tmpDir);
|
||||
|
||||
std::cout << "\n=== Step 271 Results: " << passed << " passed, "
|
||||
<< failed << " failed ===\n";
|
||||
return failed == 0 ? 0 : 1;
|
||||
}
|
||||
Reference in New Issue
Block a user