367 lines
13 KiB
C++
367 lines
13 KiB
C++
|
// Step 319: Phase 11e Integration + Sprint Summary (8 tests)
|
||
|
|
// Full workflow annotation pipeline and sprint verification.
|
||
|
|
|
||
|
|
#include "HeadlessEditorState.h"
|
||
|
|
#include "HeadlessAgentRPCHandler.h"
|
||
|
|
#include "MCPServer.h"
|
||
|
|
#include "SemannoFormat.h"
|
||
|
|
#include <nlohmann/json.hpp>
|
||
|
|
#include <iostream>
|
||
|
|
#include <string>
|
||
|
|
|
||
|
|
using json = nlohmann::json;
|
||
|
|
|
||
|
|
static int passed = 0, failed = 0;
|
||
|
|
#define TEST(name) { std::cout << " " << #name << "... "; }
|
||
|
|
#define PASS() { std::cout << "PASS\n"; ++passed; }
|
||
|
|
#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; }
|
||
|
|
#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {}
|
||
|
|
|
||
|
|
static json rpcCall(HeadlessEditorState& state, const std::string& method,
|
||
|
|
const json& params = json::object()) {
|
||
|
|
json req = {{"jsonrpc", "2.0"}, {"id", 1}, {"method", method}};
|
||
|
|
if (!params.empty()) req["params"] = params;
|
||
|
|
return handleHeadlessAgentRequest(state, req, "test-session");
|
||
|
|
}
|
||
|
|
|
||
|
|
// 1. Parse Python → infer annotations across all 8 subjects → verify suggestions
|
||
|
|
void test_infer_all_subjects() {
|
||
|
|
TEST(infer_all_subjects);
|
||
|
|
HeadlessEditorState state;
|
||
|
|
state.defaultLanguage = "python";
|
||
|
|
state.openBuffer("test",
|
||
|
|
"def pure_fn():\n return 42\n\n"
|
||
|
|
"async def fetch():\n data = await get_data()\n return data\n\n"
|
||
|
|
"def complex():\n"
|
||
|
|
" for i in range(n):\n"
|
||
|
|
" for j in range(m):\n"
|
||
|
|
" process(i, j)\n"
|
||
|
|
" return result\n",
|
||
|
|
"python");
|
||
|
|
|
||
|
|
auto* ast = state.activeAST();
|
||
|
|
CHECK(ast != nullptr, "Has AST");
|
||
|
|
|
||
|
|
AnnotationInference inf;
|
||
|
|
auto all = inf.inferAll(ast);
|
||
|
|
CHECK(!all.empty(), "Has suggestions");
|
||
|
|
|
||
|
|
// Check annotation types cover multiple subjects
|
||
|
|
std::set<std::string> types;
|
||
|
|
for (const auto& a : all) types.insert(a.annotationType);
|
||
|
|
// Should have at least exec (async), complexity, pure annotations
|
||
|
|
CHECK(types.size() >= 3, "3+ annotation types, got: " + std::to_string(types.size()));
|
||
|
|
PASS();
|
||
|
|
}
|
||
|
|
|
||
|
|
// 2. Create skeleton → add annotated functions → get project model → verify task list
|
||
|
|
void test_skeleton_workflow() {
|
||
|
|
TEST(skeleton_workflow);
|
||
|
|
HeadlessEditorState state;
|
||
|
|
state.agent.defaultRole = AgentRole::Refactor;
|
||
|
|
rpcCall(state, "createSkeleton", {{"name", "webapp"}, {"language", "python"}});
|
||
|
|
|
||
|
|
rpcCall(state, "addSkeletonNode", {
|
||
|
|
{"name", "auth_handler"}, {"contextWidth", "file"},
|
||
|
|
{"automatability", "llm"}, {"priority", "critical"}});
|
||
|
|
rpcCall(state, "addSkeletonNode", {
|
||
|
|
{"name", "data_validator"}, {"contextWidth", "local"},
|
||
|
|
{"automatability", "template"}, {"priority", "high"}});
|
||
|
|
|
||
|
|
auto resp = rpcCall(state, "getProjectModel");
|
||
|
|
CHECK(resp.contains("result"), "Has result");
|
||
|
|
auto& r = resp["result"];
|
||
|
|
CHECK(r["totalNodes"].get<int>() == 2, "2 nodes");
|
||
|
|
CHECK(r["tasks"].is_array(), "Has tasks");
|
||
|
|
CHECK(r["tasks"].size() == 2, "2 tasks");
|
||
|
|
|
||
|
|
// Verify task fields
|
||
|
|
auto& t0 = r["tasks"][0];
|
||
|
|
CHECK(t0.contains("contextWidth"), "Task has contextWidth");
|
||
|
|
CHECK(t0.contains("automatability"), "Task has automatability");
|
||
|
|
CHECK(t0.contains("priority"), "Task has priority");
|
||
|
|
PASS();
|
||
|
|
}
|
||
|
|
|
||
|
|
// 3. Infer routing: simple → deterministic, complex → llm
|
||
|
|
void test_routing_inference() {
|
||
|
|
TEST(routing_inference);
|
||
|
|
// Simple getter
|
||
|
|
auto* simpleFn = new Function();
|
||
|
|
simpleFn->id = "simple_1";
|
||
|
|
simpleFn->name = "get_value";
|
||
|
|
auto* ret = new Return();
|
||
|
|
ret->id = "ret_1";
|
||
|
|
simpleFn->addChild("body", ret);
|
||
|
|
|
||
|
|
// Complex function with nested loops
|
||
|
|
auto* complexFn = new Function();
|
||
|
|
complexFn->id = "complex_1";
|
||
|
|
complexFn->name = "analyze";
|
||
|
|
auto* inner = new ForLoop();
|
||
|
|
inner->id = "inner_1";
|
||
|
|
auto* fc = new FunctionCall();
|
||
|
|
fc->id = "fc_1";
|
||
|
|
fc->functionName = "process";
|
||
|
|
inner->addChild("body", fc);
|
||
|
|
auto* outer = new ForLoop();
|
||
|
|
outer->id = "outer_1";
|
||
|
|
outer->addChild("body", inner);
|
||
|
|
complexFn->addChild("body", outer);
|
||
|
|
|
||
|
|
auto* mod = new Module("root", "test", "python");
|
||
|
|
mod->addChild("functions", simpleFn);
|
||
|
|
mod->addChild("functions", complexFn);
|
||
|
|
|
||
|
|
AnnotationInference inf;
|
||
|
|
auto simpleRouting = inf.inferRoutingAnnotations(simpleFn);
|
||
|
|
auto complexRouting = inf.inferRoutingAnnotations(complexFn);
|
||
|
|
|
||
|
|
bool simpleDet = false, complexLlm = false;
|
||
|
|
for (const auto& r : simpleRouting)
|
||
|
|
if (r.annotationType == "AutomatabilityAnnotation" && r.value == "deterministic")
|
||
|
|
simpleDet = true;
|
||
|
|
for (const auto& r : complexRouting)
|
||
|
|
if (r.annotationType == "AutomatabilityAnnotation" && r.value == "llm")
|
||
|
|
complexLlm = true;
|
||
|
|
|
||
|
|
CHECK(simpleDet, "Simple → deterministic");
|
||
|
|
CHECK(complexLlm, "Complex → llm");
|
||
|
|
delete mod;
|
||
|
|
PASS();
|
||
|
|
}
|
||
|
|
|
||
|
|
// 4. Skeleton task list respects dependency ordering from @Priority.blockedBy
|
||
|
|
void test_dependency_ordering() {
|
||
|
|
TEST(dependency_ordering);
|
||
|
|
HeadlessEditorState state;
|
||
|
|
state.agent.defaultRole = AgentRole::Refactor;
|
||
|
|
rpcCall(state, "createSkeleton", {{"name", "pipeline"}, {"language", "python"}});
|
||
|
|
|
||
|
|
rpcCall(state, "addSkeletonNode", {
|
||
|
|
{"name", "step_a"}, {"priority", "high"}});
|
||
|
|
rpcCall(state, "addSkeletonNode", {
|
||
|
|
{"name", "step_b"}, {"priority", "medium"},
|
||
|
|
{"blockedBy", json::array({"step_a"})}});
|
||
|
|
rpcCall(state, "addSkeletonNode", {
|
||
|
|
{"name", "step_c"}, {"priority", "low"},
|
||
|
|
{"blockedBy", json::array({"step_b"})}});
|
||
|
|
|
||
|
|
auto resp = rpcCall(state, "getProjectModel");
|
||
|
|
auto& tasks = resp["result"]["tasks"];
|
||
|
|
CHECK(tasks.size() == 3, "3 tasks");
|
||
|
|
|
||
|
|
// Find step_b and step_c, verify dependencies
|
||
|
|
bool bHasDep = false, cHasDep = false;
|
||
|
|
for (const auto& t : tasks) {
|
||
|
|
if (t["nodeName"] == "step_b" && t.contains("dependencies"))
|
||
|
|
bHasDep = t["dependencies"].size() == 1;
|
||
|
|
if (t["nodeName"] == "step_c" && t.contains("dependencies"))
|
||
|
|
cHasDep = t["dependencies"].size() == 1;
|
||
|
|
}
|
||
|
|
CHECK(bHasDep, "step_b blocked by step_a");
|
||
|
|
CHECK(cHasDep, "step_c blocked by step_b");
|
||
|
|
PASS();
|
||
|
|
}
|
||
|
|
|
||
|
|
// 5. MCP tools/list returns 42+ tools with 4 workflow tools present
|
||
|
|
void test_mcp_tool_inventory() {
|
||
|
|
TEST(mcp_tool_inventory);
|
||
|
|
MCPServer server;
|
||
|
|
json initReq = {{"jsonrpc", "2.0"}, {"id", 1}, {"method", "initialize"},
|
||
|
|
{"params", {{"protocolVersion", "2024-11-05"},
|
||
|
|
{"capabilities", json::object()},
|
||
|
|
{"clientInfo", {{"name", "test"}}}}}};
|
||
|
|
server.handleRequest(initReq);
|
||
|
|
|
||
|
|
json listReq = {{"jsonrpc", "2.0"}, {"id", 2}, {"method", "tools/list"}};
|
||
|
|
json resp = server.handleRequest(listReq);
|
||
|
|
auto& tools = resp["result"]["tools"];
|
||
|
|
int count = (int)tools.size();
|
||
|
|
CHECK(count >= 42, "42+ tools, got: " + std::to_string(count));
|
||
|
|
|
||
|
|
// Verify all 4 workflow tools
|
||
|
|
std::set<std::string> names;
|
||
|
|
for (const auto& t : tools) names.insert(t["name"].get<std::string>());
|
||
|
|
CHECK(names.count("whetstone_create_skeleton"), "Has create_skeleton");
|
||
|
|
CHECK(names.count("whetstone_add_skeleton_node"), "Has add_skeleton_node");
|
||
|
|
CHECK(names.count("whetstone_get_project_model"), "Has get_project_model");
|
||
|
|
CHECK(names.count("whetstone_infer_annotations"), "Has infer_annotations");
|
||
|
|
PASS();
|
||
|
|
}
|
||
|
|
|
||
|
|
// 6. Sprint totals: 10 parsers, 10 generators, 42+ MCP tools
|
||
|
|
void test_sprint_totals() {
|
||
|
|
TEST(sprint_totals);
|
||
|
|
// 10 parsers: python, cpp, rust, go, java, js, ts, elisp, kotlin, csharp
|
||
|
|
Pipeline pipeline;
|
||
|
|
std::vector<std::string> langs = {
|
||
|
|
"python", "cpp", "rust", "go", "java",
|
||
|
|
"javascript", "typescript", "elisp", "kotlin", "csharp"
|
||
|
|
};
|
||
|
|
int parseCount = 0;
|
||
|
|
for (const auto& lang : langs) {
|
||
|
|
std::vector<ParseDiagnostic> diags;
|
||
|
|
auto mod = pipeline.parse("x = 1", lang, diags);
|
||
|
|
if (mod) parseCount++;
|
||
|
|
}
|
||
|
|
CHECK(parseCount == 10, "10 parsers, got: " + std::to_string(parseCount));
|
||
|
|
|
||
|
|
// 10 generators (same languages)
|
||
|
|
int genCount = 0;
|
||
|
|
for (const auto& lang : langs) {
|
||
|
|
auto mod = std::make_unique<Module>("root", "test", lang);
|
||
|
|
std::string code = pipeline.generate(mod.get(), lang);
|
||
|
|
genCount++; // generate always returns something
|
||
|
|
}
|
||
|
|
CHECK(genCount == 10, "10 generators");
|
||
|
|
|
||
|
|
// 42+ MCP tools
|
||
|
|
MCPServer server;
|
||
|
|
CHECK((int)server.getTools().size() >= 42,
|
||
|
|
"42+ MCP tools, got: " + std::to_string(server.getTools().size()));
|
||
|
|
|
||
|
|
// Verify no annotation types dropped — check Subject 9 types exist
|
||
|
|
auto* cw = new ContextWidthAnnotation();
|
||
|
|
auto* rv = new ReviewAnnotation();
|
||
|
|
auto* aa = new AutomatabilityAnnotation();
|
||
|
|
auto* pa = new PriorityAnnotation();
|
||
|
|
auto* is = new ImplementationStatusAnnotation();
|
||
|
|
CHECK(cw->conceptType == "ContextWidthAnnotation", "ContextWidth type");
|
||
|
|
CHECK(rv->conceptType == "ReviewAnnotation", "Review type");
|
||
|
|
CHECK(aa->conceptType == "AutomatabilityAnnotation", "Automatability type");
|
||
|
|
CHECK(pa->conceptType == "PriorityAnnotation", "Priority type");
|
||
|
|
CHECK(is->conceptType == "ImplementationStatusAnnotation", "ImplStatus type");
|
||
|
|
delete cw; delete rv; delete aa; delete pa; delete is;
|
||
|
|
PASS();
|
||
|
|
}
|
||
|
|
|
||
|
|
// 7. Inference + routing + skeleton combined
|
||
|
|
void test_combined_workflow() {
|
||
|
|
TEST(combined_workflow);
|
||
|
|
HeadlessEditorState state;
|
||
|
|
state.agent.defaultRole = AgentRole::Refactor;
|
||
|
|
|
||
|
|
// Create skeleton
|
||
|
|
rpcCall(state, "createSkeleton", {{"name", "myservice"}, {"language", "python"}});
|
||
|
|
rpcCall(state, "addSkeletonNode", {
|
||
|
|
{"name", "handle_request"}, {"contextWidth", "file"},
|
||
|
|
{"automatability", "llm"}, {"priority", "critical"}});
|
||
|
|
rpcCall(state, "addSkeletonNode", {
|
||
|
|
{"name", "validate_input"}, {"contextWidth", "local"},
|
||
|
|
{"automatability", "deterministic"}, {"priority", "high"}});
|
||
|
|
|
||
|
|
// Get project model
|
||
|
|
auto modelResp = rpcCall(state, "getProjectModel");
|
||
|
|
CHECK(modelResp.contains("result"), "Has project model");
|
||
|
|
CHECK(modelResp["result"]["totalNodes"].get<int>() == 2, "2 nodes");
|
||
|
|
|
||
|
|
// Now open a different buffer with actual code and infer
|
||
|
|
state.openBuffer("impl",
|
||
|
|
"def handle_request(req):\n"
|
||
|
|
" validated = validate_input(req)\n"
|
||
|
|
" result = process(validated)\n"
|
||
|
|
" return result\n",
|
||
|
|
"python");
|
||
|
|
state.setActiveBuffer("impl");
|
||
|
|
|
||
|
|
auto inferResp = rpcCall(state, "inferAnnotations");
|
||
|
|
CHECK(inferResp.contains("result"), "Has infer result");
|
||
|
|
CHECK(inferResp["result"]["suggestions"].is_array(), "Has suggestions");
|
||
|
|
|
||
|
|
// Infer routing on parsed functions
|
||
|
|
auto* ast = state.activeAST();
|
||
|
|
CHECK(ast != nullptr, "Has AST");
|
||
|
|
auto fns = ast->getChildren("functions");
|
||
|
|
if (!fns.empty()) {
|
||
|
|
AnnotationInference inf;
|
||
|
|
auto routing = inf.inferRoutingAnnotations(fns[0]);
|
||
|
|
CHECK(!routing.empty(), "Has routing annotations");
|
||
|
|
std::string worker = inf.suggestWorkerType(routing);
|
||
|
|
CHECK(!worker.empty(), "Has worker suggestion");
|
||
|
|
}
|
||
|
|
PASS();
|
||
|
|
}
|
||
|
|
|
||
|
|
// 8. Semanno roundtrip includes workflow annotations (Subject 9)
|
||
|
|
void test_semanno_workflow_roundtrip() {
|
||
|
|
TEST(semanno_workflow_roundtrip);
|
||
|
|
auto* fn = new Function();
|
||
|
|
fn->id = "fn_rt_1";
|
||
|
|
fn->name = "routed_function";
|
||
|
|
|
||
|
|
// Add Subject 9 annotations
|
||
|
|
auto* cw = new ContextWidthAnnotation();
|
||
|
|
cw->id = "cw_rt_1";
|
||
|
|
cw->width = "file";
|
||
|
|
fn->addChild("annotations", cw);
|
||
|
|
|
||
|
|
auto* aa = new AutomatabilityAnnotation();
|
||
|
|
aa->id = "aa_rt_1";
|
||
|
|
aa->strategy = "llm";
|
||
|
|
aa->confidence = 0.85;
|
||
|
|
fn->addChild("annotations", aa);
|
||
|
|
|
||
|
|
auto* pa = new PriorityAnnotation();
|
||
|
|
pa->id = "pa_rt_1";
|
||
|
|
pa->level = "critical";
|
||
|
|
fn->addChild("annotations", pa);
|
||
|
|
|
||
|
|
auto* rv = new ReviewAnnotation();
|
||
|
|
rv->id = "rv_rt_1";
|
||
|
|
rv->required = true;
|
||
|
|
rv->reason = "production code";
|
||
|
|
fn->addChild("annotations", rv);
|
||
|
|
|
||
|
|
auto* mod = new Module("root", "test", "python");
|
||
|
|
mod->addChild("functions", fn);
|
||
|
|
|
||
|
|
// Emit to Semanno format
|
||
|
|
SemannoEmitter emitter;
|
||
|
|
std::string semanno = emitter.emit(mod);
|
||
|
|
|
||
|
|
// Verify Subject 9 annotations appear in output
|
||
|
|
CHECK(semanno.find("contextwidth") != std::string::npos ||
|
||
|
|
semanno.find("ContextWidth") != std::string::npos,
|
||
|
|
"Semanno contains contextwidth");
|
||
|
|
CHECK(semanno.find("automatability") != std::string::npos ||
|
||
|
|
semanno.find("Automatability") != std::string::npos,
|
||
|
|
"Semanno contains automatability");
|
||
|
|
CHECK(semanno.find("priority") != std::string::npos ||
|
||
|
|
semanno.find("Priority") != std::string::npos,
|
||
|
|
"Semanno contains priority");
|
||
|
|
CHECK(semanno.find("review") != std::string::npos ||
|
||
|
|
semanno.find("Review") != std::string::npos,
|
||
|
|
"Semanno contains review");
|
||
|
|
|
||
|
|
// Verify serialization roundtrip
|
||
|
|
json astJson = toJson(mod);
|
||
|
|
std::string serialized = astJson.dump();
|
||
|
|
CHECK(serialized.find("ContextWidthAnnotation") != std::string::npos,
|
||
|
|
"JSON serialization includes ContextWidth");
|
||
|
|
CHECK(serialized.find("AutomatabilityAnnotation") != std::string::npos,
|
||
|
|
"JSON serialization includes Automatability");
|
||
|
|
|
||
|
|
delete mod;
|
||
|
|
PASS();
|
||
|
|
}
|
||
|
|
|
||
|
|
int main() {
|
||
|
|
std::cout << "Step 319: Phase 11e Integration + Sprint Summary\n";
|
||
|
|
try {
|
||
|
|
test_infer_all_subjects();
|
||
|
|
test_skeleton_workflow();
|
||
|
|
test_routing_inference();
|
||
|
|
test_dependency_ordering();
|
||
|
|
test_mcp_tool_inventory();
|
||
|
|
test_sprint_totals();
|
||
|
|
test_combined_workflow();
|
||
|
|
test_semanno_workflow_roundtrip();
|
||
|
|
} catch (const std::exception& e) {
|
||
|
|
std::cout << "\nFATAL: " << e.what() << "\n";
|
||
|
|
return 1;
|
||
|
|
}
|
||
|
|
std::cout << "\nResults: " << passed << "/" << (passed + failed) << " passed\n";
|
||
|
|
return failed > 0 ? 1 : 0;
|
||
|
|
}
|