From 718c99fef95ef48ee5ce9ca06024c2f004422c50 Mon Sep 17 00:00:00 2001 From: Bill Date: Tue, 10 Feb 2026 08:23:48 -0700 Subject: [PATCH] Step 229: Add tool definition tests --- PROGRESS.md | 1 + editor/CMakeLists.txt | 4 + editor/tests/step229_test.cpp | 202 +++++++++++++++++++++++++++++++++ sprint7_plan.md | 2 +- tools/claude/system_prompt.txt | 12 ++ tools/openai/system_prompt.txt | 12 ++ 6 files changed, 232 insertions(+), 1 deletion(-) create mode 100644 editor/tests/step229_test.cpp diff --git a/PROGRESS.md b/PROGRESS.md index fa46c2b..d821a8b 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -748,3 +748,4 @@ vcpkg's imgui 1.91.9 removed the `sdl2-binding` feature (only `sdl3-binding` exi | 2026-02-10 | Claude Opus 4.6 | Sprint 7 Phase 7c (Steps 214–219): TraceGenerator.h (6 scenario templates, 9-sample code corpus, batch engine), TraceExporter.h (Anthropic/OpenAI/JSONL/Markdown export, filtering, statistics). 294/294 tests pass. | | 2026-02-10 | Codex | Step 227: Open-source model tool definitions (ReAct-style prompt, XML tool call format, simplified schemas, adapter rules). 1/1 tests pass. | | 2026-02-10 | Codex | Step 228: Prompt engineering templates (5 prompt files with system/user templates, expected tools, success criteria). 1/1 tests pass. | +| 2026-02-10 | Codex | Step 229: Tool definition tests (schema alignment, prompt coverage, examples validation, prompt variable checks). 1/1 tests pass. | diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 9b7d746..9be8795 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -1168,6 +1168,10 @@ add_executable(step227_test tests/step227_test.cpp) target_include_directories(step227_test PRIVATE src) add_executable(step228_test tests/step228_test.cpp) target_include_directories(step228_test PRIVATE src) +target_link_libraries(step228_test PRIVATE nlohmann_json::nlohmann_json) +add_executable(step229_test tests/step229_test.cpp) +target_include_directories(step229_test PRIVATE src) +target_link_libraries(step229_test PRIVATE nlohmann_json::nlohmann_json) add_executable(step213_test tests/step213_test.cpp) target_include_directories(step213_test PRIVATE src) target_link_libraries(step213_test PRIVATE nlohmann_json::nlohmann_json) diff --git a/editor/tests/step229_test.cpp b/editor/tests/step229_test.cpp new file mode 100644 index 0000000..79df47f --- /dev/null +++ b/editor/tests/step229_test.cpp @@ -0,0 +1,202 @@ +// Step 229: Tool definition tests. + +#include +#include +#include +#include +#include +#include + +#include + +using json = nlohmann::json; + +struct ToolSpec { + std::set props; + std::set required; +}; + +static std::string readFile(const std::string& path) { + std::ifstream f(path); + if (!f.is_open()) return {}; + return std::string((std::istreambuf_iterator(f)), + std::istreambuf_iterator()); +} + +static json readJson(const std::string& path) { + std::ifstream f(path); + if (!f.is_open()) return json(); + json j; + f >> j; + return j; +} + +static void assertContains(const std::string& text, const std::string& needle) { + assert(text.find(needle) != std::string::npos); +} + +static void assertSubset(const std::set& expected, + const std::set& actual) { + for (const auto& item : expected) { + assert(actual.count(item) == 1); + } +} + +static std::map expectedToolSpecs() { + std::map specs; + specs["whetstone_get_ast"] = {{}, {}}; + specs["whetstone_mutate"] = {{"type", "nodeId", "property", "value", "parentId", "role", "node"}, {"type"}}; + specs["whetstone_batch_mutate"] = {{"mutations"}, {"mutations"}}; + specs["whetstone_get_scope"] = {{"nodeId"}, {"nodeId"}}; + specs["whetstone_get_call_hierarchy"] = {{"functionId"}, {"functionId"}}; + specs["whetstone_suggest_annotations"] = {{"nodeId", "line", "col"}, {}}; + specs["whetstone_apply_annotation"] = {{"nodeId", "annotationType", "strategy", "reason", "confidence"}, {"nodeId", "annotationType", "strategy"}}; + specs["whetstone_generate_code"] = {{"spec", "preferImports"}, {"spec"}}; + specs["whetstone_run_pipeline"] = {{"source", "sourceLanguage", "targetLanguage"}, {"source", "sourceLanguage", "targetLanguage"}}; + specs["whetstone_project_language"] = {{"targetLanguage"}, {"targetLanguage"}}; + return specs; +} + +static std::map parseClaudeTools(const json& j) { + std::map result; + for (const auto& tool : j.at("tools")) { + ToolSpec spec; + const auto& schema = tool.at("input_schema"); + if (schema.contains("properties")) { + for (auto it = schema.at("properties").begin(); it != schema.at("properties").end(); ++it) { + spec.props.insert(it.key()); + } + } + if (schema.contains("required")) { + for (const auto& req : schema.at("required")) { + spec.required.insert(req.get()); + } + } + result[tool.at("name").get()] = spec; + } + return result; +} + +static std::map parseOpenAITools(const json& j) { + std::map result; + for (const auto& tool : j.at("tools")) { + const auto& fn = tool.at("function"); + ToolSpec spec; + const auto& schema = fn.at("parameters"); + if (schema.contains("properties")) { + for (auto it = schema.at("properties").begin(); it != schema.at("properties").end(); ++it) { + spec.props.insert(it.key()); + } + } + if (schema.contains("required")) { + for (const auto& req : schema.at("required")) { + spec.required.insert(req.get()); + } + } + result[fn.at("name").get()] = spec; + } + return result; +} + +static std::map parseGenericTools(const json& j) { + std::map result; + for (const auto& tool : j.at("tools")) { + ToolSpec spec; + if (tool.contains("args")) { + for (const auto& arg : tool.at("args")) { + spec.props.insert(arg.at("name").get()); + if (arg.value("required", false)) { + spec.required.insert(arg.at("name").get()); + } + } + } + result[tool.at("name").get()] = spec; + } + return result; +} + +static void assertToolCoverage(const std::map& tools, + const std::map& expected) { + for (const auto& kv : expected) { + assert(tools.count(kv.first) == 1); + } +} + +static void assertToolSchemas(const std::map& tools, + const std::map& expected) { + for (const auto& kv : expected) { + const auto& name = kv.first; + const auto& expectedSpec = kv.second; + const auto& actualSpec = tools.at(name); + assertSubset(expectedSpec.props, actualSpec.props); + assertSubset(expectedSpec.required, actualSpec.required); + assert(expectedSpec.required.size() == actualSpec.required.size()); + } +} + +static void assertPromptMentionsTools(const std::string& prompt, + const std::map& expected) { + for (const auto& kv : expected) { + assertContains(prompt, kv.first); + } +} + +static void assertClaudeExamplesValid(const json& j, const std::map& expected) { + for (const auto& tool : j.at("tools")) { + const auto name = tool.at("name").get(); + if (!tool.contains("examples")) { + continue; + } + const auto& exp = expected.at(name); + for (const auto& example : tool.at("examples")) { + assert(example.is_object()); + for (const auto& req : exp.required) { + assert(example.contains(req)); + } + } + } +} + +static void assertPromptTemplateVars(const std::string& path, + const std::vector& vars) { + const std::string content = readFile(path); + assert(!content.empty()); + for (const auto& var : vars) { + assertContains(content, "{{" + var + "}}"); + } +} + +int main() { + const auto expected = expectedToolSpecs(); + + const json claude = readJson("../tools/claude/tools.json"); + const json openai = readJson("../tools/openai/functions.json"); + const json generic = readJson("../tools/generic/tools.json"); + + const auto claudeTools = parseClaudeTools(claude); + const auto openaiTools = parseOpenAITools(openai); + const auto genericTools = parseGenericTools(generic); + + assertToolCoverage(claudeTools, expected); + assertToolCoverage(openaiTools, expected); + assertToolCoverage(genericTools, expected); + + assertToolSchemas(claudeTools, expected); + assertToolSchemas(openaiTools, expected); + assertToolSchemas(genericTools, expected); + + assertPromptMentionsTools(readFile("../tools/claude/system_prompt.txt"), expected); + assertPromptMentionsTools(readFile("../tools/openai/system_prompt.txt"), expected); + assertPromptMentionsTools(readFile("../tools/generic/system_prompt.txt"), expected); + + assertClaudeExamplesValid(claude, expected); + + assertPromptTemplateVars("../tools/prompts/annotate_module.prompt", {"module_name"}); + assertPromptTemplateVars("../tools/prompts/cross_language.prompt", {"source_language", "target_language"}); + assertPromptTemplateVars("../tools/prompts/code_review.prompt", {"module_name"}); + assertPromptTemplateVars("../tools/prompts/refactor.prompt", {"module_name", "goal"}); + assertPromptTemplateVars("../tools/prompts/security.prompt", {"module_name"}); + + printf("step229_test: all assertions passed\n"); + return 0; +} diff --git a/sprint7_plan.md b/sprint7_plan.md index 476f084..6571d95 100644 --- a/sprint7_plan.md +++ b/sprint7_plan.md @@ -386,7 +386,7 @@ Optimized tool definitions and prompts for specific LLM families. expected tool sequence, success criteria. *New:* `tools/prompts/` directory with 5+ prompt templates -- [ ] **Step 229: Tool definition tests** +- [x] **Step 229: Tool definition tests** Tests verifying tool definitions are correct and complete: 1. All MCP tools have corresponding Claude/OpenAI/generic definitions 2. Tool input schemas match MCP tool schemas diff --git a/tools/claude/system_prompt.txt b/tools/claude/system_prompt.txt index 2b8b920..2994f49 100644 --- a/tools/claude/system_prompt.txt +++ b/tools/claude/system_prompt.txt @@ -1,5 +1,17 @@ You are a Whetstone coding assistant with access to semantic annotations and AST tools. +Available tools: +- whetstone_get_ast +- whetstone_mutate +- whetstone_batch_mutate +- whetstone_get_scope +- whetstone_get_call_hierarchy +- whetstone_suggest_annotations +- whetstone_apply_annotation +- whetstone_generate_code +- whetstone_run_pipeline +- whetstone_project_language + Guidelines: - Use tools to read or mutate the AST instead of guessing structure. - Prefer safe, minimal mutations. Validate after multi-step changes. diff --git a/tools/openai/system_prompt.txt b/tools/openai/system_prompt.txt index 9387edf..4eeae6f 100644 --- a/tools/openai/system_prompt.txt +++ b/tools/openai/system_prompt.txt @@ -1,5 +1,17 @@ You are a Whetstone coding assistant with access to AST and annotation tools. +Available tools: +- whetstone_get_ast +- whetstone_mutate +- whetstone_batch_mutate +- whetstone_get_scope +- whetstone_get_call_hierarchy +- whetstone_suggest_annotations +- whetstone_apply_annotation +- whetstone_generate_code +- whetstone_run_pipeline +- whetstone_project_language + Rules: - Prefer tool calls to inspect or modify ASTs rather than guessing. - Keep edits minimal and validate after multi-step changes.