Step 229: Add tool definition tests

PROGRESS.md

@@ -748,3 +748,4 @@ vcpkg's imgui 1.91.9 removed the `sdl2-binding` feature (only `sdl3-binding` exi
 | 2026-02-10 | Claude Opus 4.6 | Sprint 7 Phase 7c (Steps 214–219): TraceGenerator.h (6 scenario templates, 9-sample code corpus, batch engine), TraceExporter.h (Anthropic/OpenAI/JSONL/Markdown export, filtering, statistics). 294/294 tests pass. |
 | 2026-02-10 | Codex | Step 227: Open-source model tool definitions (ReAct-style prompt, XML tool call format, simplified schemas, adapter rules). 1/1 tests pass. |
 | 2026-02-10 | Codex | Step 228: Prompt engineering templates (5 prompt files with system/user templates, expected tools, success criteria). 1/1 tests pass. |
+| 2026-02-10 | Codex | Step 229: Tool definition tests (schema alignment, prompt coverage, examples validation, prompt variable checks). 1/1 tests pass. |

editor/CMakeLists.txt

@@ -1168,6 +1168,10 @@ add_executable(step227_test tests/step227_test.cpp)
 target_include_directories(step227_test PRIVATE src)
 add_executable(step228_test tests/step228_test.cpp)
 target_include_directories(step228_test PRIVATE src)
+target_link_libraries(step228_test PRIVATE nlohmann_json::nlohmann_json)
+add_executable(step229_test tests/step229_test.cpp)
+target_include_directories(step229_test PRIVATE src)
+target_link_libraries(step229_test PRIVATE nlohmann_json::nlohmann_json)
 add_executable(step213_test tests/step213_test.cpp)
 target_include_directories(step213_test PRIVATE src)
 target_link_libraries(step213_test PRIVATE nlohmann_json::nlohmann_json)

editor/tests/step229_test.cpp

@@ -0,0 +1,202 @@
+// Step 229: Tool definition tests.
+
+#include <cassert>
+#include <fstream>
+#include <map>
+#include <set>
+#include <string>
+#include <vector>
+
+#include <nlohmann/json.hpp>
+
+using json = nlohmann::json;
+
+struct ToolSpec {
+    std::set<std::string> props;
+    std::set<std::string> required;
+};
+
+static std::string readFile(const std::string& path) {
+    std::ifstream f(path);
+    if (!f.is_open()) return {};
+    return std::string((std::istreambuf_iterator<char>(f)),
+                       std::istreambuf_iterator<char>());
+}
+
+static json readJson(const std::string& path) {
+    std::ifstream f(path);
+    if (!f.is_open()) return json();
+    json j;
+    f >> j;
+    return j;
+}
+
+static void assertContains(const std::string& text, const std::string& needle) {
+    assert(text.find(needle) != std::string::npos);
+}
+
+static void assertSubset(const std::set<std::string>& expected,
+                         const std::set<std::string>& actual) {
+    for (const auto& item : expected) {
+        assert(actual.count(item) == 1);
+    }
+}
+
+static std::map<std::string, ToolSpec> expectedToolSpecs() {
+    std::map<std::string, ToolSpec> specs;
+    specs["whetstone_get_ast"] = {{}, {}};
+    specs["whetstone_mutate"] = {{"type", "nodeId", "property", "value", "parentId", "role", "node"}, {"type"}};
+    specs["whetstone_batch_mutate"] = {{"mutations"}, {"mutations"}};
+    specs["whetstone_get_scope"] = {{"nodeId"}, {"nodeId"}};
+    specs["whetstone_get_call_hierarchy"] = {{"functionId"}, {"functionId"}};
+    specs["whetstone_suggest_annotations"] = {{"nodeId", "line", "col"}, {}};
+    specs["whetstone_apply_annotation"] = {{"nodeId", "annotationType", "strategy", "reason", "confidence"}, {"nodeId", "annotationType", "strategy"}};
+    specs["whetstone_generate_code"] = {{"spec", "preferImports"}, {"spec"}};
+    specs["whetstone_run_pipeline"] = {{"source", "sourceLanguage", "targetLanguage"}, {"source", "sourceLanguage", "targetLanguage"}};
+    specs["whetstone_project_language"] = {{"targetLanguage"}, {"targetLanguage"}};
+    return specs;
+}
+
+static std::map<std::string, ToolSpec> parseClaudeTools(const json& j) {
+    std::map<std::string, ToolSpec> result;
+    for (const auto& tool : j.at("tools")) {
+        ToolSpec spec;
+        const auto& schema = tool.at("input_schema");
+        if (schema.contains("properties")) {
+            for (auto it = schema.at("properties").begin(); it != schema.at("properties").end(); ++it) {
+                spec.props.insert(it.key());
+            }
+        }
+        if (schema.contains("required")) {
+            for (const auto& req : schema.at("required")) {
+                spec.required.insert(req.get<std::string>());
+            }
+        }
+        result[tool.at("name").get<std::string>()] = spec;
+    }
+    return result;
+}
+
+static std::map<std::string, ToolSpec> parseOpenAITools(const json& j) {
+    std::map<std::string, ToolSpec> result;
+    for (const auto& tool : j.at("tools")) {
+        const auto& fn = tool.at("function");
+        ToolSpec spec;
+        const auto& schema = fn.at("parameters");
+        if (schema.contains("properties")) {
+            for (auto it = schema.at("properties").begin(); it != schema.at("properties").end(); ++it) {
+                spec.props.insert(it.key());
+            }
+        }
+        if (schema.contains("required")) {
+            for (const auto& req : schema.at("required")) {
+                spec.required.insert(req.get<std::string>());
+            }
+        }
+        result[fn.at("name").get<std::string>()] = spec;
+    }
+    return result;
+}
+
+static std::map<std::string, ToolSpec> parseGenericTools(const json& j) {
+    std::map<std::string, ToolSpec> result;
+    for (const auto& tool : j.at("tools")) {
+        ToolSpec spec;
+        if (tool.contains("args")) {
+            for (const auto& arg : tool.at("args")) {
+                spec.props.insert(arg.at("name").get<std::string>());
+                if (arg.value("required", false)) {
+                    spec.required.insert(arg.at("name").get<std::string>());
+                }
+            }
+        }
+        result[tool.at("name").get<std::string>()] = spec;
+    }
+    return result;
+}
+
+static void assertToolCoverage(const std::map<std::string, ToolSpec>& tools,
+                               const std::map<std::string, ToolSpec>& expected) {
+    for (const auto& kv : expected) {
+        assert(tools.count(kv.first) == 1);
+    }
+}
+
+static void assertToolSchemas(const std::map<std::string, ToolSpec>& tools,
+                              const std::map<std::string, ToolSpec>& expected) {
+    for (const auto& kv : expected) {
+        const auto& name = kv.first;
+        const auto& expectedSpec = kv.second;
+        const auto& actualSpec = tools.at(name);
+        assertSubset(expectedSpec.props, actualSpec.props);
+        assertSubset(expectedSpec.required, actualSpec.required);
+        assert(expectedSpec.required.size() == actualSpec.required.size());
+    }
+}
+
+static void assertPromptMentionsTools(const std::string& prompt,
+                                      const std::map<std::string, ToolSpec>& expected) {
+    for (const auto& kv : expected) {
+        assertContains(prompt, kv.first);
+    }
+}
+
+static void assertClaudeExamplesValid(const json& j, const std::map<std::string, ToolSpec>& expected) {
+    for (const auto& tool : j.at("tools")) {
+        const auto name = tool.at("name").get<std::string>();
+        if (!tool.contains("examples")) {
+            continue;
+        }
+        const auto& exp = expected.at(name);
+        for (const auto& example : tool.at("examples")) {
+            assert(example.is_object());
+            for (const auto& req : exp.required) {
+                assert(example.contains(req));
+            }
+        }
+    }
+}
+
+static void assertPromptTemplateVars(const std::string& path,
+                                     const std::vector<std::string>& vars) {
+    const std::string content = readFile(path);
+    assert(!content.empty());
+    for (const auto& var : vars) {
+        assertContains(content, "{{" + var + "}}");
+    }
+}
+
+int main() {
+    const auto expected = expectedToolSpecs();
+
+    const json claude = readJson("../tools/claude/tools.json");
+    const json openai = readJson("../tools/openai/functions.json");
+    const json generic = readJson("../tools/generic/tools.json");
+
+    const auto claudeTools = parseClaudeTools(claude);
+    const auto openaiTools = parseOpenAITools(openai);
+    const auto genericTools = parseGenericTools(generic);
+
+    assertToolCoverage(claudeTools, expected);
+    assertToolCoverage(openaiTools, expected);
+    assertToolCoverage(genericTools, expected);
+
+    assertToolSchemas(claudeTools, expected);
+    assertToolSchemas(openaiTools, expected);
+    assertToolSchemas(genericTools, expected);
+
+    assertPromptMentionsTools(readFile("../tools/claude/system_prompt.txt"), expected);
+    assertPromptMentionsTools(readFile("../tools/openai/system_prompt.txt"), expected);
+    assertPromptMentionsTools(readFile("../tools/generic/system_prompt.txt"), expected);
+
+    assertClaudeExamplesValid(claude, expected);
+
+    assertPromptTemplateVars("../tools/prompts/annotate_module.prompt", {"module_name"});
+    assertPromptTemplateVars("../tools/prompts/cross_language.prompt", {"source_language", "target_language"});
+    assertPromptTemplateVars("../tools/prompts/code_review.prompt", {"module_name"});
+    assertPromptTemplateVars("../tools/prompts/refactor.prompt", {"module_name", "goal"});
+    assertPromptTemplateVars("../tools/prompts/security.prompt", {"module_name"});
+
+    printf("step229_test: all assertions passed\n");
+    return 0;
+}

sprint7_plan.md

@@ -386,7 +386,7 @@ Optimized tool definitions and prompts for specific LLM families.
   expected tool sequence, success criteria.
   *New:* `tools/prompts/` directory with 5+ prompt templates
 
-- [ ] **Step 229: Tool definition tests**
+- [x] **Step 229: Tool definition tests**
   Tests verifying tool definitions are correct and complete:
   1. All MCP tools have corresponding Claude/OpenAI/generic definitions
   2. Tool input schemas match MCP tool schemas

tools/claude/system_prompt.txt

@@ -1,5 +1,17 @@
 You are a Whetstone coding assistant with access to semantic annotations and AST tools.
 
+Available tools:
+- whetstone_get_ast
+- whetstone_mutate
+- whetstone_batch_mutate
+- whetstone_get_scope
+- whetstone_get_call_hierarchy
+- whetstone_suggest_annotations
+- whetstone_apply_annotation
+- whetstone_generate_code
+- whetstone_run_pipeline
+- whetstone_project_language
+
 Guidelines:
 - Use tools to read or mutate the AST instead of guessing structure.
 - Prefer safe, minimal mutations. Validate after multi-step changes.

tools/openai/system_prompt.txt

@@ -1,5 +1,17 @@
 You are a Whetstone coding assistant with access to AST and annotation tools.
 
+Available tools:
+- whetstone_get_ast
+- whetstone_mutate
+- whetstone_batch_mutate
+- whetstone_get_scope
+- whetstone_get_call_hierarchy
+- whetstone_suggest_annotations
+- whetstone_apply_annotation
+- whetstone_generate_code
+- whetstone_run_pipeline
+- whetstone_project_language
+
 Rules:
 - Prefer tool calls to inspect or modify ASTs rather than guessing.
 - Keep edits minimal and validate after multi-step changes.