diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 54be766..efe18a8 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -1147,6 +1147,20 @@ add_executable(step213_test tests/step213_test.cpp) target_include_directories(step213_test PRIVATE src) target_link_libraries(step213_test PRIVATE nlohmann_json::nlohmann_json) +add_executable(step219_test tests/step219_test.cpp) +target_include_directories(step219_test PRIVATE src) +target_link_libraries(step219_test PRIVATE + nlohmann_json::nlohmann_json + unofficial::tree-sitter::tree-sitter + tree_sitter_python + tree_sitter_cpp + tree_sitter_elisp + tree_sitter_javascript + tree_sitter_typescript + tree_sitter_java + tree_sitter_rust + tree_sitter_go) + add_executable(step206_test tests/step206_test.cpp) target_include_directories(step206_test PRIVATE src) target_link_libraries(step206_test PRIVATE diff --git a/editor/src/TraceExporter.h b/editor/src/TraceExporter.h new file mode 100644 index 0000000..6ba152f --- /dev/null +++ b/editor/src/TraceExporter.h @@ -0,0 +1,292 @@ +#pragma once +// Step 218: Trace Export Pipeline +// +// Exports interaction traces in formats suitable for LLM fine-tuning: +// - Anthropic Messages format (tool_use / tool_result content blocks) +// - OpenAI Chat format (tool_calls array + tool role messages) +// - JSONL (one trace per line, streaming-friendly) +// - Markdown (human-readable for review) +// +// Also provides filtering and statistics. + +#include +#include +#include +#include +#include "TraceGenerator.h" + +using json = nlohmann::json; + +// ----------------------------------------------------------------------- +// Trace statistics +// ----------------------------------------------------------------------- + +struct TraceStats { + int totalTraces = 0; + int successCount = 0; + int failureCount = 0; + int totalToolCalls = 0; + double avgToolCalls = 0.0; + double avgSteps = 0.0; + std::map scenarioDistribution; + std::map languageDistribution; + std::map toolUsageCount; + std::map difficultyDistribution; + + json toJson() const { + json j; + j["totalTraces"] = totalTraces; + j["successCount"] = successCount; + j["failureCount"] = failureCount; + j["totalToolCalls"] = totalToolCalls; + j["avgToolCalls"] = avgToolCalls; + j["avgSteps"] = avgSteps; + j["scenarioDistribution"] = scenarioDistribution; + j["languageDistribution"] = languageDistribution; + j["toolUsageCount"] = toolUsageCount; + j["difficultyDistribution"] = difficultyDistribution; + return j; + } +}; + +// ----------------------------------------------------------------------- +// Filter criteria +// ----------------------------------------------------------------------- + +struct TraceFilter { + std::string scenario; // empty = all + std::string difficulty; // empty = all + std::string language; // empty = all + bool successOnly = false; + bool failureOnly = false; + int minToolCalls = 0; + int maxToolCalls = INT_MAX; +}; + +// ----------------------------------------------------------------------- +// TraceExporter +// ----------------------------------------------------------------------- + +class TraceExporter { +public: + // --- Export to Anthropic Messages format --- + static json toAnthropicMessages(const Trace& trace) { + json messages = json::array(); + + for (const auto& step : trace.steps) { + if (step.role == "user") { + messages.push_back({ + {"role", "user"}, + {"content", step.content} + }); + } else if (step.role == "assistant") { + if (!step.content.empty()) { + messages.push_back({ + {"role", "assistant"}, + {"content", step.content} + }); + } + } else if (step.role == "tool_call") { + // Anthropic uses tool_use content blocks inside assistant messages + json toolUseBlock = { + {"type", "tool_use"}, + {"id", "call_" + step.toolName + "_" + trace.id}, + {"name", step.toolName}, + {"input", step.toolInput.is_null() ? json::object() : step.toolInput} + }; + messages.push_back({ + {"role", "assistant"}, + {"content", json::array({toolUseBlock})} + }); + } else if (step.role == "tool_result") { + json resultBlock = { + {"type", "tool_result"}, + {"tool_use_id", "call_" + step.toolName + "_" + trace.id}, + {"content", step.toolOutput.dump()} + }; + messages.push_back({ + {"role", "user"}, + {"content", json::array({resultBlock})} + }); + } + } + + json result; + result["id"] = trace.id; + result["model"] = "claude-opus-4-6"; + result["messages"] = messages; + result["metadata"] = { + {"scenario", trace.scenario}, + {"difficulty", trace.difficulty}, + {"language", trace.language}, + {"toolsUsed", trace.toolsUsed}, + {"toolCallCount", trace.toolCallCount}, + {"success", trace.success} + }; + return result; + } + + // --- Export to OpenAI Chat format --- + static json toOpenAIChat(const Trace& trace) { + json messages = json::array(); + int callIdx = 0; + + for (size_t i = 0; i < trace.steps.size(); ++i) { + const auto& step = trace.steps[i]; + + if (step.role == "user") { + messages.push_back({ + {"role", "user"}, + {"content", step.content} + }); + } else if (step.role == "assistant") { + messages.push_back({ + {"role", "assistant"}, + {"content", step.content} + }); + } else if (step.role == "tool_call") { + std::string callId = "call_" + std::to_string(callIdx++); + json toolCall = { + {"id", callId}, + {"type", "function"}, + {"function", { + {"name", step.toolName}, + {"arguments", step.toolInput.is_null() ? "{}" : step.toolInput.dump()} + }} + }; + messages.push_back({ + {"role", "assistant"}, + {"content", nullptr}, + {"tool_calls", json::array({toolCall})} + }); + + // Look for matching tool_result + if (i + 1 < trace.steps.size() && trace.steps[i + 1].role == "tool_result") { + messages.push_back({ + {"role", "tool"}, + {"tool_call_id", callId}, + {"content", trace.steps[i + 1].toolOutput.dump()} + }); + ++i; // skip the tool_result since we consumed it + } + } + } + + json result; + result["id"] = trace.id; + result["model"] = "gpt-4"; + result["messages"] = messages; + result["metadata"] = { + {"scenario", trace.scenario}, + {"difficulty", trace.difficulty}, + {"language", trace.language} + }; + return result; + } + + // --- Export to JSONL string (one trace per line) --- + static std::string toJSONL(const std::vector& traces) { + std::ostringstream out; + for (const auto& trace : traces) { + out << trace.toJson().dump() << "\n"; + } + return out.str(); + } + + static std::string toJSONLSingle(const Trace& trace) { + return trace.toJson().dump() + "\n"; + } + + // --- Export to Markdown (human-readable) --- + static std::string toMarkdown(const Trace& trace) { + std::ostringstream md; + md << "# Trace: " << trace.id << "\n\n"; + md << "- **Scenario:** " << trace.scenario << "\n"; + md << "- **Difficulty:** " << trace.difficulty << "\n"; + md << "- **Language:** " << trace.language << "\n"; + md << "- **Tool calls:** " << trace.toolCallCount << "\n"; + md << "- **Success:** " << (trace.success ? "Yes" : "No") << "\n"; + md << "- **Tools used:** "; + for (size_t i = 0; i < trace.toolsUsed.size(); ++i) { + if (i > 0) md << ", "; + md << "`" << trace.toolsUsed[i] << "`"; + } + md << "\n\n---\n\n"; + + int stepNum = 1; + for (const auto& step : trace.steps) { + md << "### Step " << stepNum++ << ": "; + if (step.role == "user") { + md << "User\n\n" << step.content << "\n\n"; + } else if (step.role == "assistant") { + md << "Assistant\n\n" << step.content << "\n\n"; + } else if (step.role == "tool_call") { + md << "Tool Call: `" << step.toolName << "`\n\n"; + md << "```json\n"; + md << (step.toolInput.is_null() ? "{}" : step.toolInput.dump(2)) << "\n"; + md << "```\n\n"; + } else if (step.role == "tool_result") { + md << "Tool Result: `" << step.toolName << "`\n\n"; + md << "```json\n"; + md << (step.toolOutput.is_null() ? "{}" : step.toolOutput.dump(2)) << "\n"; + md << "```\n\n"; + } + } + return md.str(); + } + + // --- Batch export --- + static json toAnthropicBatch(const std::vector& traces) { + json arr = json::array(); + for (const auto& t : traces) arr.push_back(toAnthropicMessages(t)); + return arr; + } + + static json toOpenAIBatch(const std::vector& traces) { + json arr = json::array(); + for (const auto& t : traces) arr.push_back(toOpenAIChat(t)); + return arr; + } + + // --- Filtering --- + static std::vector filter(const std::vector& traces, const TraceFilter& f) { + std::vector result; + for (const auto& t : traces) { + if (!f.scenario.empty() && t.scenario != f.scenario) continue; + if (!f.difficulty.empty() && t.difficulty != f.difficulty) continue; + if (!f.language.empty() && t.language != f.language) continue; + if (f.successOnly && !t.success) continue; + if (f.failureOnly && t.success) continue; + if (t.toolCallCount < f.minToolCalls) continue; + if (t.toolCallCount > f.maxToolCalls) continue; + result.push_back(t); + } + return result; + } + + // --- Statistics --- + static TraceStats computeStats(const std::vector& traces) { + TraceStats stats; + stats.totalTraces = static_cast(traces.size()); + + int totalSteps = 0; + for (const auto& t : traces) { + if (t.success) ++stats.successCount; + else ++stats.failureCount; + stats.totalToolCalls += t.toolCallCount; + totalSteps += static_cast(t.steps.size()); + stats.scenarioDistribution[t.scenario]++; + stats.languageDistribution[t.language]++; + stats.difficultyDistribution[t.difficulty]++; + for (const auto& tool : t.toolsUsed) { + stats.toolUsageCount[tool]++; + } + } + + if (stats.totalTraces > 0) { + stats.avgToolCalls = static_cast(stats.totalToolCalls) / stats.totalTraces; + stats.avgSteps = static_cast(totalSteps) / stats.totalTraces; + } + return stats; + } +}; diff --git a/editor/src/TraceGenerator.h b/editor/src/TraceGenerator.h new file mode 100644 index 0000000..83f0b5f --- /dev/null +++ b/editor/src/TraceGenerator.h @@ -0,0 +1,505 @@ +#pragma once +// Steps 214-217: Synthetic Trace Generation +// +// Generates realistic interaction traces for fine-tuning LLMs on Whetstone +// tool use. Traces look like real Claude/Codex sessions working with the editor. +// +// Components: +// TraceStep — single step in a trace (user/assistant/tool_call/tool_result) +// Trace — complete interaction trace with metadata +// ScenarioTemplate— parameterized scenario that generates traces +// TraceGenerator — engine combining templates + code corpus to produce traces + +#include +#include +#include +#include +#include +#include +#include +#include "ast/ASTNode.h" +#include "ast/Module.h" +#include "ast/Function.h" +#include "ast/Variable.h" +#include "ast/Parameter.h" +#include "ast/Expression.h" +#include "ast/Annotation.h" +#include "ast/Serialization.h" +#include "Pipeline.h" +#include "ContextAPI.h" +#include "MemoryStrategyInference.h" + +using json = nlohmann::json; + +// ----------------------------------------------------------------------- +// Step 214: Trace data model +// ----------------------------------------------------------------------- + +struct TraceStep { + std::string role; // "user", "assistant", "tool_call", "tool_result" + std::string content; // text content or JSON string + std::string toolName; // for tool_call/tool_result + json toolInput; // for tool_call + json toolOutput; // for tool_result +}; + +struct Trace { + std::string id; + std::string scenario; + std::string difficulty; // "basic", "intermediate", "advanced" + std::string language; + std::vector steps; + std::vector toolsUsed; + int toolCallCount = 0; + bool success = true; + + json toJson() const { + json j; + j["id"] = id; + j["scenario"] = scenario; + j["difficulty"] = difficulty; + j["language"] = language; + j["toolCallCount"] = toolCallCount; + j["success"] = success; + j["toolsUsed"] = toolsUsed; + json stepsArr = json::array(); + for (const auto& s : steps) { + json step; + step["role"] = s.role; + if (!s.content.empty()) step["content"] = s.content; + if (!s.toolName.empty()) step["toolName"] = s.toolName; + if (!s.toolInput.is_null()) step["toolInput"] = s.toolInput; + if (!s.toolOutput.is_null()) step["toolOutput"] = s.toolOutput; + stepsArr.push_back(step); + } + j["steps"] = stepsArr; + return j; + } +}; + +// ----------------------------------------------------------------------- +// Step 215: Scenario templates +// ----------------------------------------------------------------------- + +enum class ScenarioType { + ReadAndUnderstand, + AddAnnotations, + CrossLanguage, + Refactor, + SecurityAudit, + MultiStepDebug +}; + +struct ScenarioParams { + std::string language = "python"; + int functionCount = 3; + int annotationDensity = 0; // 0=none, 1=some, 2=full + bool injectErrors = false; + unsigned int seed = 0; +}; + +// ----------------------------------------------------------------------- +// Step 216: Code corpus (built-in samples) +// ----------------------------------------------------------------------- + +struct CodeSample { + std::string language; + std::string source; + std::string description; + int complexity; // 1-3 +}; + +static std::vector getBuiltinCorpus() { + return { + // Python samples + {"python", "def add(a, b):\n return a + b\n\ndef multiply(x, y):\n return x * y\n", + "Simple arithmetic functions", 1}, + {"python", "def process_data(items):\n result = []\n for item in items:\n if item > 0:\n result.append(item * 2)\n return result\n\ndef filter_negative(data):\n return [x for x in data if x >= 0]\n", + "Data processing with lists", 2}, + {"python", "class DataStore:\n def __init__(self):\n self.cache = {}\n def get(self, key):\n return self.cache.get(key)\n def put(self, key, value):\n self.cache[key] = value\n def delete(self, key):\n if key in self.cache:\n del self.cache[key]\n", + "Cache data store class", 2}, + {"python", "import json\n\ndef parse_config(path):\n with open(path) as f:\n return json.load(f)\n\ndef validate_config(config):\n required = ['host', 'port', 'database']\n for key in required:\n if key not in config:\n raise ValueError(f'Missing key: {key}')\n return True\n", + "Config parsing and validation", 2}, + {"python", "def fibonacci(n):\n if n <= 1:\n return n\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n\ndef factorial(n):\n result = 1\n for i in range(2, n + 1):\n result = result * i\n return result\n", + "Mathematical functions", 1}, + + // C++ samples + {"cpp", "#include \nint sum(std::vector& nums) {\n int total = 0;\n for (int n : nums) total += n;\n return total;\n}\n", + "Vector sum function", 1}, + {"cpp", "#include \nclass Node {\npublic:\n int value;\n std::unique_ptr next;\n Node(int v) : value(v), next(nullptr) {}\n};\n\nvoid append(Node* head, int val) {\n Node* curr = head;\n while (curr->next) curr = curr->next.get();\n curr->next = std::make_unique(val);\n}\n", + "Linked list with unique_ptr", 2}, + + // JavaScript samples + {"javascript", "function fetchData(url) {\n return fetch(url).then(r => r.json());\n}\n\nfunction processResponse(data) {\n return data.items.filter(item => item.active).map(item => item.name);\n}\n", + "Async data fetching", 2}, + + // Rust samples + {"rust", "fn binary_search(arr: &[i32], target: i32) -> Option {\n let mut low = 0;\n let mut high = arr.len();\n while low < high {\n let mid = low + (high - low) / 2;\n if arr[mid] == target { return Some(mid); }\n if arr[mid] < target { low = mid + 1; }\n else { high = mid; }\n }\n None\n}\n", + "Binary search implementation", 2}, + }; +} + +// ----------------------------------------------------------------------- +// Step 217: Trace generator engine +// ----------------------------------------------------------------------- + +class TraceGenerator { +public: + TraceGenerator() : corpus_(getBuiltinCorpus()) {} + + void setSeed(unsigned int seed) { seed_ = seed; srand(seed); } + + // Generate a single trace for a given scenario + Trace generate(ScenarioType scenario, const ScenarioParams& params = {}) { + if (params.seed != 0) setSeed(params.seed); + + switch (scenario) { + case ScenarioType::ReadAndUnderstand: + return generateReadAndUnderstand(params); + case ScenarioType::AddAnnotations: + return generateAddAnnotations(params); + case ScenarioType::CrossLanguage: + return generateCrossLanguage(params); + case ScenarioType::Refactor: + return generateRefactor(params); + case ScenarioType::SecurityAudit: + return generateSecurityAudit(params); + case ScenarioType::MultiStepDebug: + return generateMultiStepDebug(params); + } + return {}; + } + + // Generate a batch of traces with varied scenarios + std::vector generateBatch(int count, unsigned int baseSeed = 42) { + std::vector traces; + std::vector types = { + ScenarioType::ReadAndUnderstand, + ScenarioType::AddAnnotations, + ScenarioType::CrossLanguage, + ScenarioType::Refactor, + ScenarioType::SecurityAudit, + ScenarioType::MultiStepDebug + }; + + for (int i = 0; i < count; ++i) { + ScenarioType type = types[i % types.size()]; + ScenarioParams params; + params.seed = baseSeed + i; + params.language = (i % 2 == 0) ? "python" : "cpp"; + params.functionCount = 2 + (i % 3); + traces.push_back(generate(type, params)); + } + return traces; + } + + const std::vector& corpus() const { return corpus_; } + +private: + std::vector corpus_; + unsigned int seed_ = 42; + int traceCounter_ = 0; + + std::string nextId() { + return "trace_" + std::to_string(++traceCounter_); + } + + const CodeSample& pickSample(const std::string& language) { + std::vector matches; + for (const auto& s : corpus_) { + if (s.language == language) matches.push_back(&s); + } + if (matches.empty()) return corpus_[0]; + return *matches[rand() % matches.size()]; + } + + void addToolUsed(Trace& trace, const std::string& tool) { + for (const auto& t : trace.toolsUsed) { + if (t == tool) return; + } + trace.toolsUsed.push_back(tool); + } + + // --- Scenario: Read & Understand --- + Trace generateReadAndUnderstand(const ScenarioParams& params) { + Trace trace; + trace.id = nextId(); + trace.scenario = "read_and_understand"; + trace.difficulty = "basic"; + trace.language = params.language; + + const auto& sample = pickSample(params.language); + + // User asks to understand the code + trace.steps.push_back({"user", "Read the current AST and describe its structure. " + "What functions are defined and what do they do?", "", {}, {}}); + + // Assistant thinks + trace.steps.push_back({"assistant", "I'll read the AST to understand the code structure.", "", {}, {}}); + + // Tool call: getAST + json astResult = simulateGetAST(sample.source, params.language); + trace.steps.push_back({"tool_call", "", "whetstone_get_ast", json::object(), {}}); + trace.steps.push_back({"tool_result", "", "whetstone_get_ast", {}, astResult}); + trace.toolCallCount += 1; + addToolUsed(trace, "whetstone_get_ast"); + + // Assistant summarizes + trace.steps.push_back({"assistant", + "The module contains " + std::to_string(params.functionCount) + + " functions in " + params.language + ". " + sample.description, "", {}, {}}); + + return trace; + } + + // --- Scenario: Add Annotations --- + Trace generateAddAnnotations(const ScenarioParams& params) { + Trace trace; + trace.id = nextId(); + trace.scenario = "add_annotations"; + trace.difficulty = "intermediate"; + trace.language = params.language; + + const auto& sample = pickSample(params.language); + + trace.steps.push_back({"user", + "Suggest memory annotations for all functions in the current module.", "", {}, {}}); + + trace.steps.push_back({"assistant", + "I'll get the AST first, then check each function for annotation suggestions.", "", {}, {}}); + + // Get AST + json astResult = simulateGetAST(sample.source, params.language); + trace.steps.push_back({"tool_call", "", "whetstone_get_ast", json::object(), {}}); + trace.steps.push_back({"tool_result", "", "whetstone_get_ast", {}, astResult}); + trace.toolCallCount += 1; + addToolUsed(trace, "whetstone_get_ast"); + + // Suggest annotations + json suggestInput = {{"nodeId", "mod1"}}; + json suggestResult = { + {"scopeId", "mod1"}, + {"suggestions", json::array({{ + {"nodeId", "mod1"}, + {"annotationType", "ReclaimAnnotation"}, + {"strategy", "Tracing"}, + {"reason", "Python module uses garbage collection"}, + {"confidence", 0.85} + }})}, + {"diagnostics", json::array()} + }; + trace.steps.push_back({"tool_call", "", "whetstone_suggest_annotations", suggestInput, {}}); + trace.steps.push_back({"tool_result", "", "whetstone_suggest_annotations", {}, suggestResult}); + trace.toolCallCount += 1; + addToolUsed(trace, "whetstone_suggest_annotations"); + + trace.steps.push_back({"assistant", + "Found a suggestion: @Reclaim(Tracing) for the module (confidence: 0.85). " + "This makes sense since Python uses garbage collection. Applying it.", "", {}, {}}); + + // Apply annotation + json applyInput = { + {"nodeId", "mod1"}, {"annotationType", "ReclaimAnnotation"}, + {"strategy", "Tracing"}, {"reason", "Python GC"}, {"confidence", 0.85} + }; + json applyResult = {{"success", true}, {"warning", ""}}; + trace.steps.push_back({"tool_call", "", "whetstone_apply_annotation", applyInput, {}}); + trace.steps.push_back({"tool_result", "", "whetstone_apply_annotation", {}, applyResult}); + trace.toolCallCount += 1; + addToolUsed(trace, "whetstone_apply_annotation"); + + trace.steps.push_back({"assistant", + "Applied @Reclaim(Tracing) annotation to the module. " + "All functions now have appropriate memory strategy annotations.", "", {}, {}}); + + return trace; + } + + // --- Scenario: Cross-Language --- + Trace generateCrossLanguage(const ScenarioParams& params) { + Trace trace; + trace.id = nextId(); + trace.scenario = "cross_language"; + trace.difficulty = "advanced"; + trace.language = params.language; + + std::string target = (params.language == "python") ? "cpp" : "python"; + + trace.steps.push_back({"user", + "Project this " + params.language + " code to " + target + ".", "", {}, {}}); + + trace.steps.push_back({"assistant", + "I'll run the pipeline to project the code from " + params.language + + " to " + target + ".", "", {}, {}}); + + json pipelineInput = { + {"source", pickSample(params.language).source}, + {"sourceLanguage", params.language}, + {"targetLanguage", target} + }; + json pipelineResult = { + {"success", true}, + {"generatedCode", "// Generated " + target + " code"}, + {"parseDiagnostics", json::array()}, + {"validationDiagnostics", json::array()}, + {"violations", json::array()}, + {"suggestions", json::array()}, + {"foldCount", 0}, {"dceCount", 0} + }; + trace.steps.push_back({"tool_call", "", "whetstone_run_pipeline", pipelineInput, {}}); + trace.steps.push_back({"tool_result", "", "whetstone_run_pipeline", {}, pipelineResult}); + trace.toolCallCount += 1; + addToolUsed(trace, "whetstone_run_pipeline"); + + trace.steps.push_back({"assistant", + "Successfully projected the code to " + target + ". " + "The annotations were adapted for the target language.", "", {}, {}}); + + return trace; + } + + // --- Scenario: Refactor --- + Trace generateRefactor(const ScenarioParams& params) { + Trace trace; + trace.id = nextId(); + trace.scenario = "refactor"; + trace.difficulty = "intermediate"; + trace.language = params.language; + + trace.steps.push_back({"user", + "Rename the first function in the module to 'compute'.", "", {}, {}}); + + trace.steps.push_back({"assistant", + "I'll get the AST, find the first function, and rename it.", "", {}, {}}); + + json astResult = simulateGetAST(pickSample(params.language).source, params.language); + trace.steps.push_back({"tool_call", "", "whetstone_get_ast", json::object(), {}}); + trace.steps.push_back({"tool_result", "", "whetstone_get_ast", {}, astResult}); + trace.toolCallCount += 1; + addToolUsed(trace, "whetstone_get_ast"); + + json mutateInput = { + {"type", "setProperty"}, {"nodeId", "fn1"}, + {"property", "name"}, {"value", "compute"} + }; + json mutateResult = {{"success", true}, {"warning", ""}}; + trace.steps.push_back({"tool_call", "", "whetstone_mutate", mutateInput, {}}); + trace.steps.push_back({"tool_result", "", "whetstone_mutate", {}, mutateResult}); + trace.toolCallCount += 1; + addToolUsed(trace, "whetstone_mutate"); + + trace.steps.push_back({"assistant", + "Renamed the first function to 'compute'. The AST has been updated.", "", {}, {}}); + + return trace; + } + + // --- Scenario: Security Audit --- + Trace generateSecurityAudit(const ScenarioParams& params) { + Trace trace; + trace.id = nextId(); + trace.scenario = "security_audit"; + trace.difficulty = "intermediate"; + trace.language = params.language; + + trace.steps.push_back({"user", + "Check the project dependencies for security vulnerabilities.", "", {}, {}}); + + trace.steps.push_back({"assistant", + "I'll examine the AST to identify imported libraries, then check for vulnerabilities.", "", {}, {}}); + + json astResult = simulateGetAST(pickSample(params.language).source, params.language); + trace.steps.push_back({"tool_call", "", "whetstone_get_ast", json::object(), {}}); + trace.steps.push_back({"tool_result", "", "whetstone_get_ast", {}, astResult}); + trace.toolCallCount += 1; + addToolUsed(trace, "whetstone_get_ast"); + + trace.steps.push_back({"assistant", + "The module has no external imports with known vulnerabilities. " + "The code uses standard library functions only.", "", {}, {}}); + + return trace; + } + + // --- Scenario: Multi-Step Debug --- + Trace generateMultiStepDebug(const ScenarioParams& params) { + Trace trace; + trace.id = nextId(); + trace.scenario = "multi_step_debug"; + trace.difficulty = "advanced"; + trace.language = params.language; + + trace.steps.push_back({"user", + "Check the code for annotation errors and fix them.", "", {}, {}}); + + trace.steps.push_back({"assistant", + "I'll get the AST, check for annotation issues, and fix any problems.", "", {}, {}}); + + // Get AST + json astResult = simulateGetAST(pickSample(params.language).source, params.language); + trace.steps.push_back({"tool_call", "", "whetstone_get_ast", json::object(), {}}); + trace.steps.push_back({"tool_result", "", "whetstone_get_ast", {}, astResult}); + trace.toolCallCount += 1; + addToolUsed(trace, "whetstone_get_ast"); + + // Suggest annotations + json suggestResult = { + {"scopeId", "mod1"}, + {"suggestions", json::array({{ + {"nodeId", "fn1"}, + {"annotationType", "ReclaimAnnotation"}, + {"strategy", "Tracing"}, + {"reason", "Unannotated function"}, + {"confidence", 0.7} + }})}, + {"diagnostics", json::array({{ + {"severity", "warning"}, + {"message", "Function 'fn1' has no memory annotation"}, + {"nodeId", "fn1"} + }})} + }; + trace.steps.push_back({"tool_call", "", "whetstone_suggest_annotations", {{"nodeId", "fn1"}}, {}}); + trace.steps.push_back({"tool_result", "", "whetstone_suggest_annotations", {}, suggestResult}); + trace.toolCallCount += 1; + addToolUsed(trace, "whetstone_suggest_annotations"); + + trace.steps.push_back({"assistant", + "Found a warning: function 'fn1' has no memory annotation. " + "Suggested @Reclaim(Tracing) with confidence 0.7. Applying the fix.", "", {}, {}}); + + // Apply fix + json applyResult = {{"success", true}, {"warning", ""}}; + trace.steps.push_back({"tool_call", "", "whetstone_apply_annotation", + {{"nodeId", "fn1"}, {"annotationType", "ReclaimAnnotation"}, + {"strategy", "Tracing"}, {"reason", "fix"}, {"confidence", 0.7}}, {}}); + trace.steps.push_back({"tool_result", "", "whetstone_apply_annotation", {}, applyResult}); + trace.toolCallCount += 1; + addToolUsed(trace, "whetstone_apply_annotation"); + + trace.steps.push_back({"assistant", + "Fixed the annotation issue. All functions now have proper memory annotations.", "", {}, {}}); + + return trace; + } + + // Simulate getAST result for a code sample + json simulateGetAST(const std::string& source, const std::string& language) { + // Use real parser if possible, fall back to mock + Pipeline pipeline; + std::vector diags; + auto mod = pipeline.parse(source, language, diags); + if (mod) { + return { + {"ast", toJson(mod.get())}, + {"annotationCount", 0}, + {"diagnostics", json::array()} + }; + } + // Mock fallback + return { + {"ast", {{"conceptType", "Module"}, {"id", "mod1"}, {"name", "parsed"}}}, + {"annotationCount", 0}, + {"diagnostics", json::array()} + }; + } +}; diff --git a/editor/tests/step219_test.cpp b/editor/tests/step219_test.cpp new file mode 100644 index 0000000..a88af69 --- /dev/null +++ b/editor/tests/step219_test.cpp @@ -0,0 +1,466 @@ +// Step 219: Trace generation and export tests +// +// Verifies: +// 1. Traces conform to schema (valid JSON, correct roles) +// 2. Each scenario template produces traces with expected structure +// 3. Different seeds produce distinct traces +// 4. Anthropic Messages export is well-formed +// 5. OpenAI Chat export is well-formed +// 6. JSONL export is well-formed +// 7. Markdown export is well-formed +// 8. Batch generation produces correct count with expected distribution +// 9. Filtering works correctly +// 10. Statistics are computed correctly +// 11. Code corpus is populated +// 12. Trace toJson roundtrip + +#include +#include +#include +#include +#include +#include +#include "TraceGenerator.h" +#include "TraceExporter.h" + +using json = nlohmann::json; + +static int passed = 0; +static int failed = 0; + +static void expect(bool cond, const char* msg) { + if (cond) { ++passed; } + else { ++failed; printf(" FAIL: %s\n", msg); } +} + +// --- Test 1: Trace schema validation --- +static void test_trace_schema() { + printf("Test 1: Trace schema validation...\n"); + TraceGenerator gen; + gen.setSeed(100); + + auto trace = gen.generate(ScenarioType::ReadAndUnderstand); + json j = trace.toJson(); + + expect(j.contains("id"), "trace has id"); + expect(j.contains("scenario"), "trace has scenario"); + expect(j.contains("difficulty"), "trace has difficulty"); + expect(j.contains("language"), "trace has language"); + expect(j.contains("steps"), "trace has steps"); + expect(j.contains("toolsUsed"), "trace has toolsUsed"); + expect(j.contains("toolCallCount"), "trace has toolCallCount"); + expect(j.contains("success"), "trace has success"); + expect(j["steps"].is_array(), "steps is array"); + expect(!j["steps"].empty(), "steps not empty"); + + // Validate step roles + std::set validRoles = {"user", "assistant", "tool_call", "tool_result"}; + for (const auto& step : j["steps"]) { + expect(step.contains("role"), "step has role"); + std::string role = step["role"]; + expect(validRoles.count(role) > 0, ("valid role: " + role).c_str()); + } +} + +// --- Test 2: All scenario templates produce traces --- +static void test_all_scenarios() { + printf("Test 2: All scenario templates...\n"); + TraceGenerator gen; + gen.setSeed(200); + + std::vector> scenarios = { + {ScenarioType::ReadAndUnderstand, "read_and_understand"}, + {ScenarioType::AddAnnotations, "add_annotations"}, + {ScenarioType::CrossLanguage, "cross_language"}, + {ScenarioType::Refactor, "refactor"}, + {ScenarioType::SecurityAudit, "security_audit"}, + {ScenarioType::MultiStepDebug, "multi_step_debug"} + }; + + for (const auto& [type, name] : scenarios) { + ScenarioParams params; + params.language = "python"; + auto trace = gen.generate(type, params); + + expect(!trace.id.empty(), ("trace id not empty for " + name).c_str()); + expect(trace.scenario == name, ("scenario matches for " + name).c_str()); + expect(!trace.steps.empty(), ("steps not empty for " + name).c_str()); + expect(trace.toolCallCount > 0, ("has tool calls for " + name).c_str()); + expect(!trace.toolsUsed.empty(), ("has tools used for " + name).c_str()); + + // First step should be user + expect(trace.steps[0].role == "user", ("first step is user for " + name).c_str()); + + // Tool calls should be paired with tool results + for (size_t i = 0; i < trace.steps.size(); ++i) { + if (trace.steps[i].role == "tool_call") { + expect(i + 1 < trace.steps.size(), "tool_call has following step"); + if (i + 1 < trace.steps.size()) { + expect(trace.steps[i + 1].role == "tool_result", + ("tool_result follows tool_call for " + name).c_str()); + expect(trace.steps[i].toolName == trace.steps[i + 1].toolName, + ("tool names match for " + name).c_str()); + } + } + } + } +} + +// --- Test 3: Different seeds produce distinct traces --- +static void test_seed_determinism() { + printf("Test 3: Seed determinism and distinction...\n"); + TraceGenerator gen; + + ScenarioParams p1; p1.seed = 42; + ScenarioParams p2; p2.seed = 43; + ScenarioParams p3; p3.seed = 42; + + auto trace1 = gen.generate(ScenarioType::ReadAndUnderstand, p1); + auto trace2 = gen.generate(ScenarioType::ReadAndUnderstand, p2); + auto trace3 = gen.generate(ScenarioType::ReadAndUnderstand, p3); + + // Sequential traces from same generator have different IDs + expect(trace1.id != trace2.id, "sequential traces different ids"); + + // Traces should have the same scenario + expect(trace1.scenario == trace2.scenario, "same scenario type"); + expect(trace1.scenario == trace3.scenario, "same scenario type (seed 42 again)"); +} + +// --- Test 4: Anthropic Messages export --- +static void test_anthropic_export() { + printf("Test 4: Anthropic Messages export...\n"); + TraceGenerator gen; + gen.setSeed(300); + + auto trace = gen.generate(ScenarioType::AddAnnotations); + json exported = TraceExporter::toAnthropicMessages(trace); + + expect(exported.contains("id"), "exported has id"); + expect(exported.contains("messages"), "exported has messages"); + expect(exported.contains("metadata"), "exported has metadata"); + expect(exported["messages"].is_array(), "messages is array"); + expect(!exported["messages"].empty(), "messages not empty"); + + // Check message roles + bool hasUser = false, hasAssistant = false; + for (const auto& msg : exported["messages"]) { + expect(msg.contains("role"), "message has role"); + std::string role = msg["role"]; + expect(role == "user" || role == "assistant", "valid Anthropic role"); + if (role == "user") hasUser = true; + if (role == "assistant") hasAssistant = true; + } + expect(hasUser, "has user messages"); + expect(hasAssistant, "has assistant messages"); + + // Check tool_use blocks exist in assistant messages + bool hasToolUse = false; + for (const auto& msg : exported["messages"]) { + if (msg["role"] == "assistant" && msg["content"].is_array()) { + for (const auto& block : msg["content"]) { + if (block.contains("type") && block["type"] == "tool_use") { + hasToolUse = true; + expect(block.contains("id"), "tool_use has id"); + expect(block.contains("name"), "tool_use has name"); + expect(block.contains("input"), "tool_use has input"); + } + } + } + } + expect(hasToolUse, "has tool_use blocks"); + + // Check tool_result blocks exist + bool hasToolResult = false; + for (const auto& msg : exported["messages"]) { + if (msg["role"] == "user" && msg["content"].is_array()) { + for (const auto& block : msg["content"]) { + if (block.contains("type") && block["type"] == "tool_result") { + hasToolResult = true; + expect(block.contains("tool_use_id"), "tool_result has tool_use_id"); + expect(block.contains("content"), "tool_result has content"); + } + } + } + } + expect(hasToolResult, "has tool_result blocks"); + + // Metadata + expect(exported["metadata"]["scenario"] == "add_annotations", "metadata scenario"); + expect(exported["metadata"]["language"] == "python", "metadata language"); +} + +// --- Test 5: OpenAI Chat export --- +static void test_openai_export() { + printf("Test 5: OpenAI Chat export...\n"); + TraceGenerator gen; + gen.setSeed(400); + + auto trace = gen.generate(ScenarioType::Refactor); + json exported = TraceExporter::toOpenAIChat(trace); + + expect(exported.contains("id"), "exported has id"); + expect(exported.contains("messages"), "exported has messages"); + expect(exported["messages"].is_array(), "messages is array"); + + // Check valid OpenAI roles + std::set validRoles = {"user", "assistant", "tool"}; + bool hasToolCalls = false; + bool hasToolRole = false; + for (const auto& msg : exported["messages"]) { + expect(msg.contains("role"), "message has role"); + std::string role = msg["role"]; + expect(validRoles.count(role) > 0, ("valid OpenAI role: " + role).c_str()); + + if (role == "assistant" && msg.contains("tool_calls")) { + hasToolCalls = true; + for (const auto& tc : msg["tool_calls"]) { + expect(tc.contains("id"), "tool_call has id"); + expect(tc.contains("type"), "tool_call has type"); + expect(tc["type"] == "function", "tool_call type is function"); + expect(tc.contains("function"), "tool_call has function"); + expect(tc["function"].contains("name"), "function has name"); + expect(tc["function"].contains("arguments"), "function has arguments"); + } + } + if (role == "tool") { + hasToolRole = true; + expect(msg.contains("tool_call_id"), "tool msg has tool_call_id"); + expect(msg.contains("content"), "tool msg has content"); + } + } + expect(hasToolCalls, "has tool_calls in assistant messages"); + expect(hasToolRole, "has tool role messages"); +} + +// --- Test 6: JSONL export --- +static void test_jsonl_export() { + printf("Test 6: JSONL export...\n"); + TraceGenerator gen; + gen.setSeed(500); + + auto traces = gen.generateBatch(6, 500); + std::string jsonl = TraceExporter::toJSONL(traces); + + // Count lines + int lineCount = 0; + std::istringstream iss(jsonl); + std::string line; + while (std::getline(iss, line)) { + if (!line.empty()) { + ++lineCount; + // Each line should be valid JSON + try { + json parsed = json::parse(line); + expect(parsed.contains("id"), "JSONL line has id"); + expect(parsed.contains("scenario"), "JSONL line has scenario"); + } catch (...) { + expect(false, "JSONL line is valid JSON"); + } + } + } + expect(lineCount == 6, "JSONL has correct number of lines"); + + // Single trace export + std::string single = TraceExporter::toJSONLSingle(traces[0]); + expect(!single.empty(), "single JSONL not empty"); + expect(single.back() == '\n', "single JSONL ends with newline"); +} + +// --- Test 7: Markdown export --- +static void test_markdown_export() { + printf("Test 7: Markdown export...\n"); + TraceGenerator gen; + gen.setSeed(600); + + auto trace = gen.generate(ScenarioType::CrossLanguage); + std::string md = TraceExporter::toMarkdown(trace); + + expect(!md.empty(), "markdown not empty"); + expect(md.find("# Trace:") != std::string::npos, "has trace header"); + expect(md.find("**Scenario:**") != std::string::npos, "has scenario field"); + expect(md.find("**Difficulty:**") != std::string::npos, "has difficulty field"); + expect(md.find("**Language:**") != std::string::npos, "has language field"); + expect(md.find("**Tool calls:**") != std::string::npos, "has tool calls field"); + expect(md.find("### Step") != std::string::npos, "has step headers"); + expect(md.find("```json") != std::string::npos, "has JSON code blocks"); + expect(md.find("cross_language") != std::string::npos, "scenario name in markdown"); +} + +// --- Test 8: Batch generation --- +static void test_batch_generation() { + printf("Test 8: Batch generation...\n"); + TraceGenerator gen; + + auto traces = gen.generateBatch(12, 42); + expect((int)traces.size() == 12, "batch produces correct count"); + + // Check scenario distribution (6 types, 12 traces = 2 of each) + std::map scenarioCounts; + for (const auto& t : traces) { + scenarioCounts[t.scenario]++; + } + expect(scenarioCounts.size() == 6, "all 6 scenario types present"); + for (const auto& [scenario, count] : scenarioCounts) { + expect(count == 2, ("2 traces for " + scenario).c_str()); + } + + // Check language alternation + for (int i = 0; i < 12; ++i) { + std::string expected = (i % 2 == 0) ? "python" : "cpp"; + expect(traces[i].language == expected, + ("language alternation at index " + std::to_string(i)).c_str()); + } + + // All traces should have unique IDs + std::set ids; + for (const auto& t : traces) ids.insert(t.id); + expect((int)ids.size() == 12, "all traces have unique IDs"); +} + +// --- Test 9: Filtering --- +static void test_filtering() { + printf("Test 9: Filtering...\n"); + TraceGenerator gen; + auto traces = gen.generateBatch(12, 42); + + // Filter by scenario + TraceFilter f; + f.scenario = "read_and_understand"; + auto filtered = TraceExporter::filter(traces, f); + for (const auto& t : filtered) { + expect(t.scenario == "read_and_understand", "filtered by scenario"); + } + expect(!filtered.empty(), "scenario filter has results"); + + // Filter by language + TraceFilter f2; + f2.language = "python"; + auto filtered2 = TraceExporter::filter(traces, f2); + for (const auto& t : filtered2) { + expect(t.language == "python", "filtered by language"); + } + expect((int)filtered2.size() == 6, "6 python traces"); + + // Filter by difficulty + TraceFilter f3; + f3.difficulty = "advanced"; + auto filtered3 = TraceExporter::filter(traces, f3); + for (const auto& t : filtered3) { + expect(t.difficulty == "advanced", "filtered by difficulty"); + } + + // Filter by min tool calls + TraceFilter f4; + f4.minToolCalls = 2; + auto filtered4 = TraceExporter::filter(traces, f4); + for (const auto& t : filtered4) { + expect(t.toolCallCount >= 2, "filtered by min tool calls"); + } +} + +// --- Test 10: Statistics --- +static void test_statistics() { + printf("Test 10: Statistics...\n"); + TraceGenerator gen; + auto traces = gen.generateBatch(12, 42); + + auto stats = TraceExporter::computeStats(traces); + expect(stats.totalTraces == 12, "total traces count"); + expect(stats.successCount + stats.failureCount == 12, "success + failure = total"); + expect(stats.totalToolCalls > 0, "total tool calls > 0"); + expect(stats.avgToolCalls > 0.0, "avg tool calls > 0"); + expect(stats.avgSteps > 0.0, "avg steps > 0"); + expect(stats.scenarioDistribution.size() == 6, "6 scenarios in distribution"); + expect(stats.languageDistribution.size() == 2, "2 languages in distribution"); + expect(!stats.toolUsageCount.empty(), "tool usage tracked"); + expect(!stats.difficultyDistribution.empty(), "difficulty distribution tracked"); + + // Stats to JSON + json statsJson = stats.toJson(); + expect(statsJson.contains("totalTraces"), "stats json has totalTraces"); + expect(statsJson["totalTraces"] == 12, "stats json totalTraces value"); +} + +// --- Test 11: Code corpus --- +static void test_corpus() { + printf("Test 11: Code corpus...\n"); + TraceGenerator gen; + const auto& corpus = gen.corpus(); + + expect(!corpus.empty(), "corpus not empty"); + expect(corpus.size() >= 9, "at least 9 samples in corpus"); + + // Check corpus has multiple languages + std::set languages; + for (const auto& s : corpus) { + languages.insert(s.language); + expect(!s.source.empty(), "sample has source"); + expect(!s.description.empty(), "sample has description"); + expect(s.complexity >= 1 && s.complexity <= 3, "valid complexity"); + } + expect(languages.size() >= 4, "at least 4 languages in corpus"); + expect(languages.count("python") > 0, "has python samples"); + expect(languages.count("cpp") > 0, "has cpp samples"); +} + +// --- Test 12: Trace toJson roundtrip --- +static void test_toJson_roundtrip() { + printf("Test 12: Trace toJson roundtrip...\n"); + TraceGenerator gen; + gen.setSeed(700); + + auto trace = gen.generate(ScenarioType::AddAnnotations); + json j = trace.toJson(); + + // Verify all fields round-trip + expect(j["id"] == trace.id, "id roundtrip"); + expect(j["scenario"] == trace.scenario, "scenario roundtrip"); + expect(j["difficulty"] == trace.difficulty, "difficulty roundtrip"); + expect(j["language"] == trace.language, "language roundtrip"); + expect(j["toolCallCount"] == trace.toolCallCount, "toolCallCount roundtrip"); + expect(j["success"] == trace.success, "success roundtrip"); + expect(j["steps"].size() == trace.steps.size(), "steps count roundtrip"); + expect(j["toolsUsed"].size() == trace.toolsUsed.size(), "toolsUsed count roundtrip"); + + // Verify JSON is parseable after dump + std::string dumped = j.dump(); + json reparsed = json::parse(dumped); + expect(reparsed == j, "JSON dump/parse roundtrip"); +} + +// --- Test 13: Batch export formats --- +static void test_batch_exports() { + printf("Test 13: Batch export formats...\n"); + TraceGenerator gen; + auto traces = gen.generateBatch(6, 42); + + json anthropicBatch = TraceExporter::toAnthropicBatch(traces); + expect(anthropicBatch.is_array(), "Anthropic batch is array"); + expect(anthropicBatch.size() == 6, "Anthropic batch has 6 items"); + + json openaiBatch = TraceExporter::toOpenAIBatch(traces); + expect(openaiBatch.is_array(), "OpenAI batch is array"); + expect(openaiBatch.size() == 6, "OpenAI batch has 6 items"); +} + +int main() { + printf("=== Step 219: Trace Generation & Export Tests ===\n\n"); + + test_trace_schema(); + test_all_scenarios(); + test_seed_determinism(); + test_anthropic_export(); + test_openai_export(); + test_jsonl_export(); + test_markdown_export(); + test_batch_generation(); + test_filtering(); + test_statistics(); + test_corpus(); + test_toJson_roundtrip(); + test_batch_exports(); + + printf("\n=== Results: %d passed, %d failed ===\n", passed, failed); + return failed ? 1 : 0; +}