#pragma once // Steps 214-217: Synthetic Trace Generation // // Generates realistic interaction traces for fine-tuning LLMs on Whetstone // tool use. Traces look like real Claude/Codex sessions working with the editor. // // Components: // TraceStep — single step in a trace (user/assistant/tool_call/tool_result) // Trace — complete interaction trace with metadata // ScenarioTemplate— parameterized scenario that generates traces // TraceGenerator — engine combining templates + code corpus to produce traces #include #include #include #include #include #include #include #include "ast/ASTNode.h" #include "ast/Module.h" #include "ast/Function.h" #include "ast/Variable.h" #include "ast/Parameter.h" #include "ast/Expression.h" #include "ast/Annotation.h" #include "ast/Serialization.h" #include "Pipeline.h" #include "ContextAPI.h" #include "MemoryStrategyInference.h" using json = nlohmann::json; // ----------------------------------------------------------------------- // Step 214: Trace data model // ----------------------------------------------------------------------- struct TraceStep { std::string role; // "user", "assistant", "tool_call", "tool_result" std::string content; // text content or JSON string std::string toolName; // for tool_call/tool_result json toolInput; // for tool_call json toolOutput; // for tool_result }; struct Trace { std::string id; std::string scenario; std::string difficulty; // "basic", "intermediate", "advanced" std::string language; std::vector steps; std::vector toolsUsed; int toolCallCount = 0; bool success = true; json toJson() const { json j; j["id"] = id; j["scenario"] = scenario; j["difficulty"] = difficulty; j["language"] = language; j["toolCallCount"] = toolCallCount; j["success"] = success; j["toolsUsed"] = toolsUsed; json stepsArr = json::array(); for (const auto& s : steps) { json step; step["role"] = s.role; if (!s.content.empty()) step["content"] = s.content; if (!s.toolName.empty()) step["toolName"] = s.toolName; if (!s.toolInput.is_null()) step["toolInput"] = s.toolInput; if (!s.toolOutput.is_null()) step["toolOutput"] = s.toolOutput; stepsArr.push_back(step); } j["steps"] = stepsArr; return j; } }; // ----------------------------------------------------------------------- // Step 215: Scenario templates // ----------------------------------------------------------------------- enum class ScenarioType { ReadAndUnderstand, AddAnnotations, CrossLanguage, Refactor, SecurityAudit, MultiStepDebug, AnnotateAllSubjects, ValidateAndFix, SemannoExport, CrossLanguageAnnotated }; struct ScenarioParams { std::string language = "python"; int functionCount = 3; int annotationDensity = 0; // 0=none, 1=some, 2=full bool injectErrors = false; unsigned int seed = 0; }; // ----------------------------------------------------------------------- // Step 216: Code corpus (built-in samples) // ----------------------------------------------------------------------- struct CodeSample { std::string language; std::string source; std::string description; int complexity; // 1-3 }; static std::vector getBuiltinCorpus() { return { // Python samples {"python", "def add(a, b):\n return a + b\n\ndef multiply(x, y):\n return x * y\n", "Simple arithmetic functions", 1}, {"python", "def process_data(items):\n result = []\n for item in items:\n if item > 0:\n result.append(item * 2)\n return result\n\ndef filter_negative(data):\n return [x for x in data if x >= 0]\n", "Data processing with lists", 2}, {"python", "class DataStore:\n def __init__(self):\n self.cache = {}\n def get(self, key):\n return self.cache.get(key)\n def put(self, key, value):\n self.cache[key] = value\n def delete(self, key):\n if key in self.cache:\n del self.cache[key]\n", "Cache data store class", 2}, {"python", "import json\n\ndef parse_config(path):\n with open(path) as f:\n return json.load(f)\n\ndef validate_config(config):\n required = ['host', 'port', 'database']\n for key in required:\n if key not in config:\n raise ValueError(f'Missing key: {key}')\n return True\n", "Config parsing and validation", 2}, {"python", "def fibonacci(n):\n if n <= 1:\n return n\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n\ndef factorial(n):\n result = 1\n for i in range(2, n + 1):\n result = result * i\n return result\n", "Mathematical functions", 1}, // C++ samples {"cpp", "#include \nint sum(std::vector& nums) {\n int total = 0;\n for (int n : nums) total += n;\n return total;\n}\n", "Vector sum function", 1}, {"cpp", "#include \nclass Node {\npublic:\n int value;\n std::unique_ptr next;\n Node(int v) : value(v), next(nullptr) {}\n};\n\nvoid append(Node* head, int val) {\n Node* curr = head;\n while (curr->next) curr = curr->next.get();\n curr->next = std::make_unique(val);\n}\n", "Linked list with unique_ptr", 2}, // JavaScript samples {"javascript", "function fetchData(url) {\n return fetch(url).then(r => r.json());\n}\n\nfunction processResponse(data) {\n return data.items.filter(item => item.active).map(item => item.name);\n}\n", "Async data fetching", 2}, // Rust samples {"rust", "fn binary_search(arr: &[i32], target: i32) -> Option {\n let mut low = 0;\n let mut high = arr.len();\n while low < high {\n let mid = low + (high - low) / 2;\n if arr[mid] == target { return Some(mid); }\n if arr[mid] < target { low = mid + 1; }\n else { high = mid; }\n }\n None\n}\n", "Binary search implementation", 2}, // Kotlin samples {"kotlin", "fun greet(name: String): String {\n return \"Hello, $name!\"\n}\n\nfun sum(a: Int, b: Int): Int = a + b\n", "Simple Kotlin functions", 1}, {"kotlin", "data class Point(val x: Double, val y: Double)\n\nfun distance(a: Point, b: Point): Double {\n val dx = a.x - b.x\n val dy = a.y - b.y\n return Math.sqrt(dx * dx + dy * dy)\n}\n", "Kotlin data class and math", 2}, // C# samples {"csharp", "public class Calculator {\n public int Add(int a, int b) {\n return a + b;\n }\n public int Multiply(int a, int b) {\n return a * b;\n }\n}\n", "Simple C# calculator class", 1}, {"csharp", "using System.Collections.Generic;\n\npublic class Stack {\n private List items = new List();\n public void Push(T item) { items.Add(item); }\n public T Pop() { var item = items[items.Count - 1]; items.RemoveAt(items.Count - 1); return item; }\n public int Count => items.Count;\n}\n", "Generic stack implementation", 2}, // Go samples {"go", "package main\n\nfunc fibonacci(n int) int {\n if n <= 1 {\n return n\n }\n return fibonacci(n-1) + fibonacci(n-2)\n}\n", "Recursive fibonacci", 1}, // Java samples {"java", "public class StringUtils {\n public static String reverse(String s) {\n return new StringBuilder(s).reverse().toString();\n }\n public static boolean isPalindrome(String s) {\n String rev = reverse(s);\n return s.equals(rev);\n }\n}\n", "String utility methods", 1}, }; } // ----------------------------------------------------------------------- // Step 217: Trace generator engine // ----------------------------------------------------------------------- class TraceGenerator { public: TraceGenerator() : corpus_(getBuiltinCorpus()) {} void setSeed(unsigned int seed) { seed_ = seed; srand(seed); } // Generate a single trace for a given scenario Trace generate(ScenarioType scenario, const ScenarioParams& params = {}) { if (params.seed != 0) setSeed(params.seed); switch (scenario) { case ScenarioType::ReadAndUnderstand: return generateReadAndUnderstand(params); case ScenarioType::AddAnnotations: return generateAddAnnotations(params); case ScenarioType::CrossLanguage: return generateCrossLanguage(params); case ScenarioType::Refactor: return generateRefactor(params); case ScenarioType::SecurityAudit: return generateSecurityAudit(params); case ScenarioType::MultiStepDebug: return generateMultiStepDebug(params); case ScenarioType::AnnotateAllSubjects: return generateAnnotateAllSubjects(params); case ScenarioType::ValidateAndFix: return generateValidateAndFix(params); case ScenarioType::SemannoExport: return generateSemannoExport(params); case ScenarioType::CrossLanguageAnnotated: return generateCrossLanguageAnnotated(params); } return {}; } // Generate a batch of traces with varied scenarios std::vector generateBatch(int count, unsigned int baseSeed = 42) { std::vector traces; std::vector types = { ScenarioType::ReadAndUnderstand, ScenarioType::AddAnnotations, ScenarioType::CrossLanguage, ScenarioType::Refactor, ScenarioType::SecurityAudit, ScenarioType::MultiStepDebug, ScenarioType::AnnotateAllSubjects, ScenarioType::ValidateAndFix, ScenarioType::SemannoExport, ScenarioType::CrossLanguageAnnotated }; for (int i = 0; i < count; ++i) { ScenarioType type = types[i % types.size()]; ScenarioParams params; params.seed = baseSeed + i; params.language = (i % 2 == 0) ? "python" : "cpp"; params.functionCount = 2 + (i % 3); traces.push_back(generate(type, params)); } return traces; } const std::vector& corpus() const { return corpus_; } private: std::vector corpus_; unsigned int seed_ = 42; int traceCounter_ = 0; std::string nextId() { return "trace_" + std::to_string(++traceCounter_); } const CodeSample& pickSample(const std::string& language) { std::vector matches; for (const auto& s : corpus_) { if (s.language == language) matches.push_back(&s); } if (matches.empty()) return corpus_[0]; return *matches[rand() % matches.size()]; } void addToolUsed(Trace& trace, const std::string& tool) { for (const auto& t : trace.toolsUsed) { if (t == tool) return; } trace.toolsUsed.push_back(tool); } // --- Scenario: Read & Understand --- Trace generateReadAndUnderstand(const ScenarioParams& params) { Trace trace; trace.id = nextId(); trace.scenario = "read_and_understand"; trace.difficulty = "basic"; trace.language = params.language; const auto& sample = pickSample(params.language); // User asks to understand the code trace.steps.push_back({"user", "Read the current AST and describe its structure. " "What functions are defined and what do they do?", "", {}, {}}); // Assistant thinks trace.steps.push_back({"assistant", "I'll read the AST to understand the code structure.", "", {}, {}}); // Tool call: getAST json astResult = simulateGetAST(sample.source, params.language); trace.steps.push_back({"tool_call", "", "whetstone_get_ast", json::object(), {}}); trace.steps.push_back({"tool_result", "", "whetstone_get_ast", {}, astResult}); trace.toolCallCount += 1; addToolUsed(trace, "whetstone_get_ast"); // Assistant summarizes trace.steps.push_back({"assistant", "The module contains " + std::to_string(params.functionCount) + " functions in " + params.language + ". " + sample.description, "", {}, {}}); return trace; } // --- Scenario: Add Annotations --- Trace generateAddAnnotations(const ScenarioParams& params) { Trace trace; trace.id = nextId(); trace.scenario = "add_annotations"; trace.difficulty = "intermediate"; trace.language = params.language; const auto& sample = pickSample(params.language); trace.steps.push_back({"user", "Suggest memory annotations for all functions in the current module.", "", {}, {}}); trace.steps.push_back({"assistant", "I'll get the AST first, then check each function for annotation suggestions.", "", {}, {}}); // Get AST json astResult = simulateGetAST(sample.source, params.language); trace.steps.push_back({"tool_call", "", "whetstone_get_ast", json::object(), {}}); trace.steps.push_back({"tool_result", "", "whetstone_get_ast", {}, astResult}); trace.toolCallCount += 1; addToolUsed(trace, "whetstone_get_ast"); // Suggest annotations json suggestInput = {{"nodeId", "mod1"}}; json suggestResult = { {"scopeId", "mod1"}, {"suggestions", json::array({{ {"nodeId", "mod1"}, {"annotationType", "ReclaimAnnotation"}, {"strategy", "Tracing"}, {"reason", "Python module uses garbage collection"}, {"confidence", 0.85} }})}, {"diagnostics", json::array()} }; trace.steps.push_back({"tool_call", "", "whetstone_suggest_annotations", suggestInput, {}}); trace.steps.push_back({"tool_result", "", "whetstone_suggest_annotations", {}, suggestResult}); trace.toolCallCount += 1; addToolUsed(trace, "whetstone_suggest_annotations"); trace.steps.push_back({"assistant", "Found a suggestion: @Reclaim(Tracing) for the module (confidence: 0.85). " "This makes sense since Python uses garbage collection. Applying it.", "", {}, {}}); // Apply annotation json applyInput = { {"nodeId", "mod1"}, {"annotationType", "ReclaimAnnotation"}, {"strategy", "Tracing"}, {"reason", "Python GC"}, {"confidence", 0.85} }; json applyResult = {{"success", true}, {"warning", ""}}; trace.steps.push_back({"tool_call", "", "whetstone_apply_annotation", applyInput, {}}); trace.steps.push_back({"tool_result", "", "whetstone_apply_annotation", {}, applyResult}); trace.toolCallCount += 1; addToolUsed(trace, "whetstone_apply_annotation"); trace.steps.push_back({"assistant", "Applied @Reclaim(Tracing) annotation to the module. " "All functions now have appropriate memory strategy annotations.", "", {}, {}}); return trace; } // --- Scenario: Cross-Language --- Trace generateCrossLanguage(const ScenarioParams& params) { Trace trace; trace.id = nextId(); trace.scenario = "cross_language"; trace.difficulty = "advanced"; trace.language = params.language; std::string target = (params.language == "python") ? "cpp" : "python"; trace.steps.push_back({"user", "Project this " + params.language + " code to " + target + ".", "", {}, {}}); trace.steps.push_back({"assistant", "I'll run the pipeline to project the code from " + params.language + " to " + target + ".", "", {}, {}}); json pipelineInput = { {"source", pickSample(params.language).source}, {"sourceLanguage", params.language}, {"targetLanguage", target} }; json pipelineResult = { {"success", true}, {"generatedCode", "// Generated " + target + " code"}, {"parseDiagnostics", json::array()}, {"validationDiagnostics", json::array()}, {"violations", json::array()}, {"suggestions", json::array()}, {"foldCount", 0}, {"dceCount", 0} }; trace.steps.push_back({"tool_call", "", "whetstone_run_pipeline", pipelineInput, {}}); trace.steps.push_back({"tool_result", "", "whetstone_run_pipeline", {}, pipelineResult}); trace.toolCallCount += 1; addToolUsed(trace, "whetstone_run_pipeline"); trace.steps.push_back({"assistant", "Successfully projected the code to " + target + ". " "The annotations were adapted for the target language.", "", {}, {}}); return trace; } // --- Scenario: Refactor --- Trace generateRefactor(const ScenarioParams& params) { Trace trace; trace.id = nextId(); trace.scenario = "refactor"; trace.difficulty = "intermediate"; trace.language = params.language; trace.steps.push_back({"user", "Rename the first function in the module to 'compute'.", "", {}, {}}); trace.steps.push_back({"assistant", "I'll get the AST, find the first function, and rename it.", "", {}, {}}); json astResult = simulateGetAST(pickSample(params.language).source, params.language); trace.steps.push_back({"tool_call", "", "whetstone_get_ast", json::object(), {}}); trace.steps.push_back({"tool_result", "", "whetstone_get_ast", {}, astResult}); trace.toolCallCount += 1; addToolUsed(trace, "whetstone_get_ast"); json mutateInput = { {"type", "setProperty"}, {"nodeId", "fn1"}, {"property", "name"}, {"value", "compute"} }; json mutateResult = {{"success", true}, {"warning", ""}}; trace.steps.push_back({"tool_call", "", "whetstone_mutate", mutateInput, {}}); trace.steps.push_back({"tool_result", "", "whetstone_mutate", {}, mutateResult}); trace.toolCallCount += 1; addToolUsed(trace, "whetstone_mutate"); trace.steps.push_back({"assistant", "Renamed the first function to 'compute'. The AST has been updated.", "", {}, {}}); return trace; } // --- Scenario: Security Audit --- Trace generateSecurityAudit(const ScenarioParams& params) { Trace trace; trace.id = nextId(); trace.scenario = "security_audit"; trace.difficulty = "intermediate"; trace.language = params.language; trace.steps.push_back({"user", "Check the project dependencies for security vulnerabilities.", "", {}, {}}); trace.steps.push_back({"assistant", "I'll examine the AST to identify imported libraries, then check for vulnerabilities.", "", {}, {}}); json astResult = simulateGetAST(pickSample(params.language).source, params.language); trace.steps.push_back({"tool_call", "", "whetstone_get_ast", json::object(), {}}); trace.steps.push_back({"tool_result", "", "whetstone_get_ast", {}, astResult}); trace.toolCallCount += 1; addToolUsed(trace, "whetstone_get_ast"); trace.steps.push_back({"assistant", "The module has no external imports with known vulnerabilities. " "The code uses standard library functions only.", "", {}, {}}); return trace; } // --- Scenario: Multi-Step Debug --- Trace generateMultiStepDebug(const ScenarioParams& params) { Trace trace; trace.id = nextId(); trace.scenario = "multi_step_debug"; trace.difficulty = "advanced"; trace.language = params.language; trace.steps.push_back({"user", "Check the code for annotation errors and fix them.", "", {}, {}}); trace.steps.push_back({"assistant", "I'll get the AST, check for annotation issues, and fix any problems.", "", {}, {}}); // Get AST json astResult = simulateGetAST(pickSample(params.language).source, params.language); trace.steps.push_back({"tool_call", "", "whetstone_get_ast", json::object(), {}}); trace.steps.push_back({"tool_result", "", "whetstone_get_ast", {}, astResult}); trace.toolCallCount += 1; addToolUsed(trace, "whetstone_get_ast"); // Suggest annotations json suggestResult = { {"scopeId", "mod1"}, {"suggestions", json::array({{ {"nodeId", "fn1"}, {"annotationType", "ReclaimAnnotation"}, {"strategy", "Tracing"}, {"reason", "Unannotated function"}, {"confidence", 0.7} }})}, {"diagnostics", json::array({{ {"severity", "warning"}, {"message", "Function 'fn1' has no memory annotation"}, {"nodeId", "fn1"} }})} }; trace.steps.push_back({"tool_call", "", "whetstone_suggest_annotations", {{"nodeId", "fn1"}}, {}}); trace.steps.push_back({"tool_result", "", "whetstone_suggest_annotations", {}, suggestResult}); trace.toolCallCount += 1; addToolUsed(trace, "whetstone_suggest_annotations"); trace.steps.push_back({"assistant", "Found a warning: function 'fn1' has no memory annotation. " "Suggested @Reclaim(Tracing) with confidence 0.7. Applying the fix.", "", {}, {}}); // Apply fix json applyResult = {{"success", true}, {"warning", ""}}; trace.steps.push_back({"tool_call", "", "whetstone_apply_annotation", {{"nodeId", "fn1"}, {"annotationType", "ReclaimAnnotation"}, {"strategy", "Tracing"}, {"reason", "fix"}, {"confidence", 0.7}}, {}}); trace.steps.push_back({"tool_result", "", "whetstone_apply_annotation", {}, applyResult}); trace.toolCallCount += 1; addToolUsed(trace, "whetstone_apply_annotation"); trace.steps.push_back({"assistant", "Fixed the annotation issue. All functions now have proper memory annotations.", "", {}, {}}); return trace; } // --- Scenario: Annotate All Subjects --- Trace generateAnnotateAllSubjects(const ScenarioParams& params) { Trace trace; trace.id = nextId(); trace.scenario = "annotate_all_subjects"; trace.difficulty = "advanced"; trace.language = params.language; const auto& sample = pickSample(params.language); trace.steps.push_back({"user", "Add annotations from all 8 subjects (memory, type system, concurrency, " "scope, shims, optimization, meta-programming, policy) to this code.", "", {}, {}}); trace.steps.push_back({"assistant", "I'll analyze the code and apply annotations across all subject areas.", "", {}, {}}); json astResult = simulateGetAST(sample.source, params.language); trace.steps.push_back({"tool_call", "", "whetstone_get_ast", json::object(), {}}); trace.steps.push_back({"tool_result", "", "whetstone_get_ast", {}, astResult}); trace.toolCallCount += 1; addToolUsed(trace, "whetstone_get_ast"); trace.steps.push_back({"assistant", "Applied annotations: @Reclaim(Tracing), @BitWidth(64), @Exec(sync), " "@Visibility(public), @TailCall, @Policy(strict). All 8 subjects covered.", "", {}, {}}); return trace; } // --- Scenario: Validate and Fix --- Trace generateValidateAndFix(const ScenarioParams& params) { Trace trace; trace.id = nextId(); trace.scenario = "validate_and_fix"; trace.difficulty = "intermediate"; trace.language = params.language; trace.steps.push_back({"user", "Validate all annotations and fix any conflicts or errors.", "", {}, {}}); trace.steps.push_back({"assistant", "I'll run validation and conflict detection, then fix issues.", "", {}, {}}); json suggestResult = { {"scopeId", "mod1"}, {"suggestions", json::array()}, {"diagnostics", json::array({{ {"severity", "error"}, {"message", "E0700: @Pure conflicts with @Blocking"}, {"nodeId", "fn1"} }})} }; trace.steps.push_back({"tool_call", "", "whetstone_suggest_annotations", json::object(), {}}); trace.steps.push_back({"tool_result", "", "whetstone_suggest_annotations", {}, suggestResult}); trace.toolCallCount += 1; addToolUsed(trace, "whetstone_suggest_annotations"); trace.steps.push_back({"assistant", "Found conflict E0700: @Pure vs @Blocking. Removed @Blocking to resolve.", "", {}, {}}); return trace; } // --- Scenario: Semanno Export --- Trace generateSemannoExport(const ScenarioParams& params) { Trace trace; trace.id = nextId(); trace.scenario = "semanno_export"; trace.difficulty = "basic"; trace.language = params.language; trace.steps.push_back({"user", "Export the annotated code with Semanno inline comments.", "", {}, {}}); trace.steps.push_back({"assistant", "I'll generate code with @semanno inline comments for all annotations.", "", {}, {}}); const auto& sample = pickSample(params.language); json pipelineResult = { {"success", true}, {"generatedCode", "// @semanno:intent(summary=\"math helpers\")\n" + sample.source}, {"parseDiagnostics", json::array()}, {"validationDiagnostics", json::array()}, {"violations", json::array()}, {"suggestions", json::array()}, {"foldCount", 0}, {"dceCount", 0} }; trace.steps.push_back({"tool_call", "", "whetstone_run_pipeline", {{"source", sample.source}, {"sourceLanguage", params.language}, {"targetLanguage", params.language}}, {}}); trace.steps.push_back({"tool_result", "", "whetstone_run_pipeline", {}, pipelineResult}); trace.toolCallCount += 1; addToolUsed(trace, "whetstone_run_pipeline"); trace.steps.push_back({"assistant", "Exported code with Semanno comments. All annotations are now " "embedded as parseable inline comments.", "", {}, {}}); return trace; } // --- Scenario: Cross-Language Annotated --- Trace generateCrossLanguageAnnotated(const ScenarioParams& params) { Trace trace; trace.id = nextId(); trace.scenario = "cross_language_annotated"; trace.difficulty = "advanced"; trace.language = params.language; std::string target = (params.language == "python") ? "kotlin" : (params.language == "kotlin") ? "csharp" : "python"; trace.steps.push_back({"user", "Project this annotated " + params.language + " code to " + target + ", preserving all semantic annotations.", "", {}, {}}); trace.steps.push_back({"assistant", "I'll project to " + target + " while ensuring annotations are preserved.", "", {}, {}}); json pipelineResult = { {"success", true}, {"generatedCode", "// Generated annotated " + target + " code"}, {"parseDiagnostics", json::array()}, {"validationDiagnostics", json::array()}, {"violations", json::array()}, {"suggestions", json::array()}, {"foldCount", 0}, {"dceCount", 0} }; trace.steps.push_back({"tool_call", "", "whetstone_run_pipeline", {{"source", pickSample(params.language).source}, {"sourceLanguage", params.language}, {"targetLanguage", target}}, {}}); trace.steps.push_back({"tool_result", "", "whetstone_run_pipeline", {}, pipelineResult}); trace.toolCallCount += 1; addToolUsed(trace, "whetstone_run_pipeline"); trace.steps.push_back({"assistant", "Successfully projected to " + target + " with all annotations preserved. " "Semanno comments are language-appropriate.", "", {}, {}}); return trace; } // Simulate getAST result for a code sample json simulateGetAST(const std::string& source, const std::string& language) { // Use real parser if possible, fall back to mock Pipeline pipeline; std::vector diags; auto mod = pipeline.parse(source, language, diags); if (mod) { return { {"ast", toJson(mod.get())}, {"annotationCount", 0}, {"diagnostics", json::array()} }; } // Mock fallback return { {"ast", {{"conceptType", "Module"}, {"id", "mod1"}, {"name", "parsed"}}}, {"annotationCount", 0}, {"diagnostics", json::array()} }; } };