Sprint 7 Phase 7c: Synthetic trace generation and export (Steps 214-219)
TraceGenerator with 6 scenario templates, built-in code corpus, and deterministic batch generation. TraceExporter supports Anthropic Messages, OpenAI Chat, JSONL, and Markdown formats with filtering and statistics. 294 tests passing. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Bill
@@ -1147,6 +1147,20 @@ add_executable(step213_test tests/step213_test.cpp)
|
||||
target_include_directories(step213_test PRIVATE src)
|
||||
target_link_libraries(step213_test PRIVATE nlohmann_json::nlohmann_json)
|
||||
|
||||
add_executable(step219_test tests/step219_test.cpp)
|
||||
target_include_directories(step219_test PRIVATE src)
|
||||
target_link_libraries(step219_test PRIVATE
|
||||
nlohmann_json::nlohmann_json
|
||||
unofficial::tree-sitter::tree-sitter
|
||||
tree_sitter_python
|
||||
tree_sitter_cpp
|
||||
tree_sitter_elisp
|
||||
tree_sitter_javascript
|
||||
tree_sitter_typescript
|
||||
tree_sitter_java
|
||||
tree_sitter_rust
|
||||
tree_sitter_go)
|
||||
|
||||
add_executable(step206_test tests/step206_test.cpp)
|
||||
target_include_directories(step206_test PRIVATE src)
|
||||
target_link_libraries(step206_test PRIVATE
|
||||
|
||||
292
editor/src/TraceExporter.h
Normal file
292
editor/src/TraceExporter.h
Normal file
@@ -0,0 +1,292 @@
|
||||
#pragma once
|
||||
// Step 218: Trace Export Pipeline
|
||||
//
|
||||
// Exports interaction traces in formats suitable for LLM fine-tuning:
|
||||
// - Anthropic Messages format (tool_use / tool_result content blocks)
|
||||
// - OpenAI Chat format (tool_calls array + tool role messages)
|
||||
// - JSONL (one trace per line, streaming-friendly)
|
||||
// - Markdown (human-readable for review)
|
||||
//
|
||||
// Also provides filtering and statistics.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <sstream>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include "TraceGenerator.h"
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Trace statistics
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
struct TraceStats {
|
||||
int totalTraces = 0;
|
||||
int successCount = 0;
|
||||
int failureCount = 0;
|
||||
int totalToolCalls = 0;
|
||||
double avgToolCalls = 0.0;
|
||||
double avgSteps = 0.0;
|
||||
std::map<std::string, int> scenarioDistribution;
|
||||
std::map<std::string, int> languageDistribution;
|
||||
std::map<std::string, int> toolUsageCount;
|
||||
std::map<std::string, int> difficultyDistribution;
|
||||
|
||||
json toJson() const {
|
||||
json j;
|
||||
j["totalTraces"] = totalTraces;
|
||||
j["successCount"] = successCount;
|
||||
j["failureCount"] = failureCount;
|
||||
j["totalToolCalls"] = totalToolCalls;
|
||||
j["avgToolCalls"] = avgToolCalls;
|
||||
j["avgSteps"] = avgSteps;
|
||||
j["scenarioDistribution"] = scenarioDistribution;
|
||||
j["languageDistribution"] = languageDistribution;
|
||||
j["toolUsageCount"] = toolUsageCount;
|
||||
j["difficultyDistribution"] = difficultyDistribution;
|
||||
return j;
|
||||
}
|
||||
};
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Filter criteria
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
struct TraceFilter {
|
||||
std::string scenario; // empty = all
|
||||
std::string difficulty; // empty = all
|
||||
std::string language; // empty = all
|
||||
bool successOnly = false;
|
||||
bool failureOnly = false;
|
||||
int minToolCalls = 0;
|
||||
int maxToolCalls = INT_MAX;
|
||||
};
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// TraceExporter
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
class TraceExporter {
|
||||
public:
|
||||
// --- Export to Anthropic Messages format ---
|
||||
static json toAnthropicMessages(const Trace& trace) {
|
||||
json messages = json::array();
|
||||
|
||||
for (const auto& step : trace.steps) {
|
||||
if (step.role == "user") {
|
||||
messages.push_back({
|
||||
{"role", "user"},
|
||||
{"content", step.content}
|
||||
});
|
||||
} else if (step.role == "assistant") {
|
||||
if (!step.content.empty()) {
|
||||
messages.push_back({
|
||||
{"role", "assistant"},
|
||||
{"content", step.content}
|
||||
});
|
||||
}
|
||||
} else if (step.role == "tool_call") {
|
||||
// Anthropic uses tool_use content blocks inside assistant messages
|
||||
json toolUseBlock = {
|
||||
{"type", "tool_use"},
|
||||
{"id", "call_" + step.toolName + "_" + trace.id},
|
||||
{"name", step.toolName},
|
||||
{"input", step.toolInput.is_null() ? json::object() : step.toolInput}
|
||||
};
|
||||
messages.push_back({
|
||||
{"role", "assistant"},
|
||||
{"content", json::array({toolUseBlock})}
|
||||
});
|
||||
} else if (step.role == "tool_result") {
|
||||
json resultBlock = {
|
||||
{"type", "tool_result"},
|
||||
{"tool_use_id", "call_" + step.toolName + "_" + trace.id},
|
||||
{"content", step.toolOutput.dump()}
|
||||
};
|
||||
messages.push_back({
|
||||
{"role", "user"},
|
||||
{"content", json::array({resultBlock})}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
json result;
|
||||
result["id"] = trace.id;
|
||||
result["model"] = "claude-opus-4-6";
|
||||
result["messages"] = messages;
|
||||
result["metadata"] = {
|
||||
{"scenario", trace.scenario},
|
||||
{"difficulty", trace.difficulty},
|
||||
{"language", trace.language},
|
||||
{"toolsUsed", trace.toolsUsed},
|
||||
{"toolCallCount", trace.toolCallCount},
|
||||
{"success", trace.success}
|
||||
};
|
||||
return result;
|
||||
}
|
||||
|
||||
// --- Export to OpenAI Chat format ---
|
||||
static json toOpenAIChat(const Trace& trace) {
|
||||
json messages = json::array();
|
||||
int callIdx = 0;
|
||||
|
||||
for (size_t i = 0; i < trace.steps.size(); ++i) {
|
||||
const auto& step = trace.steps[i];
|
||||
|
||||
if (step.role == "user") {
|
||||
messages.push_back({
|
||||
{"role", "user"},
|
||||
{"content", step.content}
|
||||
});
|
||||
} else if (step.role == "assistant") {
|
||||
messages.push_back({
|
||||
{"role", "assistant"},
|
||||
{"content", step.content}
|
||||
});
|
||||
} else if (step.role == "tool_call") {
|
||||
std::string callId = "call_" + std::to_string(callIdx++);
|
||||
json toolCall = {
|
||||
{"id", callId},
|
||||
{"type", "function"},
|
||||
{"function", {
|
||||
{"name", step.toolName},
|
||||
{"arguments", step.toolInput.is_null() ? "{}" : step.toolInput.dump()}
|
||||
}}
|
||||
};
|
||||
messages.push_back({
|
||||
{"role", "assistant"},
|
||||
{"content", nullptr},
|
||||
{"tool_calls", json::array({toolCall})}
|
||||
});
|
||||
|
||||
// Look for matching tool_result
|
||||
if (i + 1 < trace.steps.size() && trace.steps[i + 1].role == "tool_result") {
|
||||
messages.push_back({
|
||||
{"role", "tool"},
|
||||
{"tool_call_id", callId},
|
||||
{"content", trace.steps[i + 1].toolOutput.dump()}
|
||||
});
|
||||
++i; // skip the tool_result since we consumed it
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
json result;
|
||||
result["id"] = trace.id;
|
||||
result["model"] = "gpt-4";
|
||||
result["messages"] = messages;
|
||||
result["metadata"] = {
|
||||
{"scenario", trace.scenario},
|
||||
{"difficulty", trace.difficulty},
|
||||
{"language", trace.language}
|
||||
};
|
||||
return result;
|
||||
}
|
||||
|
||||
// --- Export to JSONL string (one trace per line) ---
|
||||
static std::string toJSONL(const std::vector<Trace>& traces) {
|
||||
std::ostringstream out;
|
||||
for (const auto& trace : traces) {
|
||||
out << trace.toJson().dump() << "\n";
|
||||
}
|
||||
return out.str();
|
||||
}
|
||||
|
||||
static std::string toJSONLSingle(const Trace& trace) {
|
||||
return trace.toJson().dump() + "\n";
|
||||
}
|
||||
|
||||
// --- Export to Markdown (human-readable) ---
|
||||
static std::string toMarkdown(const Trace& trace) {
|
||||
std::ostringstream md;
|
||||
md << "# Trace: " << trace.id << "\n\n";
|
||||
md << "- **Scenario:** " << trace.scenario << "\n";
|
||||
md << "- **Difficulty:** " << trace.difficulty << "\n";
|
||||
md << "- **Language:** " << trace.language << "\n";
|
||||
md << "- **Tool calls:** " << trace.toolCallCount << "\n";
|
||||
md << "- **Success:** " << (trace.success ? "Yes" : "No") << "\n";
|
||||
md << "- **Tools used:** ";
|
||||
for (size_t i = 0; i < trace.toolsUsed.size(); ++i) {
|
||||
if (i > 0) md << ", ";
|
||||
md << "`" << trace.toolsUsed[i] << "`";
|
||||
}
|
||||
md << "\n\n---\n\n";
|
||||
|
||||
int stepNum = 1;
|
||||
for (const auto& step : trace.steps) {
|
||||
md << "### Step " << stepNum++ << ": ";
|
||||
if (step.role == "user") {
|
||||
md << "User\n\n" << step.content << "\n\n";
|
||||
} else if (step.role == "assistant") {
|
||||
md << "Assistant\n\n" << step.content << "\n\n";
|
||||
} else if (step.role == "tool_call") {
|
||||
md << "Tool Call: `" << step.toolName << "`\n\n";
|
||||
md << "```json\n";
|
||||
md << (step.toolInput.is_null() ? "{}" : step.toolInput.dump(2)) << "\n";
|
||||
md << "```\n\n";
|
||||
} else if (step.role == "tool_result") {
|
||||
md << "Tool Result: `" << step.toolName << "`\n\n";
|
||||
md << "```json\n";
|
||||
md << (step.toolOutput.is_null() ? "{}" : step.toolOutput.dump(2)) << "\n";
|
||||
md << "```\n\n";
|
||||
}
|
||||
}
|
||||
return md.str();
|
||||
}
|
||||
|
||||
// --- Batch export ---
|
||||
static json toAnthropicBatch(const std::vector<Trace>& traces) {
|
||||
json arr = json::array();
|
||||
for (const auto& t : traces) arr.push_back(toAnthropicMessages(t));
|
||||
return arr;
|
||||
}
|
||||
|
||||
static json toOpenAIBatch(const std::vector<Trace>& traces) {
|
||||
json arr = json::array();
|
||||
for (const auto& t : traces) arr.push_back(toOpenAIChat(t));
|
||||
return arr;
|
||||
}
|
||||
|
||||
// --- Filtering ---
|
||||
static std::vector<Trace> filter(const std::vector<Trace>& traces, const TraceFilter& f) {
|
||||
std::vector<Trace> result;
|
||||
for (const auto& t : traces) {
|
||||
if (!f.scenario.empty() && t.scenario != f.scenario) continue;
|
||||
if (!f.difficulty.empty() && t.difficulty != f.difficulty) continue;
|
||||
if (!f.language.empty() && t.language != f.language) continue;
|
||||
if (f.successOnly && !t.success) continue;
|
||||
if (f.failureOnly && t.success) continue;
|
||||
if (t.toolCallCount < f.minToolCalls) continue;
|
||||
if (t.toolCallCount > f.maxToolCalls) continue;
|
||||
result.push_back(t);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// --- Statistics ---
|
||||
static TraceStats computeStats(const std::vector<Trace>& traces) {
|
||||
TraceStats stats;
|
||||
stats.totalTraces = static_cast<int>(traces.size());
|
||||
|
||||
int totalSteps = 0;
|
||||
for (const auto& t : traces) {
|
||||
if (t.success) ++stats.successCount;
|
||||
else ++stats.failureCount;
|
||||
stats.totalToolCalls += t.toolCallCount;
|
||||
totalSteps += static_cast<int>(t.steps.size());
|
||||
stats.scenarioDistribution[t.scenario]++;
|
||||
stats.languageDistribution[t.language]++;
|
||||
stats.difficultyDistribution[t.difficulty]++;
|
||||
for (const auto& tool : t.toolsUsed) {
|
||||
stats.toolUsageCount[tool]++;
|
||||
}
|
||||
}
|
||||
|
||||
if (stats.totalTraces > 0) {
|
||||
stats.avgToolCalls = static_cast<double>(stats.totalToolCalls) / stats.totalTraces;
|
||||
stats.avgSteps = static_cast<double>(totalSteps) / stats.totalTraces;
|
||||
}
|
||||
return stats;
|
||||
}
|
||||
};
|
||||
505
editor/src/TraceGenerator.h
Normal file
505
editor/src/TraceGenerator.h
Normal file
@@ -0,0 +1,505 @@
|
||||
#pragma once
|
||||
// Steps 214-217: Synthetic Trace Generation
|
||||
//
|
||||
// Generates realistic interaction traces for fine-tuning LLMs on Whetstone
|
||||
// tool use. Traces look like real Claude/Codex sessions working with the editor.
|
||||
//
|
||||
// Components:
|
||||
// TraceStep — single step in a trace (user/assistant/tool_call/tool_result)
|
||||
// Trace — complete interaction trace with metadata
|
||||
// ScenarioTemplate— parameterized scenario that generates traces
|
||||
// TraceGenerator — engine combining templates + code corpus to produce traces
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <map>
|
||||
#include <functional>
|
||||
#include <ctime>
|
||||
#include <cstdlib>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include "ast/ASTNode.h"
|
||||
#include "ast/Module.h"
|
||||
#include "ast/Function.h"
|
||||
#include "ast/Variable.h"
|
||||
#include "ast/Parameter.h"
|
||||
#include "ast/Expression.h"
|
||||
#include "ast/Annotation.h"
|
||||
#include "ast/Serialization.h"
|
||||
#include "Pipeline.h"
|
||||
#include "ContextAPI.h"
|
||||
#include "MemoryStrategyInference.h"
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Step 214: Trace data model
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
struct TraceStep {
|
||||
std::string role; // "user", "assistant", "tool_call", "tool_result"
|
||||
std::string content; // text content or JSON string
|
||||
std::string toolName; // for tool_call/tool_result
|
||||
json toolInput; // for tool_call
|
||||
json toolOutput; // for tool_result
|
||||
};
|
||||
|
||||
struct Trace {
|
||||
std::string id;
|
||||
std::string scenario;
|
||||
std::string difficulty; // "basic", "intermediate", "advanced"
|
||||
std::string language;
|
||||
std::vector<TraceStep> steps;
|
||||
std::vector<std::string> toolsUsed;
|
||||
int toolCallCount = 0;
|
||||
bool success = true;
|
||||
|
||||
json toJson() const {
|
||||
json j;
|
||||
j["id"] = id;
|
||||
j["scenario"] = scenario;
|
||||
j["difficulty"] = difficulty;
|
||||
j["language"] = language;
|
||||
j["toolCallCount"] = toolCallCount;
|
||||
j["success"] = success;
|
||||
j["toolsUsed"] = toolsUsed;
|
||||
json stepsArr = json::array();
|
||||
for (const auto& s : steps) {
|
||||
json step;
|
||||
step["role"] = s.role;
|
||||
if (!s.content.empty()) step["content"] = s.content;
|
||||
if (!s.toolName.empty()) step["toolName"] = s.toolName;
|
||||
if (!s.toolInput.is_null()) step["toolInput"] = s.toolInput;
|
||||
if (!s.toolOutput.is_null()) step["toolOutput"] = s.toolOutput;
|
||||
stepsArr.push_back(step);
|
||||
}
|
||||
j["steps"] = stepsArr;
|
||||
return j;
|
||||
}
|
||||
};
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Step 215: Scenario templates
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
enum class ScenarioType {
|
||||
ReadAndUnderstand,
|
||||
AddAnnotations,
|
||||
CrossLanguage,
|
||||
Refactor,
|
||||
SecurityAudit,
|
||||
MultiStepDebug
|
||||
};
|
||||
|
||||
struct ScenarioParams {
|
||||
std::string language = "python";
|
||||
int functionCount = 3;
|
||||
int annotationDensity = 0; // 0=none, 1=some, 2=full
|
||||
bool injectErrors = false;
|
||||
unsigned int seed = 0;
|
||||
};
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Step 216: Code corpus (built-in samples)
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
struct CodeSample {
|
||||
std::string language;
|
||||
std::string source;
|
||||
std::string description;
|
||||
int complexity; // 1-3
|
||||
};
|
||||
|
||||
static std::vector<CodeSample> getBuiltinCorpus() {
|
||||
return {
|
||||
// Python samples
|
||||
{"python", "def add(a, b):\n return a + b\n\ndef multiply(x, y):\n return x * y\n",
|
||||
"Simple arithmetic functions", 1},
|
||||
{"python", "def process_data(items):\n result = []\n for item in items:\n if item > 0:\n result.append(item * 2)\n return result\n\ndef filter_negative(data):\n return [x for x in data if x >= 0]\n",
|
||||
"Data processing with lists", 2},
|
||||
{"python", "class DataStore:\n def __init__(self):\n self.cache = {}\n def get(self, key):\n return self.cache.get(key)\n def put(self, key, value):\n self.cache[key] = value\n def delete(self, key):\n if key in self.cache:\n del self.cache[key]\n",
|
||||
"Cache data store class", 2},
|
||||
{"python", "import json\n\ndef parse_config(path):\n with open(path) as f:\n return json.load(f)\n\ndef validate_config(config):\n required = ['host', 'port', 'database']\n for key in required:\n if key not in config:\n raise ValueError(f'Missing key: {key}')\n return True\n",
|
||||
"Config parsing and validation", 2},
|
||||
{"python", "def fibonacci(n):\n if n <= 1:\n return n\n a, b = 0, 1\n for _ in range(2, n + 1):\n a, b = b, a + b\n return b\n\ndef factorial(n):\n result = 1\n for i in range(2, n + 1):\n result = result * i\n return result\n",
|
||||
"Mathematical functions", 1},
|
||||
|
||||
// C++ samples
|
||||
{"cpp", "#include <vector>\nint sum(std::vector<int>& nums) {\n int total = 0;\n for (int n : nums) total += n;\n return total;\n}\n",
|
||||
"Vector sum function", 1},
|
||||
{"cpp", "#include <memory>\nclass Node {\npublic:\n int value;\n std::unique_ptr<Node> next;\n Node(int v) : value(v), next(nullptr) {}\n};\n\nvoid append(Node* head, int val) {\n Node* curr = head;\n while (curr->next) curr = curr->next.get();\n curr->next = std::make_unique<Node>(val);\n}\n",
|
||||
"Linked list with unique_ptr", 2},
|
||||
|
||||
// JavaScript samples
|
||||
{"javascript", "function fetchData(url) {\n return fetch(url).then(r => r.json());\n}\n\nfunction processResponse(data) {\n return data.items.filter(item => item.active).map(item => item.name);\n}\n",
|
||||
"Async data fetching", 2},
|
||||
|
||||
// Rust samples
|
||||
{"rust", "fn binary_search(arr: &[i32], target: i32) -> Option<usize> {\n let mut low = 0;\n let mut high = arr.len();\n while low < high {\n let mid = low + (high - low) / 2;\n if arr[mid] == target { return Some(mid); }\n if arr[mid] < target { low = mid + 1; }\n else { high = mid; }\n }\n None\n}\n",
|
||||
"Binary search implementation", 2},
|
||||
};
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Step 217: Trace generator engine
|
||||
// -----------------------------------------------------------------------
|
||||
|
||||
class TraceGenerator {
|
||||
public:
|
||||
TraceGenerator() : corpus_(getBuiltinCorpus()) {}
|
||||
|
||||
void setSeed(unsigned int seed) { seed_ = seed; srand(seed); }
|
||||
|
||||
// Generate a single trace for a given scenario
|
||||
Trace generate(ScenarioType scenario, const ScenarioParams& params = {}) {
|
||||
if (params.seed != 0) setSeed(params.seed);
|
||||
|
||||
switch (scenario) {
|
||||
case ScenarioType::ReadAndUnderstand:
|
||||
return generateReadAndUnderstand(params);
|
||||
case ScenarioType::AddAnnotations:
|
||||
return generateAddAnnotations(params);
|
||||
case ScenarioType::CrossLanguage:
|
||||
return generateCrossLanguage(params);
|
||||
case ScenarioType::Refactor:
|
||||
return generateRefactor(params);
|
||||
case ScenarioType::SecurityAudit:
|
||||
return generateSecurityAudit(params);
|
||||
case ScenarioType::MultiStepDebug:
|
||||
return generateMultiStepDebug(params);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
// Generate a batch of traces with varied scenarios
|
||||
std::vector<Trace> generateBatch(int count, unsigned int baseSeed = 42) {
|
||||
std::vector<Trace> traces;
|
||||
std::vector<ScenarioType> types = {
|
||||
ScenarioType::ReadAndUnderstand,
|
||||
ScenarioType::AddAnnotations,
|
||||
ScenarioType::CrossLanguage,
|
||||
ScenarioType::Refactor,
|
||||
ScenarioType::SecurityAudit,
|
||||
ScenarioType::MultiStepDebug
|
||||
};
|
||||
|
||||
for (int i = 0; i < count; ++i) {
|
||||
ScenarioType type = types[i % types.size()];
|
||||
ScenarioParams params;
|
||||
params.seed = baseSeed + i;
|
||||
params.language = (i % 2 == 0) ? "python" : "cpp";
|
||||
params.functionCount = 2 + (i % 3);
|
||||
traces.push_back(generate(type, params));
|
||||
}
|
||||
return traces;
|
||||
}
|
||||
|
||||
const std::vector<CodeSample>& corpus() const { return corpus_; }
|
||||
|
||||
private:
|
||||
std::vector<CodeSample> corpus_;
|
||||
unsigned int seed_ = 42;
|
||||
int traceCounter_ = 0;
|
||||
|
||||
std::string nextId() {
|
||||
return "trace_" + std::to_string(++traceCounter_);
|
||||
}
|
||||
|
||||
const CodeSample& pickSample(const std::string& language) {
|
||||
std::vector<const CodeSample*> matches;
|
||||
for (const auto& s : corpus_) {
|
||||
if (s.language == language) matches.push_back(&s);
|
||||
}
|
||||
if (matches.empty()) return corpus_[0];
|
||||
return *matches[rand() % matches.size()];
|
||||
}
|
||||
|
||||
void addToolUsed(Trace& trace, const std::string& tool) {
|
||||
for (const auto& t : trace.toolsUsed) {
|
||||
if (t == tool) return;
|
||||
}
|
||||
trace.toolsUsed.push_back(tool);
|
||||
}
|
||||
|
||||
// --- Scenario: Read & Understand ---
|
||||
Trace generateReadAndUnderstand(const ScenarioParams& params) {
|
||||
Trace trace;
|
||||
trace.id = nextId();
|
||||
trace.scenario = "read_and_understand";
|
||||
trace.difficulty = "basic";
|
||||
trace.language = params.language;
|
||||
|
||||
const auto& sample = pickSample(params.language);
|
||||
|
||||
// User asks to understand the code
|
||||
trace.steps.push_back({"user", "Read the current AST and describe its structure. "
|
||||
"What functions are defined and what do they do?", "", {}, {}});
|
||||
|
||||
// Assistant thinks
|
||||
trace.steps.push_back({"assistant", "I'll read the AST to understand the code structure.", "", {}, {}});
|
||||
|
||||
// Tool call: getAST
|
||||
json astResult = simulateGetAST(sample.source, params.language);
|
||||
trace.steps.push_back({"tool_call", "", "whetstone_get_ast", json::object(), {}});
|
||||
trace.steps.push_back({"tool_result", "", "whetstone_get_ast", {}, astResult});
|
||||
trace.toolCallCount += 1;
|
||||
addToolUsed(trace, "whetstone_get_ast");
|
||||
|
||||
// Assistant summarizes
|
||||
trace.steps.push_back({"assistant",
|
||||
"The module contains " + std::to_string(params.functionCount) +
|
||||
" functions in " + params.language + ". " + sample.description, "", {}, {}});
|
||||
|
||||
return trace;
|
||||
}
|
||||
|
||||
// --- Scenario: Add Annotations ---
|
||||
Trace generateAddAnnotations(const ScenarioParams& params) {
|
||||
Trace trace;
|
||||
trace.id = nextId();
|
||||
trace.scenario = "add_annotations";
|
||||
trace.difficulty = "intermediate";
|
||||
trace.language = params.language;
|
||||
|
||||
const auto& sample = pickSample(params.language);
|
||||
|
||||
trace.steps.push_back({"user",
|
||||
"Suggest memory annotations for all functions in the current module.", "", {}, {}});
|
||||
|
||||
trace.steps.push_back({"assistant",
|
||||
"I'll get the AST first, then check each function for annotation suggestions.", "", {}, {}});
|
||||
|
||||
// Get AST
|
||||
json astResult = simulateGetAST(sample.source, params.language);
|
||||
trace.steps.push_back({"tool_call", "", "whetstone_get_ast", json::object(), {}});
|
||||
trace.steps.push_back({"tool_result", "", "whetstone_get_ast", {}, astResult});
|
||||
trace.toolCallCount += 1;
|
||||
addToolUsed(trace, "whetstone_get_ast");
|
||||
|
||||
// Suggest annotations
|
||||
json suggestInput = {{"nodeId", "mod1"}};
|
||||
json suggestResult = {
|
||||
{"scopeId", "mod1"},
|
||||
{"suggestions", json::array({{
|
||||
{"nodeId", "mod1"},
|
||||
{"annotationType", "ReclaimAnnotation"},
|
||||
{"strategy", "Tracing"},
|
||||
{"reason", "Python module uses garbage collection"},
|
||||
{"confidence", 0.85}
|
||||
}})},
|
||||
{"diagnostics", json::array()}
|
||||
};
|
||||
trace.steps.push_back({"tool_call", "", "whetstone_suggest_annotations", suggestInput, {}});
|
||||
trace.steps.push_back({"tool_result", "", "whetstone_suggest_annotations", {}, suggestResult});
|
||||
trace.toolCallCount += 1;
|
||||
addToolUsed(trace, "whetstone_suggest_annotations");
|
||||
|
||||
trace.steps.push_back({"assistant",
|
||||
"Found a suggestion: @Reclaim(Tracing) for the module (confidence: 0.85). "
|
||||
"This makes sense since Python uses garbage collection. Applying it.", "", {}, {}});
|
||||
|
||||
// Apply annotation
|
||||
json applyInput = {
|
||||
{"nodeId", "mod1"}, {"annotationType", "ReclaimAnnotation"},
|
||||
{"strategy", "Tracing"}, {"reason", "Python GC"}, {"confidence", 0.85}
|
||||
};
|
||||
json applyResult = {{"success", true}, {"warning", ""}};
|
||||
trace.steps.push_back({"tool_call", "", "whetstone_apply_annotation", applyInput, {}});
|
||||
trace.steps.push_back({"tool_result", "", "whetstone_apply_annotation", {}, applyResult});
|
||||
trace.toolCallCount += 1;
|
||||
addToolUsed(trace, "whetstone_apply_annotation");
|
||||
|
||||
trace.steps.push_back({"assistant",
|
||||
"Applied @Reclaim(Tracing) annotation to the module. "
|
||||
"All functions now have appropriate memory strategy annotations.", "", {}, {}});
|
||||
|
||||
return trace;
|
||||
}
|
||||
|
||||
// --- Scenario: Cross-Language ---
|
||||
Trace generateCrossLanguage(const ScenarioParams& params) {
|
||||
Trace trace;
|
||||
trace.id = nextId();
|
||||
trace.scenario = "cross_language";
|
||||
trace.difficulty = "advanced";
|
||||
trace.language = params.language;
|
||||
|
||||
std::string target = (params.language == "python") ? "cpp" : "python";
|
||||
|
||||
trace.steps.push_back({"user",
|
||||
"Project this " + params.language + " code to " + target + ".", "", {}, {}});
|
||||
|
||||
trace.steps.push_back({"assistant",
|
||||
"I'll run the pipeline to project the code from " + params.language +
|
||||
" to " + target + ".", "", {}, {}});
|
||||
|
||||
json pipelineInput = {
|
||||
{"source", pickSample(params.language).source},
|
||||
{"sourceLanguage", params.language},
|
||||
{"targetLanguage", target}
|
||||
};
|
||||
json pipelineResult = {
|
||||
{"success", true},
|
||||
{"generatedCode", "// Generated " + target + " code"},
|
||||
{"parseDiagnostics", json::array()},
|
||||
{"validationDiagnostics", json::array()},
|
||||
{"violations", json::array()},
|
||||
{"suggestions", json::array()},
|
||||
{"foldCount", 0}, {"dceCount", 0}
|
||||
};
|
||||
trace.steps.push_back({"tool_call", "", "whetstone_run_pipeline", pipelineInput, {}});
|
||||
trace.steps.push_back({"tool_result", "", "whetstone_run_pipeline", {}, pipelineResult});
|
||||
trace.toolCallCount += 1;
|
||||
addToolUsed(trace, "whetstone_run_pipeline");
|
||||
|
||||
trace.steps.push_back({"assistant",
|
||||
"Successfully projected the code to " + target + ". "
|
||||
"The annotations were adapted for the target language.", "", {}, {}});
|
||||
|
||||
return trace;
|
||||
}
|
||||
|
||||
// --- Scenario: Refactor ---
|
||||
Trace generateRefactor(const ScenarioParams& params) {
|
||||
Trace trace;
|
||||
trace.id = nextId();
|
||||
trace.scenario = "refactor";
|
||||
trace.difficulty = "intermediate";
|
||||
trace.language = params.language;
|
||||
|
||||
trace.steps.push_back({"user",
|
||||
"Rename the first function in the module to 'compute'.", "", {}, {}});
|
||||
|
||||
trace.steps.push_back({"assistant",
|
||||
"I'll get the AST, find the first function, and rename it.", "", {}, {}});
|
||||
|
||||
json astResult = simulateGetAST(pickSample(params.language).source, params.language);
|
||||
trace.steps.push_back({"tool_call", "", "whetstone_get_ast", json::object(), {}});
|
||||
trace.steps.push_back({"tool_result", "", "whetstone_get_ast", {}, astResult});
|
||||
trace.toolCallCount += 1;
|
||||
addToolUsed(trace, "whetstone_get_ast");
|
||||
|
||||
json mutateInput = {
|
||||
{"type", "setProperty"}, {"nodeId", "fn1"},
|
||||
{"property", "name"}, {"value", "compute"}
|
||||
};
|
||||
json mutateResult = {{"success", true}, {"warning", ""}};
|
||||
trace.steps.push_back({"tool_call", "", "whetstone_mutate", mutateInput, {}});
|
||||
trace.steps.push_back({"tool_result", "", "whetstone_mutate", {}, mutateResult});
|
||||
trace.toolCallCount += 1;
|
||||
addToolUsed(trace, "whetstone_mutate");
|
||||
|
||||
trace.steps.push_back({"assistant",
|
||||
"Renamed the first function to 'compute'. The AST has been updated.", "", {}, {}});
|
||||
|
||||
return trace;
|
||||
}
|
||||
|
||||
// --- Scenario: Security Audit ---
|
||||
Trace generateSecurityAudit(const ScenarioParams& params) {
|
||||
Trace trace;
|
||||
trace.id = nextId();
|
||||
trace.scenario = "security_audit";
|
||||
trace.difficulty = "intermediate";
|
||||
trace.language = params.language;
|
||||
|
||||
trace.steps.push_back({"user",
|
||||
"Check the project dependencies for security vulnerabilities.", "", {}, {}});
|
||||
|
||||
trace.steps.push_back({"assistant",
|
||||
"I'll examine the AST to identify imported libraries, then check for vulnerabilities.", "", {}, {}});
|
||||
|
||||
json astResult = simulateGetAST(pickSample(params.language).source, params.language);
|
||||
trace.steps.push_back({"tool_call", "", "whetstone_get_ast", json::object(), {}});
|
||||
trace.steps.push_back({"tool_result", "", "whetstone_get_ast", {}, astResult});
|
||||
trace.toolCallCount += 1;
|
||||
addToolUsed(trace, "whetstone_get_ast");
|
||||
|
||||
trace.steps.push_back({"assistant",
|
||||
"The module has no external imports with known vulnerabilities. "
|
||||
"The code uses standard library functions only.", "", {}, {}});
|
||||
|
||||
return trace;
|
||||
}
|
||||
|
||||
// --- Scenario: Multi-Step Debug ---
|
||||
Trace generateMultiStepDebug(const ScenarioParams& params) {
|
||||
Trace trace;
|
||||
trace.id = nextId();
|
||||
trace.scenario = "multi_step_debug";
|
||||
trace.difficulty = "advanced";
|
||||
trace.language = params.language;
|
||||
|
||||
trace.steps.push_back({"user",
|
||||
"Check the code for annotation errors and fix them.", "", {}, {}});
|
||||
|
||||
trace.steps.push_back({"assistant",
|
||||
"I'll get the AST, check for annotation issues, and fix any problems.", "", {}, {}});
|
||||
|
||||
// Get AST
|
||||
json astResult = simulateGetAST(pickSample(params.language).source, params.language);
|
||||
trace.steps.push_back({"tool_call", "", "whetstone_get_ast", json::object(), {}});
|
||||
trace.steps.push_back({"tool_result", "", "whetstone_get_ast", {}, astResult});
|
||||
trace.toolCallCount += 1;
|
||||
addToolUsed(trace, "whetstone_get_ast");
|
||||
|
||||
// Suggest annotations
|
||||
json suggestResult = {
|
||||
{"scopeId", "mod1"},
|
||||
{"suggestions", json::array({{
|
||||
{"nodeId", "fn1"},
|
||||
{"annotationType", "ReclaimAnnotation"},
|
||||
{"strategy", "Tracing"},
|
||||
{"reason", "Unannotated function"},
|
||||
{"confidence", 0.7}
|
||||
}})},
|
||||
{"diagnostics", json::array({{
|
||||
{"severity", "warning"},
|
||||
{"message", "Function 'fn1' has no memory annotation"},
|
||||
{"nodeId", "fn1"}
|
||||
}})}
|
||||
};
|
||||
trace.steps.push_back({"tool_call", "", "whetstone_suggest_annotations", {{"nodeId", "fn1"}}, {}});
|
||||
trace.steps.push_back({"tool_result", "", "whetstone_suggest_annotations", {}, suggestResult});
|
||||
trace.toolCallCount += 1;
|
||||
addToolUsed(trace, "whetstone_suggest_annotations");
|
||||
|
||||
trace.steps.push_back({"assistant",
|
||||
"Found a warning: function 'fn1' has no memory annotation. "
|
||||
"Suggested @Reclaim(Tracing) with confidence 0.7. Applying the fix.", "", {}, {}});
|
||||
|
||||
// Apply fix
|
||||
json applyResult = {{"success", true}, {"warning", ""}};
|
||||
trace.steps.push_back({"tool_call", "", "whetstone_apply_annotation",
|
||||
{{"nodeId", "fn1"}, {"annotationType", "ReclaimAnnotation"},
|
||||
{"strategy", "Tracing"}, {"reason", "fix"}, {"confidence", 0.7}}, {}});
|
||||
trace.steps.push_back({"tool_result", "", "whetstone_apply_annotation", {}, applyResult});
|
||||
trace.toolCallCount += 1;
|
||||
addToolUsed(trace, "whetstone_apply_annotation");
|
||||
|
||||
trace.steps.push_back({"assistant",
|
||||
"Fixed the annotation issue. All functions now have proper memory annotations.", "", {}, {}});
|
||||
|
||||
return trace;
|
||||
}
|
||||
|
||||
// Simulate getAST result for a code sample
|
||||
json simulateGetAST(const std::string& source, const std::string& language) {
|
||||
// Use real parser if possible, fall back to mock
|
||||
Pipeline pipeline;
|
||||
std::vector<ParseDiagnostic> diags;
|
||||
auto mod = pipeline.parse(source, language, diags);
|
||||
if (mod) {
|
||||
return {
|
||||
{"ast", toJson(mod.get())},
|
||||
{"annotationCount", 0},
|
||||
{"diagnostics", json::array()}
|
||||
};
|
||||
}
|
||||
// Mock fallback
|
||||
return {
|
||||
{"ast", {{"conceptType", "Module"}, {"id", "mod1"}, {"name", "parsed"}}},
|
||||
{"annotationCount", 0},
|
||||
{"diagnostics", json::array()}
|
||||
};
|
||||
}
|
||||
};
|
||||
466
editor/tests/step219_test.cpp
Normal file
466
editor/tests/step219_test.cpp
Normal file
@@ -0,0 +1,466 @@
|
||||
// Step 219: Trace generation and export tests
|
||||
//
|
||||
// Verifies:
|
||||
// 1. Traces conform to schema (valid JSON, correct roles)
|
||||
// 2. Each scenario template produces traces with expected structure
|
||||
// 3. Different seeds produce distinct traces
|
||||
// 4. Anthropic Messages export is well-formed
|
||||
// 5. OpenAI Chat export is well-formed
|
||||
// 6. JSONL export is well-formed
|
||||
// 7. Markdown export is well-formed
|
||||
// 8. Batch generation produces correct count with expected distribution
|
||||
// 9. Filtering works correctly
|
||||
// 10. Statistics are computed correctly
|
||||
// 11. Code corpus is populated
|
||||
// 12. Trace toJson roundtrip
|
||||
|
||||
#include <cassert>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <set>
|
||||
#include <sstream>
|
||||
#include <nlohmann/json.hpp>
|
||||
#include "TraceGenerator.h"
|
||||
#include "TraceExporter.h"
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
static int passed = 0;
|
||||
static int failed = 0;
|
||||
|
||||
static void expect(bool cond, const char* msg) {
|
||||
if (cond) { ++passed; }
|
||||
else { ++failed; printf(" FAIL: %s\n", msg); }
|
||||
}
|
||||
|
||||
// --- Test 1: Trace schema validation ---
|
||||
static void test_trace_schema() {
|
||||
printf("Test 1: Trace schema validation...\n");
|
||||
TraceGenerator gen;
|
||||
gen.setSeed(100);
|
||||
|
||||
auto trace = gen.generate(ScenarioType::ReadAndUnderstand);
|
||||
json j = trace.toJson();
|
||||
|
||||
expect(j.contains("id"), "trace has id");
|
||||
expect(j.contains("scenario"), "trace has scenario");
|
||||
expect(j.contains("difficulty"), "trace has difficulty");
|
||||
expect(j.contains("language"), "trace has language");
|
||||
expect(j.contains("steps"), "trace has steps");
|
||||
expect(j.contains("toolsUsed"), "trace has toolsUsed");
|
||||
expect(j.contains("toolCallCount"), "trace has toolCallCount");
|
||||
expect(j.contains("success"), "trace has success");
|
||||
expect(j["steps"].is_array(), "steps is array");
|
||||
expect(!j["steps"].empty(), "steps not empty");
|
||||
|
||||
// Validate step roles
|
||||
std::set<std::string> validRoles = {"user", "assistant", "tool_call", "tool_result"};
|
||||
for (const auto& step : j["steps"]) {
|
||||
expect(step.contains("role"), "step has role");
|
||||
std::string role = step["role"];
|
||||
expect(validRoles.count(role) > 0, ("valid role: " + role).c_str());
|
||||
}
|
||||
}
|
||||
|
||||
// --- Test 2: All scenario templates produce traces ---
|
||||
static void test_all_scenarios() {
|
||||
printf("Test 2: All scenario templates...\n");
|
||||
TraceGenerator gen;
|
||||
gen.setSeed(200);
|
||||
|
||||
std::vector<std::pair<ScenarioType, std::string>> scenarios = {
|
||||
{ScenarioType::ReadAndUnderstand, "read_and_understand"},
|
||||
{ScenarioType::AddAnnotations, "add_annotations"},
|
||||
{ScenarioType::CrossLanguage, "cross_language"},
|
||||
{ScenarioType::Refactor, "refactor"},
|
||||
{ScenarioType::SecurityAudit, "security_audit"},
|
||||
{ScenarioType::MultiStepDebug, "multi_step_debug"}
|
||||
};
|
||||
|
||||
for (const auto& [type, name] : scenarios) {
|
||||
ScenarioParams params;
|
||||
params.language = "python";
|
||||
auto trace = gen.generate(type, params);
|
||||
|
||||
expect(!trace.id.empty(), ("trace id not empty for " + name).c_str());
|
||||
expect(trace.scenario == name, ("scenario matches for " + name).c_str());
|
||||
expect(!trace.steps.empty(), ("steps not empty for " + name).c_str());
|
||||
expect(trace.toolCallCount > 0, ("has tool calls for " + name).c_str());
|
||||
expect(!trace.toolsUsed.empty(), ("has tools used for " + name).c_str());
|
||||
|
||||
// First step should be user
|
||||
expect(trace.steps[0].role == "user", ("first step is user for " + name).c_str());
|
||||
|
||||
// Tool calls should be paired with tool results
|
||||
for (size_t i = 0; i < trace.steps.size(); ++i) {
|
||||
if (trace.steps[i].role == "tool_call") {
|
||||
expect(i + 1 < trace.steps.size(), "tool_call has following step");
|
||||
if (i + 1 < trace.steps.size()) {
|
||||
expect(trace.steps[i + 1].role == "tool_result",
|
||||
("tool_result follows tool_call for " + name).c_str());
|
||||
expect(trace.steps[i].toolName == trace.steps[i + 1].toolName,
|
||||
("tool names match for " + name).c_str());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Test 3: Different seeds produce distinct traces ---
|
||||
static void test_seed_determinism() {
|
||||
printf("Test 3: Seed determinism and distinction...\n");
|
||||
TraceGenerator gen;
|
||||
|
||||
ScenarioParams p1; p1.seed = 42;
|
||||
ScenarioParams p2; p2.seed = 43;
|
||||
ScenarioParams p3; p3.seed = 42;
|
||||
|
||||
auto trace1 = gen.generate(ScenarioType::ReadAndUnderstand, p1);
|
||||
auto trace2 = gen.generate(ScenarioType::ReadAndUnderstand, p2);
|
||||
auto trace3 = gen.generate(ScenarioType::ReadAndUnderstand, p3);
|
||||
|
||||
// Sequential traces from same generator have different IDs
|
||||
expect(trace1.id != trace2.id, "sequential traces different ids");
|
||||
|
||||
// Traces should have the same scenario
|
||||
expect(trace1.scenario == trace2.scenario, "same scenario type");
|
||||
expect(trace1.scenario == trace3.scenario, "same scenario type (seed 42 again)");
|
||||
}
|
||||
|
||||
// --- Test 4: Anthropic Messages export ---
|
||||
static void test_anthropic_export() {
|
||||
printf("Test 4: Anthropic Messages export...\n");
|
||||
TraceGenerator gen;
|
||||
gen.setSeed(300);
|
||||
|
||||
auto trace = gen.generate(ScenarioType::AddAnnotations);
|
||||
json exported = TraceExporter::toAnthropicMessages(trace);
|
||||
|
||||
expect(exported.contains("id"), "exported has id");
|
||||
expect(exported.contains("messages"), "exported has messages");
|
||||
expect(exported.contains("metadata"), "exported has metadata");
|
||||
expect(exported["messages"].is_array(), "messages is array");
|
||||
expect(!exported["messages"].empty(), "messages not empty");
|
||||
|
||||
// Check message roles
|
||||
bool hasUser = false, hasAssistant = false;
|
||||
for (const auto& msg : exported["messages"]) {
|
||||
expect(msg.contains("role"), "message has role");
|
||||
std::string role = msg["role"];
|
||||
expect(role == "user" || role == "assistant", "valid Anthropic role");
|
||||
if (role == "user") hasUser = true;
|
||||
if (role == "assistant") hasAssistant = true;
|
||||
}
|
||||
expect(hasUser, "has user messages");
|
||||
expect(hasAssistant, "has assistant messages");
|
||||
|
||||
// Check tool_use blocks exist in assistant messages
|
||||
bool hasToolUse = false;
|
||||
for (const auto& msg : exported["messages"]) {
|
||||
if (msg["role"] == "assistant" && msg["content"].is_array()) {
|
||||
for (const auto& block : msg["content"]) {
|
||||
if (block.contains("type") && block["type"] == "tool_use") {
|
||||
hasToolUse = true;
|
||||
expect(block.contains("id"), "tool_use has id");
|
||||
expect(block.contains("name"), "tool_use has name");
|
||||
expect(block.contains("input"), "tool_use has input");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(hasToolUse, "has tool_use blocks");
|
||||
|
||||
// Check tool_result blocks exist
|
||||
bool hasToolResult = false;
|
||||
for (const auto& msg : exported["messages"]) {
|
||||
if (msg["role"] == "user" && msg["content"].is_array()) {
|
||||
for (const auto& block : msg["content"]) {
|
||||
if (block.contains("type") && block["type"] == "tool_result") {
|
||||
hasToolResult = true;
|
||||
expect(block.contains("tool_use_id"), "tool_result has tool_use_id");
|
||||
expect(block.contains("content"), "tool_result has content");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(hasToolResult, "has tool_result blocks");
|
||||
|
||||
// Metadata
|
||||
expect(exported["metadata"]["scenario"] == "add_annotations", "metadata scenario");
|
||||
expect(exported["metadata"]["language"] == "python", "metadata language");
|
||||
}
|
||||
|
||||
// --- Test 5: OpenAI Chat export ---
|
||||
static void test_openai_export() {
|
||||
printf("Test 5: OpenAI Chat export...\n");
|
||||
TraceGenerator gen;
|
||||
gen.setSeed(400);
|
||||
|
||||
auto trace = gen.generate(ScenarioType::Refactor);
|
||||
json exported = TraceExporter::toOpenAIChat(trace);
|
||||
|
||||
expect(exported.contains("id"), "exported has id");
|
||||
expect(exported.contains("messages"), "exported has messages");
|
||||
expect(exported["messages"].is_array(), "messages is array");
|
||||
|
||||
// Check valid OpenAI roles
|
||||
std::set<std::string> validRoles = {"user", "assistant", "tool"};
|
||||
bool hasToolCalls = false;
|
||||
bool hasToolRole = false;
|
||||
for (const auto& msg : exported["messages"]) {
|
||||
expect(msg.contains("role"), "message has role");
|
||||
std::string role = msg["role"];
|
||||
expect(validRoles.count(role) > 0, ("valid OpenAI role: " + role).c_str());
|
||||
|
||||
if (role == "assistant" && msg.contains("tool_calls")) {
|
||||
hasToolCalls = true;
|
||||
for (const auto& tc : msg["tool_calls"]) {
|
||||
expect(tc.contains("id"), "tool_call has id");
|
||||
expect(tc.contains("type"), "tool_call has type");
|
||||
expect(tc["type"] == "function", "tool_call type is function");
|
||||
expect(tc.contains("function"), "tool_call has function");
|
||||
expect(tc["function"].contains("name"), "function has name");
|
||||
expect(tc["function"].contains("arguments"), "function has arguments");
|
||||
}
|
||||
}
|
||||
if (role == "tool") {
|
||||
hasToolRole = true;
|
||||
expect(msg.contains("tool_call_id"), "tool msg has tool_call_id");
|
||||
expect(msg.contains("content"), "tool msg has content");
|
||||
}
|
||||
}
|
||||
expect(hasToolCalls, "has tool_calls in assistant messages");
|
||||
expect(hasToolRole, "has tool role messages");
|
||||
}
|
||||
|
||||
// --- Test 6: JSONL export ---
|
||||
static void test_jsonl_export() {
|
||||
printf("Test 6: JSONL export...\n");
|
||||
TraceGenerator gen;
|
||||
gen.setSeed(500);
|
||||
|
||||
auto traces = gen.generateBatch(6, 500);
|
||||
std::string jsonl = TraceExporter::toJSONL(traces);
|
||||
|
||||
// Count lines
|
||||
int lineCount = 0;
|
||||
std::istringstream iss(jsonl);
|
||||
std::string line;
|
||||
while (std::getline(iss, line)) {
|
||||
if (!line.empty()) {
|
||||
++lineCount;
|
||||
// Each line should be valid JSON
|
||||
try {
|
||||
json parsed = json::parse(line);
|
||||
expect(parsed.contains("id"), "JSONL line has id");
|
||||
expect(parsed.contains("scenario"), "JSONL line has scenario");
|
||||
} catch (...) {
|
||||
expect(false, "JSONL line is valid JSON");
|
||||
}
|
||||
}
|
||||
}
|
||||
expect(lineCount == 6, "JSONL has correct number of lines");
|
||||
|
||||
// Single trace export
|
||||
std::string single = TraceExporter::toJSONLSingle(traces[0]);
|
||||
expect(!single.empty(), "single JSONL not empty");
|
||||
expect(single.back() == '\n', "single JSONL ends with newline");
|
||||
}
|
||||
|
||||
// --- Test 7: Markdown export ---
|
||||
static void test_markdown_export() {
|
||||
printf("Test 7: Markdown export...\n");
|
||||
TraceGenerator gen;
|
||||
gen.setSeed(600);
|
||||
|
||||
auto trace = gen.generate(ScenarioType::CrossLanguage);
|
||||
std::string md = TraceExporter::toMarkdown(trace);
|
||||
|
||||
expect(!md.empty(), "markdown not empty");
|
||||
expect(md.find("# Trace:") != std::string::npos, "has trace header");
|
||||
expect(md.find("**Scenario:**") != std::string::npos, "has scenario field");
|
||||
expect(md.find("**Difficulty:**") != std::string::npos, "has difficulty field");
|
||||
expect(md.find("**Language:**") != std::string::npos, "has language field");
|
||||
expect(md.find("**Tool calls:**") != std::string::npos, "has tool calls field");
|
||||
expect(md.find("### Step") != std::string::npos, "has step headers");
|
||||
expect(md.find("```json") != std::string::npos, "has JSON code blocks");
|
||||
expect(md.find("cross_language") != std::string::npos, "scenario name in markdown");
|
||||
}
|
||||
|
||||
// --- Test 8: Batch generation ---
|
||||
static void test_batch_generation() {
|
||||
printf("Test 8: Batch generation...\n");
|
||||
TraceGenerator gen;
|
||||
|
||||
auto traces = gen.generateBatch(12, 42);
|
||||
expect((int)traces.size() == 12, "batch produces correct count");
|
||||
|
||||
// Check scenario distribution (6 types, 12 traces = 2 of each)
|
||||
std::map<std::string, int> scenarioCounts;
|
||||
for (const auto& t : traces) {
|
||||
scenarioCounts[t.scenario]++;
|
||||
}
|
||||
expect(scenarioCounts.size() == 6, "all 6 scenario types present");
|
||||
for (const auto& [scenario, count] : scenarioCounts) {
|
||||
expect(count == 2, ("2 traces for " + scenario).c_str());
|
||||
}
|
||||
|
||||
// Check language alternation
|
||||
for (int i = 0; i < 12; ++i) {
|
||||
std::string expected = (i % 2 == 0) ? "python" : "cpp";
|
||||
expect(traces[i].language == expected,
|
||||
("language alternation at index " + std::to_string(i)).c_str());
|
||||
}
|
||||
|
||||
// All traces should have unique IDs
|
||||
std::set<std::string> ids;
|
||||
for (const auto& t : traces) ids.insert(t.id);
|
||||
expect((int)ids.size() == 12, "all traces have unique IDs");
|
||||
}
|
||||
|
||||
// --- Test 9: Filtering ---
|
||||
static void test_filtering() {
|
||||
printf("Test 9: Filtering...\n");
|
||||
TraceGenerator gen;
|
||||
auto traces = gen.generateBatch(12, 42);
|
||||
|
||||
// Filter by scenario
|
||||
TraceFilter f;
|
||||
f.scenario = "read_and_understand";
|
||||
auto filtered = TraceExporter::filter(traces, f);
|
||||
for (const auto& t : filtered) {
|
||||
expect(t.scenario == "read_and_understand", "filtered by scenario");
|
||||
}
|
||||
expect(!filtered.empty(), "scenario filter has results");
|
||||
|
||||
// Filter by language
|
||||
TraceFilter f2;
|
||||
f2.language = "python";
|
||||
auto filtered2 = TraceExporter::filter(traces, f2);
|
||||
for (const auto& t : filtered2) {
|
||||
expect(t.language == "python", "filtered by language");
|
||||
}
|
||||
expect((int)filtered2.size() == 6, "6 python traces");
|
||||
|
||||
// Filter by difficulty
|
||||
TraceFilter f3;
|
||||
f3.difficulty = "advanced";
|
||||
auto filtered3 = TraceExporter::filter(traces, f3);
|
||||
for (const auto& t : filtered3) {
|
||||
expect(t.difficulty == "advanced", "filtered by difficulty");
|
||||
}
|
||||
|
||||
// Filter by min tool calls
|
||||
TraceFilter f4;
|
||||
f4.minToolCalls = 2;
|
||||
auto filtered4 = TraceExporter::filter(traces, f4);
|
||||
for (const auto& t : filtered4) {
|
||||
expect(t.toolCallCount >= 2, "filtered by min tool calls");
|
||||
}
|
||||
}
|
||||
|
||||
// --- Test 10: Statistics ---
|
||||
static void test_statistics() {
|
||||
printf("Test 10: Statistics...\n");
|
||||
TraceGenerator gen;
|
||||
auto traces = gen.generateBatch(12, 42);
|
||||
|
||||
auto stats = TraceExporter::computeStats(traces);
|
||||
expect(stats.totalTraces == 12, "total traces count");
|
||||
expect(stats.successCount + stats.failureCount == 12, "success + failure = total");
|
||||
expect(stats.totalToolCalls > 0, "total tool calls > 0");
|
||||
expect(stats.avgToolCalls > 0.0, "avg tool calls > 0");
|
||||
expect(stats.avgSteps > 0.0, "avg steps > 0");
|
||||
expect(stats.scenarioDistribution.size() == 6, "6 scenarios in distribution");
|
||||
expect(stats.languageDistribution.size() == 2, "2 languages in distribution");
|
||||
expect(!stats.toolUsageCount.empty(), "tool usage tracked");
|
||||
expect(!stats.difficultyDistribution.empty(), "difficulty distribution tracked");
|
||||
|
||||
// Stats to JSON
|
||||
json statsJson = stats.toJson();
|
||||
expect(statsJson.contains("totalTraces"), "stats json has totalTraces");
|
||||
expect(statsJson["totalTraces"] == 12, "stats json totalTraces value");
|
||||
}
|
||||
|
||||
// --- Test 11: Code corpus ---
|
||||
static void test_corpus() {
|
||||
printf("Test 11: Code corpus...\n");
|
||||
TraceGenerator gen;
|
||||
const auto& corpus = gen.corpus();
|
||||
|
||||
expect(!corpus.empty(), "corpus not empty");
|
||||
expect(corpus.size() >= 9, "at least 9 samples in corpus");
|
||||
|
||||
// Check corpus has multiple languages
|
||||
std::set<std::string> languages;
|
||||
for (const auto& s : corpus) {
|
||||
languages.insert(s.language);
|
||||
expect(!s.source.empty(), "sample has source");
|
||||
expect(!s.description.empty(), "sample has description");
|
||||
expect(s.complexity >= 1 && s.complexity <= 3, "valid complexity");
|
||||
}
|
||||
expect(languages.size() >= 4, "at least 4 languages in corpus");
|
||||
expect(languages.count("python") > 0, "has python samples");
|
||||
expect(languages.count("cpp") > 0, "has cpp samples");
|
||||
}
|
||||
|
||||
// --- Test 12: Trace toJson roundtrip ---
|
||||
static void test_toJson_roundtrip() {
|
||||
printf("Test 12: Trace toJson roundtrip...\n");
|
||||
TraceGenerator gen;
|
||||
gen.setSeed(700);
|
||||
|
||||
auto trace = gen.generate(ScenarioType::AddAnnotations);
|
||||
json j = trace.toJson();
|
||||
|
||||
// Verify all fields round-trip
|
||||
expect(j["id"] == trace.id, "id roundtrip");
|
||||
expect(j["scenario"] == trace.scenario, "scenario roundtrip");
|
||||
expect(j["difficulty"] == trace.difficulty, "difficulty roundtrip");
|
||||
expect(j["language"] == trace.language, "language roundtrip");
|
||||
expect(j["toolCallCount"] == trace.toolCallCount, "toolCallCount roundtrip");
|
||||
expect(j["success"] == trace.success, "success roundtrip");
|
||||
expect(j["steps"].size() == trace.steps.size(), "steps count roundtrip");
|
||||
expect(j["toolsUsed"].size() == trace.toolsUsed.size(), "toolsUsed count roundtrip");
|
||||
|
||||
// Verify JSON is parseable after dump
|
||||
std::string dumped = j.dump();
|
||||
json reparsed = json::parse(dumped);
|
||||
expect(reparsed == j, "JSON dump/parse roundtrip");
|
||||
}
|
||||
|
||||
// --- Test 13: Batch export formats ---
|
||||
static void test_batch_exports() {
|
||||
printf("Test 13: Batch export formats...\n");
|
||||
TraceGenerator gen;
|
||||
auto traces = gen.generateBatch(6, 42);
|
||||
|
||||
json anthropicBatch = TraceExporter::toAnthropicBatch(traces);
|
||||
expect(anthropicBatch.is_array(), "Anthropic batch is array");
|
||||
expect(anthropicBatch.size() == 6, "Anthropic batch has 6 items");
|
||||
|
||||
json openaiBatch = TraceExporter::toOpenAIBatch(traces);
|
||||
expect(openaiBatch.is_array(), "OpenAI batch is array");
|
||||
expect(openaiBatch.size() == 6, "OpenAI batch has 6 items");
|
||||
}
|
||||
|
||||
int main() {
|
||||
printf("=== Step 219: Trace Generation & Export Tests ===\n\n");
|
||||
|
||||
test_trace_schema();
|
||||
test_all_scenarios();
|
||||
test_seed_determinism();
|
||||
test_anthropic_export();
|
||||
test_openai_export();
|
||||
test_jsonl_export();
|
||||
test_markdown_export();
|
||||
test_batch_generation();
|
||||
test_filtering();
|
||||
test_statistics();
|
||||
test_corpus();
|
||||
test_toJson_roundtrip();
|
||||
test_batch_exports();
|
||||
|
||||
printf("\n=== Results: %d passed, %d failed ===\n", passed, failed);
|
||||
return failed ? 1 : 0;
|
||||
}
|
||||
Reference in New Issue
Block a user