#pragma once // Step 218: Trace Export Pipeline // // Exports interaction traces in formats suitable for LLM fine-tuning: // - Anthropic Messages format (tool_use / tool_result content blocks) // - OpenAI Chat format (tool_calls array + tool role messages) // - JSONL (one trace per line, streaming-friendly) // - Markdown (human-readable for review) // // Also provides filtering and statistics. #include #include #include #include #include "TraceGenerator.h" using json = nlohmann::json; // ----------------------------------------------------------------------- // Trace statistics // ----------------------------------------------------------------------- struct TraceStats { int totalTraces = 0; int successCount = 0; int failureCount = 0; int totalToolCalls = 0; double avgToolCalls = 0.0; double avgSteps = 0.0; std::map scenarioDistribution; std::map languageDistribution; std::map toolUsageCount; std::map difficultyDistribution; json toJson() const { json j; j["totalTraces"] = totalTraces; j["successCount"] = successCount; j["failureCount"] = failureCount; j["totalToolCalls"] = totalToolCalls; j["avgToolCalls"] = avgToolCalls; j["avgSteps"] = avgSteps; j["scenarioDistribution"] = scenarioDistribution; j["languageDistribution"] = languageDistribution; j["toolUsageCount"] = toolUsageCount; j["difficultyDistribution"] = difficultyDistribution; return j; } }; // ----------------------------------------------------------------------- // Filter criteria // ----------------------------------------------------------------------- struct TraceFilter { std::string scenario; // empty = all std::string difficulty; // empty = all std::string language; // empty = all bool successOnly = false; bool failureOnly = false; int minToolCalls = 0; int maxToolCalls = INT_MAX; }; // ----------------------------------------------------------------------- // TraceExporter // ----------------------------------------------------------------------- class TraceExporter { public: // --- Export to Anthropic Messages format --- static json toAnthropicMessages(const Trace& trace) { json messages = json::array(); for (const auto& step : trace.steps) { if (step.role == "user") { messages.push_back({ {"role", "user"}, {"content", step.content} }); } else if (step.role == "assistant") { if (!step.content.empty()) { messages.push_back({ {"role", "assistant"}, {"content", step.content} }); } } else if (step.role == "tool_call") { // Anthropic uses tool_use content blocks inside assistant messages json toolUseBlock = { {"type", "tool_use"}, {"id", "call_" + step.toolName + "_" + trace.id}, {"name", step.toolName}, {"input", step.toolInput.is_null() ? json::object() : step.toolInput} }; messages.push_back({ {"role", "assistant"}, {"content", json::array({toolUseBlock})} }); } else if (step.role == "tool_result") { json resultBlock = { {"type", "tool_result"}, {"tool_use_id", "call_" + step.toolName + "_" + trace.id}, {"content", step.toolOutput.dump()} }; messages.push_back({ {"role", "user"}, {"content", json::array({resultBlock})} }); } } json result; result["id"] = trace.id; result["model"] = "claude-opus-4-6"; result["messages"] = messages; result["metadata"] = { {"scenario", trace.scenario}, {"difficulty", trace.difficulty}, {"language", trace.language}, {"toolsUsed", trace.toolsUsed}, {"toolCallCount", trace.toolCallCount}, {"success", trace.success} }; return result; } // --- Export to OpenAI Chat format --- static json toOpenAIChat(const Trace& trace) { json messages = json::array(); int callIdx = 0; for (size_t i = 0; i < trace.steps.size(); ++i) { const auto& step = trace.steps[i]; if (step.role == "user") { messages.push_back({ {"role", "user"}, {"content", step.content} }); } else if (step.role == "assistant") { messages.push_back({ {"role", "assistant"}, {"content", step.content} }); } else if (step.role == "tool_call") { std::string callId = "call_" + std::to_string(callIdx++); json toolCall = { {"id", callId}, {"type", "function"}, {"function", { {"name", step.toolName}, {"arguments", step.toolInput.is_null() ? "{}" : step.toolInput.dump()} }} }; messages.push_back({ {"role", "assistant"}, {"content", nullptr}, {"tool_calls", json::array({toolCall})} }); // Look for matching tool_result if (i + 1 < trace.steps.size() && trace.steps[i + 1].role == "tool_result") { messages.push_back({ {"role", "tool"}, {"tool_call_id", callId}, {"content", trace.steps[i + 1].toolOutput.dump()} }); ++i; // skip the tool_result since we consumed it } } } json result; result["id"] = trace.id; result["model"] = "gpt-4"; result["messages"] = messages; result["metadata"] = { {"scenario", trace.scenario}, {"difficulty", trace.difficulty}, {"language", trace.language} }; return result; } // --- Export to JSONL string (one trace per line) --- static std::string toJSONL(const std::vector& traces) { std::ostringstream out; for (const auto& trace : traces) { out << trace.toJson().dump() << "\n"; } return out.str(); } static std::string toJSONLSingle(const Trace& trace) { return trace.toJson().dump() + "\n"; } // --- Export to Markdown (human-readable) --- static std::string toMarkdown(const Trace& trace) { std::ostringstream md; md << "# Trace: " << trace.id << "\n\n"; md << "- **Scenario:** " << trace.scenario << "\n"; md << "- **Difficulty:** " << trace.difficulty << "\n"; md << "- **Language:** " << trace.language << "\n"; md << "- **Tool calls:** " << trace.toolCallCount << "\n"; md << "- **Success:** " << (trace.success ? "Yes" : "No") << "\n"; md << "- **Tools used:** "; for (size_t i = 0; i < trace.toolsUsed.size(); ++i) { if (i > 0) md << ", "; md << "`" << trace.toolsUsed[i] << "`"; } md << "\n\n---\n\n"; int stepNum = 1; for (const auto& step : trace.steps) { md << "### Step " << stepNum++ << ": "; if (step.role == "user") { md << "User\n\n" << step.content << "\n\n"; } else if (step.role == "assistant") { md << "Assistant\n\n" << step.content << "\n\n"; } else if (step.role == "tool_call") { md << "Tool Call: `" << step.toolName << "`\n\n"; md << "```json\n"; md << (step.toolInput.is_null() ? "{}" : step.toolInput.dump(2)) << "\n"; md << "```\n\n"; } else if (step.role == "tool_result") { md << "Tool Result: `" << step.toolName << "`\n\n"; md << "```json\n"; md << (step.toolOutput.is_null() ? "{}" : step.toolOutput.dump(2)) << "\n"; md << "```\n\n"; } } return md.str(); } // --- Batch export --- static json toAnthropicBatch(const std::vector& traces) { json arr = json::array(); for (const auto& t : traces) arr.push_back(toAnthropicMessages(t)); return arr; } static json toOpenAIBatch(const std::vector& traces) { json arr = json::array(); for (const auto& t : traces) arr.push_back(toOpenAIChat(t)); return arr; } // --- Filtering --- static std::vector filter(const std::vector& traces, const TraceFilter& f) { std::vector result; for (const auto& t : traces) { if (!f.scenario.empty() && t.scenario != f.scenario) continue; if (!f.difficulty.empty() && t.difficulty != f.difficulty) continue; if (!f.language.empty() && t.language != f.language) continue; if (f.successOnly && !t.success) continue; if (f.failureOnly && t.success) continue; if (t.toolCallCount < f.minToolCalls) continue; if (t.toolCallCount > f.maxToolCalls) continue; result.push_back(t); } return result; } // --- Statistics --- static TraceStats computeStats(const std::vector& traces) { TraceStats stats; stats.totalTraces = static_cast(traces.size()); int totalSteps = 0; for (const auto& t : traces) { if (t.success) ++stats.successCount; else ++stats.failureCount; stats.totalToolCalls += t.toolCallCount; totalSteps += static_cast(t.steps.size()); stats.scenarioDistribution[t.scenario]++; stats.languageDistribution[t.language]++; stats.difficultyDistribution[t.difficulty]++; for (const auto& tool : t.toolsUsed) { stats.toolUsageCount[tool]++; } } if (stats.totalTraces > 0) { stats.avgToolCalls = static_cast(stats.totalToolCalls) / stats.totalTraces; stats.avgSteps = static_cast(totalSteps) / stats.totalTraces; } return stats; } };