Step 233: Add training data pipeline

editor/src/SessionPipeline.h

@@ -0,0 +1,153 @@
+#pragma once
+// Step 233: Training data pipeline helpers
+
+#include <string>
+#include <vector>
+#include <unordered_set>
+#include <filesystem>
+#include <fstream>
+#include <sstream>
+#include <nlohmann/json.hpp>
+#include "SessionAnonymizer.h"
+#include "SessionToTrace.h"
+#include "TraceExporter.h"
+
+using json = nlohmann::json;
+
+struct PipelineStats {
+    int sessionsProcessed = 0;
+    int tracesGenerated = 0;
+    int tracesFiltered = 0;
+    int tracesDeduped = 0;
+    int tracesValidated = 0;
+};
+
+struct PipelineOptions {
+    SessionAnonymizer::Level anonymizeLevel = SessionAnonymizer::Level::Medium;
+    bool anonymizeEnabled = true;
+    bool deduplicate = true;
+    std::string format = "jsonl"; // jsonl | anthropic | openai | markdown
+};
+
+static std::vector<json> loadSessionsFromDir(const std::string& dir) {
+    std::vector<json> sessions;
+    std::error_code ec;
+    if (!std::filesystem::exists(dir, ec)) return sessions;
+    for (const auto& entry : std::filesystem::directory_iterator(dir, ec)) {
+        if (ec) break;
+        if (!entry.is_regular_file()) continue;
+        auto path = entry.path();
+        if (path.extension() != ".json") continue;
+        std::ifstream in(path.string());
+        if (!in.is_open()) continue;
+        try {
+            json j;
+            in >> j;
+            sessions.push_back(j);
+        } catch (...) {
+            continue;
+        }
+    }
+    return sessions;
+}
+
+static std::vector<json> anonymizeSessions(const std::vector<json>& sessions,
+                                           const PipelineOptions& opts,
+                                           SessionAnonymizer& anonymizer) {
+    if (!opts.anonymizeEnabled) return sessions;
+    anonymizer.setLevel(opts.anonymizeLevel);
+    std::vector<json> out;
+    out.reserve(sessions.size());
+    for (const auto& s : sessions) {
+        out.push_back(anonymizer.anonymize(s));
+    }
+    return out;
+}
+
+static std::vector<Trace> sessionsToTraces(const std::vector<json>& sessions) {
+    std::vector<Trace> traces;
+    SessionTraceConverter converter;
+    for (const auto& s : sessions) {
+        auto out = converter.convert(s);
+        traces.insert(traces.end(), out.begin(), out.end());
+    }
+    return traces;
+}
+
+static bool traceHasMeaningfulWork(const Trace& trace) {
+    if (trace.toolCallCount == 0) return false;
+    if (trace.steps.size() < 2) return false;
+    return true;
+}
+
+static std::vector<Trace> filterTraces(const std::vector<Trace>& traces, PipelineStats& stats) {
+    std::vector<Trace> filtered;
+    for (const auto& t : traces) {
+        if (!traceHasMeaningfulWork(t)) {
+            stats.tracesFiltered++;
+            continue;
+        }
+        if (!t.success && t.toolCallCount < 2) {
+            stats.tracesFiltered++;
+            continue;
+        }
+        filtered.push_back(t);
+    }
+    return filtered;
+}
+
+static std::string traceSignature(const Trace& trace) {
+    std::ostringstream oss;
+    for (const auto& step : trace.steps) {
+        oss << step.role << ":";
+        if (step.role == "tool_call" || step.role == "tool_result") {
+            oss << step.toolName;
+        } else {
+            oss << step.content.substr(0, 48);
+        }
+        oss << "|";
+    }
+    return oss.str();
+}
+
+static std::vector<Trace> dedupeTraces(const std::vector<Trace>& traces, PipelineStats& stats) {
+    std::vector<Trace> out;
+    std::unordered_set<std::string> seen;
+    for (const auto& t : traces) {
+        std::string sig = traceSignature(t);
+        if (seen.count(sig)) {
+            stats.tracesDeduped++;
+            continue;
+        }
+        seen.insert(std::move(sig));
+        out.push_back(t);
+    }
+    return out;
+}
+
+static bool validateTrace(const Trace& trace) {
+    if (trace.steps.empty()) return false;
+    for (const auto& step : trace.steps) {
+        if (step.role == "tool_call" || step.role == "tool_result") {
+            if (step.toolName.empty()) return false;
+        }
+    }
+    return true;
+}
+
+static std::string exportTraces(const std::vector<Trace>& traces, const std::string& format) {
+    if (format == "anthropic") {
+        return TraceExporter::toAnthropicBatch(traces).dump(2);
+    }
+    if (format == "openai") {
+        return TraceExporter::toOpenAIBatch(traces).dump(2);
+    }
+    if (format == "markdown") {
+        std::ostringstream md;
+        for (const auto& t : traces) {
+            md << TraceExporter::toMarkdown(t) << "\n";
+        }
+        return md.str();
+    }
+    return TraceExporter::toJSONL(traces);
+}

editor/src/SessionToTrace.h

@@ -67,6 +67,9 @@ public:
 
                 current.toolsUsed.push_back(tool);
                 current.toolCallCount++;
+                if (item.response.contains("error")) {
+                    current.success = false;
+                }
             }
         }
 

editor/src/pipeline_main.cpp

@@ -0,0 +1,126 @@
+#include <iostream>
+#include <string>
+#include <filesystem>
+#include "SessionPipeline.h"
+
+static std::string getArg(int argc, char** argv, const std::string& key) {
+    for (int i = 1; i < argc - 1; ++i) {
+        if (argv[i] == key) return argv[i + 1];
+    }
+    return "";
+}
+
+static bool hasFlag(int argc, char** argv, const std::string& key) {
+    for (int i = 1; i < argc; ++i) {
+        if (argv[i] == key) return true;
+    }
+    return false;
+}
+
+static SessionAnonymizer::Level parseLevel(const std::string& value, bool& enabled) {
+    std::string lower;
+    for (char c : value) lower.push_back((char)std::tolower((unsigned char)c));
+    if (lower == "off" || lower == "none") {
+        enabled = false;
+        return SessionAnonymizer::Level::Medium;
+    }
+    enabled = true;
+    if (lower == "light") return SessionAnonymizer::Level::Light;
+    if (lower == "full") return SessionAnonymizer::Level::Full;
+    return SessionAnonymizer::Level::Medium;
+}
+
+static std::string defaultOutputFile(const std::string& format) {
+    if (format == "markdown") return "traces.md";
+    if (format == "anthropic" || format == "openai") return "traces.json";
+    return "traces.jsonl";
+}
+
+int main(int argc, char** argv) {
+    std::string inputDir = getArg(argc, argv, "--input");
+    std::string outputPath = getArg(argc, argv, "--output");
+    std::string format = getArg(argc, argv, "--format");
+    std::string anonymize = getArg(argc, argv, "--anonymize");
+    std::string statsPath = getArg(argc, argv, "--stats");
+    std::string seedStr = getArg(argc, argv, "--seed");
+
+    if (inputDir.empty()) {
+        std::cerr << "Usage: whetstone_pipeline --input <dir> [--output path] [--format jsonl|anthropic|openai|markdown] "
+                     "[--anonymize off|light|medium|full] [--dedup] [--stats file.json] [--seed N]\n";
+        return 1;
+    }
+
+    PipelineOptions opts;
+    if (!format.empty()) opts.format = format;
+    if (!anonymize.empty()) opts.anonymizeLevel = parseLevel(anonymize, opts.anonymizeEnabled);
+    if (hasFlag(argc, argv, "--dedup")) opts.deduplicate = true;
+    if (hasFlag(argc, argv, "--no-dedup")) opts.deduplicate = false;
+
+    SessionAnonymizer anonymizer;
+    if (!seedStr.empty()) {
+        try {
+            anonymizer.setSeed(std::stoull(seedStr));
+        } catch (...) {
+            std::cerr << "Invalid seed value.\n";
+        }
+    }
+
+    PipelineStats stats;
+    auto sessions = loadSessionsFromDir(inputDir);
+    stats.sessionsProcessed = static_cast<int>(sessions.size());
+
+    auto anonSessions = anonymizeSessions(sessions, opts, anonymizer);
+    auto traces = sessionsToTraces(anonSessions);
+    stats.tracesGenerated = static_cast<int>(traces.size());
+
+    auto filtered = filterTraces(traces, stats);
+    if (opts.deduplicate) {
+        filtered = dedupeTraces(filtered, stats);
+    }
+
+    std::vector<Trace> validated;
+    for (const auto& t : filtered) {
+        if (validateTrace(t)) {
+            validated.push_back(t);
+            stats.tracesValidated++;
+        }
+    }
+
+    std::string output = exportTraces(validated, opts.format);
+
+    if (!outputPath.empty()) {
+        std::filesystem::path outPath(outputPath);
+        std::error_code ec;
+        if (std::filesystem::is_directory(outPath, ec)) {
+            outPath /= defaultOutputFile(opts.format);
+        }
+        std::filesystem::create_directories(outPath.parent_path(), ec);
+        std::ofstream out(outPath.string(), std::ios::binary);
+        if (!out.is_open()) {
+            std::cerr << "Failed to write output: " << outPath.string() << "\n";
+            return 2;
+        }
+        out << output;
+    } else {
+        std::cout << output << "\n";
+    }
+
+    if (!statsPath.empty()) {
+        TraceStats traceStats = TraceExporter::computeStats(validated);
+        json report;
+        report["sessionsProcessed"] = stats.sessionsProcessed;
+        report["tracesGenerated"] = stats.tracesGenerated;
+        report["tracesFiltered"] = stats.tracesFiltered;
+        report["tracesDeduped"] = stats.tracesDeduped;
+        report["tracesValidated"] = stats.tracesValidated;
+        report["traceStats"] = traceStats.toJson();
+        std::ofstream out(statsPath, std::ios::binary);
+        if (!out.is_open()) {
+            std::cerr << "Failed to write stats: " << statsPath << "\n";
+            return 3;
+        }
+        out << report.dump(2);
+    }
+
+    return 0;
+}