From 423db8cb22f081b709afc22a6c0f406acd89e0ea Mon Sep 17 00:00:00 2001 From: Bill Date: Tue, 10 Feb 2026 08:47:07 -0700 Subject: [PATCH] Step 233: Add training data pipeline --- PROGRESS.md | 1 + editor/CMakeLists.txt | 7 ++ editor/src/SessionPipeline.h | 153 ++++++++++++++++++++++++++++++++++ editor/src/SessionToTrace.h | 3 + editor/src/pipeline_main.cpp | 126 ++++++++++++++++++++++++++++ editor/tests/step233_test.cpp | 52 ++++++++++++ sprint7_plan.md | 2 +- 7 files changed, 343 insertions(+), 1 deletion(-) create mode 100644 editor/src/SessionPipeline.h create mode 100644 editor/src/pipeline_main.cpp create mode 100644 editor/tests/step233_test.cpp diff --git a/PROGRESS.md b/PROGRESS.md index 8303623..c307979 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -752,3 +752,4 @@ vcpkg's imgui 1.91.9 removed the `sdl2-binding` feature (only `sdl3-binding` exi | 2026-02-10 | Codex | Step 230: Enhanced session recorder (metadata, event capture, ms timestamps, UI toggle, auto-record setting). 1/1 tests pass. | | 2026-02-10 | Codex | Step 231: Session anonymizer (paths, secrets, identifier anonymization, comment/string stripping, levels, deterministic seed). 1/1 tests pass. | | 2026-02-10 | Codex | Step 232: Session-to-trace converter (events->user msgs, RPC->tool calls/results, gap splitting, synthetic thinking). 1/1 tests pass. | +| 2026-02-10 | Codex | Step 233: Training data pipeline CLI (load/anonymize/convert/filter/dedup/export + stats). 1/1 tests pass. | diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 6d7c4b8..830ea00 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -1181,6 +1181,9 @@ target_link_libraries(step231_test PRIVATE nlohmann_json::nlohmann_json) add_executable(step232_test tests/step232_test.cpp) target_include_directories(step232_test PRIVATE src) target_link_libraries(step232_test PRIVATE nlohmann_json::nlohmann_json) +add_executable(step233_test tests/step233_test.cpp) +target_include_directories(step233_test PRIVATE src) +target_link_libraries(step233_test PRIVATE nlohmann_json::nlohmann_json) add_executable(step213_test tests/step213_test.cpp) target_include_directories(step213_test PRIVATE src) target_link_libraries(step213_test PRIVATE nlohmann_json::nlohmann_json) @@ -1295,4 +1298,8 @@ add_executable(whetstone_eval src/eval_main.cpp) target_include_directories(whetstone_eval PRIVATE src) target_link_libraries(whetstone_eval PRIVATE nlohmann_json::nlohmann_json) +add_executable(whetstone_pipeline src/pipeline_main.cpp) +target_include_directories(whetstone_pipeline PRIVATE src) +target_link_libraries(whetstone_pipeline PRIVATE nlohmann_json::nlohmann_json) + # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies) diff --git a/editor/src/SessionPipeline.h b/editor/src/SessionPipeline.h new file mode 100644 index 0000000..4baddf0 --- /dev/null +++ b/editor/src/SessionPipeline.h @@ -0,0 +1,153 @@ +#pragma once +// Step 233: Training data pipeline helpers + +#include +#include +#include +#include +#include +#include +#include +#include "SessionAnonymizer.h" +#include "SessionToTrace.h" +#include "TraceExporter.h" + +using json = nlohmann::json; + +struct PipelineStats { + int sessionsProcessed = 0; + int tracesGenerated = 0; + int tracesFiltered = 0; + int tracesDeduped = 0; + int tracesValidated = 0; +}; + +struct PipelineOptions { + SessionAnonymizer::Level anonymizeLevel = SessionAnonymizer::Level::Medium; + bool anonymizeEnabled = true; + bool deduplicate = true; + std::string format = "jsonl"; // jsonl | anthropic | openai | markdown +}; + +static std::vector loadSessionsFromDir(const std::string& dir) { + std::vector sessions; + std::error_code ec; + if (!std::filesystem::exists(dir, ec)) return sessions; + for (const auto& entry : std::filesystem::directory_iterator(dir, ec)) { + if (ec) break; + if (!entry.is_regular_file()) continue; + auto path = entry.path(); + if (path.extension() != ".json") continue; + std::ifstream in(path.string()); + if (!in.is_open()) continue; + try { + json j; + in >> j; + sessions.push_back(j); + } catch (...) { + continue; + } + } + return sessions; +} + +static std::vector anonymizeSessions(const std::vector& sessions, + const PipelineOptions& opts, + SessionAnonymizer& anonymizer) { + if (!opts.anonymizeEnabled) return sessions; + anonymizer.setLevel(opts.anonymizeLevel); + std::vector out; + out.reserve(sessions.size()); + for (const auto& s : sessions) { + out.push_back(anonymizer.anonymize(s)); + } + return out; +} + +static std::vector sessionsToTraces(const std::vector& sessions) { + std::vector traces; + SessionTraceConverter converter; + for (const auto& s : sessions) { + auto out = converter.convert(s); + traces.insert(traces.end(), out.begin(), out.end()); + } + return traces; +} + +static bool traceHasMeaningfulWork(const Trace& trace) { + if (trace.toolCallCount == 0) return false; + if (trace.steps.size() < 2) return false; + return true; +} + +static std::vector filterTraces(const std::vector& traces, PipelineStats& stats) { + std::vector filtered; + for (const auto& t : traces) { + if (!traceHasMeaningfulWork(t)) { + stats.tracesFiltered++; + continue; + } + if (!t.success && t.toolCallCount < 2) { + stats.tracesFiltered++; + continue; + } + filtered.push_back(t); + } + return filtered; +} + +static std::string traceSignature(const Trace& trace) { + std::ostringstream oss; + for (const auto& step : trace.steps) { + oss << step.role << ":"; + if (step.role == "tool_call" || step.role == "tool_result") { + oss << step.toolName; + } else { + oss << step.content.substr(0, 48); + } + oss << "|"; + } + return oss.str(); +} + +static std::vector dedupeTraces(const std::vector& traces, PipelineStats& stats) { + std::vector out; + std::unordered_set seen; + for (const auto& t : traces) { + std::string sig = traceSignature(t); + if (seen.count(sig)) { + stats.tracesDeduped++; + continue; + } + seen.insert(std::move(sig)); + out.push_back(t); + } + return out; +} + +static bool validateTrace(const Trace& trace) { + if (trace.steps.empty()) return false; + for (const auto& step : trace.steps) { + if (step.role == "tool_call" || step.role == "tool_result") { + if (step.toolName.empty()) return false; + } + } + return true; +} + +static std::string exportTraces(const std::vector& traces, const std::string& format) { + if (format == "anthropic") { + return TraceExporter::toAnthropicBatch(traces).dump(2); + } + if (format == "openai") { + return TraceExporter::toOpenAIBatch(traces).dump(2); + } + if (format == "markdown") { + std::ostringstream md; + for (const auto& t : traces) { + md << TraceExporter::toMarkdown(t) << "\n"; + } + return md.str(); + } + return TraceExporter::toJSONL(traces); +} diff --git a/editor/src/SessionToTrace.h b/editor/src/SessionToTrace.h index dcfdd54..23db8e3 100644 --- a/editor/src/SessionToTrace.h +++ b/editor/src/SessionToTrace.h @@ -67,6 +67,9 @@ public: current.toolsUsed.push_back(tool); current.toolCallCount++; + if (item.response.contains("error")) { + current.success = false; + } } } diff --git a/editor/src/pipeline_main.cpp b/editor/src/pipeline_main.cpp new file mode 100644 index 0000000..6b6d7ad --- /dev/null +++ b/editor/src/pipeline_main.cpp @@ -0,0 +1,126 @@ +#include +#include +#include +#include "SessionPipeline.h" + +static std::string getArg(int argc, char** argv, const std::string& key) { + for (int i = 1; i < argc - 1; ++i) { + if (argv[i] == key) return argv[i + 1]; + } + return ""; +} + +static bool hasFlag(int argc, char** argv, const std::string& key) { + for (int i = 1; i < argc; ++i) { + if (argv[i] == key) return true; + } + return false; +} + +static SessionAnonymizer::Level parseLevel(const std::string& value, bool& enabled) { + std::string lower; + for (char c : value) lower.push_back((char)std::tolower((unsigned char)c)); + if (lower == "off" || lower == "none") { + enabled = false; + return SessionAnonymizer::Level::Medium; + } + enabled = true; + if (lower == "light") return SessionAnonymizer::Level::Light; + if (lower == "full") return SessionAnonymizer::Level::Full; + return SessionAnonymizer::Level::Medium; +} + +static std::string defaultOutputFile(const std::string& format) { + if (format == "markdown") return "traces.md"; + if (format == "anthropic" || format == "openai") return "traces.json"; + return "traces.jsonl"; +} + +int main(int argc, char** argv) { + std::string inputDir = getArg(argc, argv, "--input"); + std::string outputPath = getArg(argc, argv, "--output"); + std::string format = getArg(argc, argv, "--format"); + std::string anonymize = getArg(argc, argv, "--anonymize"); + std::string statsPath = getArg(argc, argv, "--stats"); + std::string seedStr = getArg(argc, argv, "--seed"); + + if (inputDir.empty()) { + std::cerr << "Usage: whetstone_pipeline --input [--output path] [--format jsonl|anthropic|openai|markdown] " + "[--anonymize off|light|medium|full] [--dedup] [--stats file.json] [--seed N]\n"; + return 1; + } + + PipelineOptions opts; + if (!format.empty()) opts.format = format; + if (!anonymize.empty()) opts.anonymizeLevel = parseLevel(anonymize, opts.anonymizeEnabled); + if (hasFlag(argc, argv, "--dedup")) opts.deduplicate = true; + if (hasFlag(argc, argv, "--no-dedup")) opts.deduplicate = false; + + SessionAnonymizer anonymizer; + if (!seedStr.empty()) { + try { + anonymizer.setSeed(std::stoull(seedStr)); + } catch (...) { + std::cerr << "Invalid seed value.\n"; + } + } + + PipelineStats stats; + auto sessions = loadSessionsFromDir(inputDir); + stats.sessionsProcessed = static_cast(sessions.size()); + + auto anonSessions = anonymizeSessions(sessions, opts, anonymizer); + auto traces = sessionsToTraces(anonSessions); + stats.tracesGenerated = static_cast(traces.size()); + + auto filtered = filterTraces(traces, stats); + if (opts.deduplicate) { + filtered = dedupeTraces(filtered, stats); + } + + std::vector validated; + for (const auto& t : filtered) { + if (validateTrace(t)) { + validated.push_back(t); + stats.tracesValidated++; + } + } + + std::string output = exportTraces(validated, opts.format); + + if (!outputPath.empty()) { + std::filesystem::path outPath(outputPath); + std::error_code ec; + if (std::filesystem::is_directory(outPath, ec)) { + outPath /= defaultOutputFile(opts.format); + } + std::filesystem::create_directories(outPath.parent_path(), ec); + std::ofstream out(outPath.string(), std::ios::binary); + if (!out.is_open()) { + std::cerr << "Failed to write output: " << outPath.string() << "\n"; + return 2; + } + out << output; + } else { + std::cout << output << "\n"; + } + + if (!statsPath.empty()) { + TraceStats traceStats = TraceExporter::computeStats(validated); + json report; + report["sessionsProcessed"] = stats.sessionsProcessed; + report["tracesGenerated"] = stats.tracesGenerated; + report["tracesFiltered"] = stats.tracesFiltered; + report["tracesDeduped"] = stats.tracesDeduped; + report["tracesValidated"] = stats.tracesValidated; + report["traceStats"] = traceStats.toJson(); + std::ofstream out(statsPath, std::ios::binary); + if (!out.is_open()) { + std::cerr << "Failed to write stats: " << statsPath << "\n"; + return 3; + } + out << report.dump(2); + } + + return 0; +} diff --git a/editor/tests/step233_test.cpp b/editor/tests/step233_test.cpp new file mode 100644 index 0000000..a1373e3 --- /dev/null +++ b/editor/tests/step233_test.cpp @@ -0,0 +1,52 @@ +// Step 233: Training data pipeline CLI helpers. +#include "SessionPipeline.h" +#include + +static void expect(bool cond, const std::string& name, int& passed, int& failed) { + if (cond) { + std::cout << "Test " << (passed + failed + 1) << " PASS: " << name << "\n"; + ++passed; + } else { + std::cout << "Test " << (passed + failed + 1) << " FAIL: " << name << "\n"; + ++failed; + } +} + +int main() { + int passed = 0; + int failed = 0; + + json session = { + {"sessionId", "s1"}, + {"metadata", {{"language", "python"}}}, + {"events", json::array({{ + {"timestamp", "2026-02-10T10:00:00.000"}, + {"type", "file_opened"}, + {"payload", {{"path", "main.py"}}} + }})}, + {"calls", json::array({{ + {"timestamp", "2026-02-10T10:00:01.000"}, + {"method", "getAST"}, + {"request", {{"params", json::object()}}}, + {"response", {{"result", json::object()}}} + }})} + }; + + PipelineOptions opts; + opts.anonymizeEnabled = false; + PipelineStats stats; + SessionAnonymizer anonymizer; + auto anon = anonymizeSessions({session}, opts, anonymizer); + auto traces = sessionsToTraces(anon); + stats.tracesGenerated = (int)traces.size(); + + auto filtered = filterTraces(traces, stats); + auto output = exportTraces(filtered, "jsonl"); + + expect(stats.tracesGenerated == 1, "generated one trace", passed, failed); + expect(!output.empty(), "export produced output", passed, failed); + + std::cout << "\n=== Step 233 Results: " << passed << " passed, " + << failed << " failed ===\n"; + return failed == 0 ? 0 : 1; +} diff --git a/sprint7_plan.md b/sprint7_plan.md index e7bd3aa..43429b8 100644 --- a/sprint7_plan.md +++ b/sprint7_plan.md @@ -431,7 +431,7 @@ Capture real editing sessions, anonymize them, and export as training data. - Quality filter: discard sessions with too many errors or no meaningful work *New:* `editor/src/SessionToTrace.h` -- [ ] **Step 233: Training data pipeline CLI** +- [x] **Step 233: Training data pipeline CLI** End-to-end pipeline from raw sessions to training data: - `whetstone_pipeline --input sessions/ --anonymize medium --format anthropic --output training/` - Steps: load sessions → anonymize → convert to traces → filter → export