bill/whetstone_DSL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Step 233: Add training data pipeline

Browse Source
This commit is contained in:
Bill
2026-02-10 08:47:07 -07:00
parent b68d24c881
commit 423db8cb22
7 changed files with 343 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
PROGRESS.md
View File

@@ -752,3 +752,4 @@ vcpkg's imgui 1.91.9 removed the `sdl2-binding` feature (only `sdl3-binding` exi
| 2026-02-10 | Codex | Step 230: Enhanced session recorder (metadata, event capture, ms timestamps, UI toggle, auto-record setting). 1/1 tests pass. |
| 2026-02-10 | Codex | Step 231: Session anonymizer (paths, secrets, identifier anonymization, comment/string stripping, levels, deterministic seed). 1/1 tests pass. |
| 2026-02-10 | Codex | Step 232: Session-to-trace converter (events->user msgs, RPC->tool calls/results, gap splitting, synthetic thinking). 1/1 tests pass. |
| 2026-02-10 | Codex | Step 233: Training data pipeline CLI (load/anonymize/convert/filter/dedup/export + stats). 1/1 tests pass. |

7
editor/CMakeLists.txt
View File

@@ -1181,6 +1181,9 @@ target_link_libraries(step231_test PRIVATE nlohmann_json::nlohmann_json)
add_executable(step232_test tests/step232_test.cpp)
target_include_directories(step232_test PRIVATE src)
target_link_libraries(step232_test PRIVATE nlohmann_json::nlohmann_json)
add_executable(step233_test tests/step233_test.cpp)
target_include_directories(step233_test PRIVATE src)
target_link_libraries(step233_test PRIVATE nlohmann_json::nlohmann_json)
add_executable(step213_test tests/step213_test.cpp)
target_include_directories(step213_test PRIVATE src)
target_link_libraries(step213_test PRIVATE nlohmann_json::nlohmann_json)
@@ -1295,4 +1298,8 @@ add_executable(whetstone_eval src/eval_main.cpp)
target_include_directories(whetstone_eval PRIVATE src)
target_link_libraries(whetstone_eval PRIVATE nlohmann_json::nlohmann_json)
add_executable(whetstone_pipeline src/pipeline_main.cpp)
target_include_directories(whetstone_pipeline PRIVATE src)
target_link_libraries(whetstone_pipeline PRIVATE nlohmann_json::nlohmann_json)
# Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies)

153
editor/src/SessionPipeline.h Normal file
View File

@@ -0,0 +1,153 @@
#pragma once
// Step 233: Training data pipeline helpers
#include <string>
#include <vector>
#include <unordered_set>
#include <filesystem>
#include <fstream>
#include <sstream>
#include <nlohmann/json.hpp>
#include "SessionAnonymizer.h"
#include "SessionToTrace.h"
#include "TraceExporter.h"
using json = nlohmann::json;
struct PipelineStats {
int sessionsProcessed = 0;
int tracesGenerated = 0;
int tracesFiltered = 0;
int tracesDeduped = 0;
int tracesValidated = 0;
};
struct PipelineOptions {
SessionAnonymizer::Level anonymizeLevel = SessionAnonymizer::Level::Medium;
bool anonymizeEnabled = true;
bool deduplicate = true;
std::string format = "jsonl"; // jsonl | anthropic | openai | markdown
};
static std::vector<json> loadSessionsFromDir(const std::string& dir) {
std::vector<json> sessions;
std::error_code ec;
if (!std::filesystem::exists(dir, ec)) return sessions;
for (const auto& entry : std::filesystem::directory_iterator(dir, ec)) {
if (ec) break;
if (!entry.is_regular_file()) continue;
auto path = entry.path();
if (path.extension() != ".json") continue;
std::ifstream in(path.string());
if (!in.is_open()) continue;
try {
json j;
in >> j;
sessions.push_back(j);
} catch (...) {
continue;
}
}
return sessions;
}
static std::vector<json> anonymizeSessions(const std::vector<json>& sessions,
const PipelineOptions& opts,
SessionAnonymizer& anonymizer) {
if (!opts.anonymizeEnabled) return sessions;
anonymizer.setLevel(opts.anonymizeLevel);
std::vector<json> out;
out.reserve(sessions.size());
for (const auto& s : sessions) {
out.push_back(anonymizer.anonymize(s));
}
return out;
}
static std::vector<Trace> sessionsToTraces(const std::vector<json>& sessions) {
std::vector<Trace> traces;
SessionTraceConverter converter;
for (const auto& s : sessions) {
auto out = converter.convert(s);
traces.insert(traces.end(), out.begin(), out.end());
}
return traces;
}
static bool traceHasMeaningfulWork(const Trace& trace) {
if (trace.toolCallCount == 0) return false;
if (trace.steps.size() < 2) return false;
return true;
}
static std::vector<Trace> filterTraces(const std::vector<Trace>& traces, PipelineStats& stats) {
std::vector<Trace> filtered;
for (const auto& t : traces) {
if (!traceHasMeaningfulWork(t)) {
stats.tracesFiltered++;
continue;
}
if (!t.success && t.toolCallCount < 2) {
stats.tracesFiltered++;
continue;
}
filtered.push_back(t);
}
return filtered;
}
static std::string traceSignature(const Trace& trace) {
std::ostringstream oss;
for (const auto& step : trace.steps) {
oss << step.role << ":";
if (step.role == "tool_call" || step.role == "tool_result") {
oss << step.toolName;
} else {
oss << step.content.substr(0, 48);
}
oss << "|";
}
return oss.str();
}
static std::vector<Trace> dedupeTraces(const std::vector<Trace>& traces, PipelineStats& stats) {
std::vector<Trace> out;
std::unordered_set<std::string> seen;
for (const auto& t : traces) {
std::string sig = traceSignature(t);
if (seen.count(sig)) {
stats.tracesDeduped++;
continue;
}
seen.insert(std::move(sig));
out.push_back(t);
}
return out;
}
static bool validateTrace(const Trace& trace) {
if (trace.steps.empty()) return false;
for (const auto& step : trace.steps) {
if (step.role == "tool_call" || step.role == "tool_result") {
if (step.toolName.empty()) return false;
}
}
return true;
}
static std::string exportTraces(const std::vector<Trace>& traces, const std::string& format) {
if (format == "anthropic") {
return TraceExporter::toAnthropicBatch(traces).dump(2);
}
if (format == "openai") {
return TraceExporter::toOpenAIBatch(traces).dump(2);
}
if (format == "markdown") {
std::ostringstream md;
for (const auto& t : traces) {
md << TraceExporter::toMarkdown(t) << "\n";
}
return md.str();
}
return TraceExporter::toJSONL(traces);
}

3
editor/src/SessionToTrace.h
View File

@@ -67,6 +67,9 @@ public:
current.toolsUsed.push_back(tool);
current.toolCallCount++;
if (item.response.contains("error")) {
current.success = false;
}
}
}

126
editor/src/pipeline_main.cpp Normal file
View File

@@ -0,0 +1,126 @@
#include <iostream>
#include <string>
#include <filesystem>
#include "SessionPipeline.h"
static std::string getArg(int argc, char** argv, const std::string& key) {
for (int i = 1; i < argc - 1; ++i) {
if (argv[i] == key) return argv[i + 1];
}
return "";
}
static bool hasFlag(int argc, char** argv, const std::string& key) {
for (int i = 1; i < argc; ++i) {
if (argv[i] == key) return true;
}
return false;
}
static SessionAnonymizer::Level parseLevel(const std::string& value, bool& enabled) {
std::string lower;
for (char c : value) lower.push_back((char)std::tolower((unsigned char)c));
if (lower == "off" || lower == "none") {
enabled = false;
return SessionAnonymizer::Level::Medium;
}
enabled = true;
if (lower == "light") return SessionAnonymizer::Level::Light;
if (lower == "full") return SessionAnonymizer::Level::Full;
return SessionAnonymizer::Level::Medium;
}
static std::string defaultOutputFile(const std::string& format) {
if (format == "markdown") return "traces.md";
if (format == "anthropic" || format == "openai") return "traces.json";
return "traces.jsonl";
}
int main(int argc, char** argv) {
std::string inputDir = getArg(argc, argv, "--input");
std::string outputPath = getArg(argc, argv, "--output");
std::string format = getArg(argc, argv, "--format");
std::string anonymize = getArg(argc, argv, "--anonymize");
std::string statsPath = getArg(argc, argv, "--stats");
std::string seedStr = getArg(argc, argv, "--seed");
if (inputDir.empty()) {
std::cerr << "Usage: whetstone_pipeline --input <dir> [--output path] [--format jsonl|anthropic|openai|markdown] "
"[--anonymize off|light|medium|full] [--dedup] [--stats file.json] [--seed N]\n";
return 1;
}
PipelineOptions opts;
if (!format.empty()) opts.format = format;
if (!anonymize.empty()) opts.anonymizeLevel = parseLevel(anonymize, opts.anonymizeEnabled);
if (hasFlag(argc, argv, "--dedup")) opts.deduplicate = true;
if (hasFlag(argc, argv, "--no-dedup")) opts.deduplicate = false;
SessionAnonymizer anonymizer;
if (!seedStr.empty()) {
try {
anonymizer.setSeed(std::stoull(seedStr));
} catch (...) {
std::cerr << "Invalid seed value.\n";
}
}
PipelineStats stats;
auto sessions = loadSessionsFromDir(inputDir);
stats.sessionsProcessed = static_cast<int>(sessions.size());
auto anonSessions = anonymizeSessions(sessions, opts, anonymizer);
auto traces = sessionsToTraces(anonSessions);
stats.tracesGenerated = static_cast<int>(traces.size());
auto filtered = filterTraces(traces, stats);
if (opts.deduplicate) {
filtered = dedupeTraces(filtered, stats);
}
std::vector<Trace> validated;
for (const auto& t : filtered) {
if (validateTrace(t)) {
validated.push_back(t);
stats.tracesValidated++;
}
}
std::string output = exportTraces(validated, opts.format);
if (!outputPath.empty()) {
std::filesystem::path outPath(outputPath);
std::error_code ec;
if (std::filesystem::is_directory(outPath, ec)) {
outPath /= defaultOutputFile(opts.format);
}
std::filesystem::create_directories(outPath.parent_path(), ec);
std::ofstream out(outPath.string(), std::ios::binary);
if (!out.is_open()) {
std::cerr << "Failed to write output: " << outPath.string() << "\n";
return 2;
}
out << output;
} else {
std::cout << output << "\n";
}
if (!statsPath.empty()) {
TraceStats traceStats = TraceExporter::computeStats(validated);
json report;
report["sessionsProcessed"] = stats.sessionsProcessed;
report["tracesGenerated"] = stats.tracesGenerated;
report["tracesFiltered"] = stats.tracesFiltered;
report["tracesDeduped"] = stats.tracesDeduped;
report["tracesValidated"] = stats.tracesValidated;
report["traceStats"] = traceStats.toJson();
std::ofstream out(statsPath, std::ios::binary);
if (!out.is_open()) {
std::cerr << "Failed to write stats: " << statsPath << "\n";
return 3;
}
out << report.dump(2);
}
return 0;
}

52
editor/tests/step233_test.cpp Normal file
View File

@@ -0,0 +1,52 @@
// Step 233: Training data pipeline CLI helpers.
#include "SessionPipeline.h"
#include <iostream>
static void expect(bool cond, const std::string& name, int& passed, int& failed) {
if (cond) {
std::cout << "Test " << (passed + failed + 1) << " PASS: " << name << "\n";
++passed;
} else {
std::cout << "Test " << (passed + failed + 1) << " FAIL: " << name << "\n";
++failed;
}
}
int main() {
int passed = 0;
int failed = 0;
json session = {
{"sessionId", "s1"},
{"metadata", {{"language", "python"}}},
{"events", json::array({{
{"timestamp", "2026-02-10T10:00:00.000"},
{"type", "file_opened"},
{"payload", {{"path", "main.py"}}}
}})},
{"calls", json::array({{
{"timestamp", "2026-02-10T10:00:01.000"},
{"method", "getAST"},
{"request", {{"params", json::object()}}},
{"response", {{"result", json::object()}}}
}})}
};
PipelineOptions opts;
opts.anonymizeEnabled = false;
PipelineStats stats;
SessionAnonymizer anonymizer;
auto anon = anonymizeSessions({session}, opts, anonymizer);
auto traces = sessionsToTraces(anon);
stats.tracesGenerated = (int)traces.size();
auto filtered = filterTraces(traces, stats);
auto output = exportTraces(filtered, "jsonl");
expect(stats.tracesGenerated == 1, "generated one trace", passed, failed);
expect(!output.empty(), "export produced output", passed, failed);
std::cout << "\n=== Step 233 Results: " << passed << " passed, "
<< failed << " failed ===\n";
return failed == 0 ? 0 : 1;
}

2
sprint7_plan.md
View File

@@ -431,7 +431,7 @@ Capture real editing sessions, anonymize them, and export as training data.
- Quality filter: discard sessions with too many errors or no meaningful work
*New:* `editor/src/SessionToTrace.h`
- [ ] **Step 233: Training data pipeline CLI**
- [x] **Step 233: Training data pipeline CLI**
End-to-end pipeline from raw sessions to training data:
- `whetstone_pipeline --input sessions/ --anonymize medium --format anthropic --output training/`
- Steps: load sessions → anonymize → convert to traces → filter → export