bill/whetstone_DSL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Step 381: add workflow progress and ETA tracker

Browse Source
This commit is contained in:
Bill
2026-02-16 12:39:37 -07:00
parent b99957e2fc
commit 5300be080a
4 changed files with 548 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
editor/CMakeLists.txt
View File

@@ -2308,4 +2308,13 @@ target_link_libraries(step380_test PRIVATE
tree_sitter_javascript tree_sitter_typescript
tree_sitter_java tree_sitter_rust tree_sitter_go)
add_executable(step381_test tests/step381_test.cpp)
target_include_directories(step381_test PRIVATE src)
target_link_libraries(step381_test PRIVATE
nlohmann_json::nlohmann_json
unofficial::tree-sitter::tree-sitter
tree_sitter_python tree_sitter_cpp tree_sitter_elisp
tree_sitter_javascript tree_sitter_typescript
tree_sitter_java tree_sitter_rust tree_sitter_go)
# Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies)

303
editor/src/WorkflowProgress.h Normal file
View File

@@ -0,0 +1,303 @@
#pragma once
// Step 381: Workflow progress + ETA tracking
//
// Aggregates orchestrator events into live workflow metrics for completion
// tracking, throughput, ETA, worker performance, and visualization timeline.
#include "WorkflowOrchestrator.h"
#include <algorithm>
#include <cctype>
#include <ctime>
#include <map>
#include <set>
#include <string>
#include <vector>
struct WorkerStats {
int completed = 0;
float avgConfidence = 0.0f;
int avgTokensUsed = 0;
float rejectionRate = 0.0f;
json toJson() const {
return json{
{"completed", completed},
{"avgConfidence", avgConfidence},
{"avgTokensUsed", avgTokensUsed},
{"rejectionRate", rejectionRate}
};
}
};
struct TimelineEntry {
std::string type;
std::string itemId;
std::string timestamp;
json detail;
json toJson() const {
return json{
{"type", type},
{"itemId", itemId},
{"timestamp", timestamp},
{"detail", detail}
};
}
};
struct ProgressSnapshot {
int totalItems = 0;
int completedItems = 0;
float completionPercent = 0.0f;
float itemsPerMinute = 0.0f;
float estimatedRemainingMinutes = -1.0f; // -1 = unknown/unavailable
std::map<std::string, WorkerStats> byWorkerType;
std::string currentPhase;
std::vector<BlockerInfo> blockers;
std::string startedAt;
std::string lastActivityAt;
json toJson() const {
json workers = json::object();
for (const auto& [k, v] : byWorkerType) {
workers[k] = v.toJson();
}
json blockerArr = json::array();
for (const auto& b : blockers) {
blockerArr.push_back(json{
{"type", b.type},
{"itemIds", b.itemIds},
{"description", b.description}
});
}
return json{
{"totalItems", totalItems},
{"completedItems", completedItems},
{"completionPercent", completionPercent},
{"itemsPerMinute", itemsPerMinute},
{"estimatedRemainingMinutes", estimatedRemainingMinutes},
{"byWorkerType", workers},
{"currentPhase", currentPhase},
{"blockers", blockerArr},
{"startedAt", startedAt},
{"lastActivityAt", lastActivityAt}
};
}
};
class WorkflowProgress {
public:
explicit WorkflowProgress(int totalItems = 0) : totalItems_(std::max(0, totalItems)) {}
void setTotalItems(int totalItems) {
totalItems_ = std::max(0, totalItems);
}
void recordEvent(const OrchestratorEvent& event) {
TimelineEntry entry{event.type, event.itemId, event.timestamp, event.detail};
timeline_.push_back(entry);
if (startedAt_.empty()) startedAt_ = event.timestamp;
lastActivityAt_ = event.timestamp;
if (event.type == "routed") {
noteWorkerType(event.itemId, event.detail.value("workerType", ""));
clearMatchingBlockers(event.itemId);
} else if (event.type == "executed") {
std::string worker = event.detail.value("workerType", "");
noteWorkerType(event.itemId, worker);
if (!worker.empty()) {
workerAttempts_[worker]++;
workerConfidenceSum_[worker] += event.detail.value("confidence", 0.0f);
workerTokenSum_[worker] += event.detail.value("tokensGenerated", 0);
}
clearMatchingBlockers(event.itemId);
} else if (event.type == "completed") {
if (completedIds_.insert(event.itemId).second) {
completedItems_++;
std::string worker = workerForItem(event.itemId);
if (!worker.empty()) workerCompleted_[worker]++;
}
clearMatchingBlockers(event.itemId);
} else if (event.type == "rejected") {
std::string worker = workerForItem(event.itemId);
if (!worker.empty()) workerRejected_[worker]++;
} else if (event.type == "blocked") {
updateBlockersFromEvent(event.detail);
}
}
ProgressSnapshot getSnapshot() const {
ProgressSnapshot s;
s.totalItems = totalItems_ > 0 ? totalItems_ : observedTotalItems();
s.completedItems = completedItems_;
s.completionPercent = completionPercent(s.totalItems, s.completedItems);
s.itemsPerMinute = throughputPerMinute();
int remaining = std::max(0, s.totalItems - s.completedItems);
if (s.itemsPerMinute > 0.0f && remaining > 0) {
s.estimatedRemainingMinutes = static_cast<float>(remaining) / s.itemsPerMinute;
} else if (remaining == 0 && s.totalItems > 0) {
s.estimatedRemainingMinutes = 0.0f;
}
s.byWorkerType = getWorkerStats();
s.currentPhase = inferPhase(s);
s.blockers = blockers_;
s.startedAt = startedAt_;
s.lastActivityAt = lastActivityAt_;
return s;
}
std::vector<TimelineEntry> getTimeline() const {
return timeline_;
}
std::map<std::string, WorkerStats> getWorkerStats() const {
std::map<std::string, WorkerStats> out;
std::set<std::string> workerKeys;
for (const auto& [k, _] : workerAttempts_) workerKeys.insert(k);
for (const auto& [k, _] : workerCompleted_) workerKeys.insert(k);
for (const auto& [k, _] : workerRejected_) workerKeys.insert(k);
for (const auto& worker : workerKeys) {
WorkerStats ws;
ws.completed = mapGet(workerCompleted_, worker);
int attempts = mapGet(workerAttempts_, worker);
if (attempts > 0) {
ws.avgConfidence = mapGet(workerConfidenceSum_, worker) /
static_cast<float>(attempts);
ws.avgTokensUsed = mapGet(workerTokenSum_, worker) / attempts;
ws.rejectionRate = static_cast<float>(mapGet(workerRejected_, worker)) /
static_cast<float>(attempts);
}
out[worker] = ws;
}
return out;
}
private:
int totalItems_ = 0;
int completedItems_ = 0;
std::string startedAt_;
std::string lastActivityAt_;
std::vector<TimelineEntry> timeline_;
std::vector<BlockerInfo> blockers_;
std::map<std::string, std::string> itemWorkerType_;
std::set<std::string> completedIds_;
std::map<std::string, int> workerAttempts_;
std::map<std::string, int> workerCompleted_;
std::map<std::string, int> workerRejected_;
std::map<std::string, float> workerConfidenceSum_;
std::map<std::string, int> workerTokenSum_;
static bool isDigits(const std::string& s) {
for (char c : s) {
if (!std::isdigit(static_cast<unsigned char>(c))) return false;
}
return !s.empty();
}
static int toIntOrDefault(const std::string& s, int fallback) {
if (!isDigits(s)) return fallback;
return std::stoi(s);
}
static std::time_t parseIso8601(const std::string& ts) {
// Expected format: YYYY-MM-DDTHH:MM:SSZ
if (ts.size() < 20) return 0;
std::tm t{};
t.tm_year = toIntOrDefault(ts.substr(0, 4), 1970) - 1900;
t.tm_mon = toIntOrDefault(ts.substr(5, 2), 1) - 1;
t.tm_mday = toIntOrDefault(ts.substr(8, 2), 1);
t.tm_hour = toIntOrDefault(ts.substr(11, 2), 0);
t.tm_min = toIntOrDefault(ts.substr(14, 2), 0);
t.tm_sec = toIntOrDefault(ts.substr(17, 2), 0);
t.tm_isdst = 0;
return timegm(&t);
}
static float completionPercent(int total, int completed) {
if (total <= 0) return 0.0f;
return 100.0f * static_cast<float>(completed) / static_cast<float>(total);
}
float throughputPerMinute() const {
if (timeline_.size() < 2 || startedAt_.empty() || lastActivityAt_.empty()) {
return 0.0f;
}
std::time_t start = parseIso8601(startedAt_);
std::time_t end = parseIso8601(lastActivityAt_);
if (end <= start) return 0.0f;
double minutes = static_cast<double>(end - start) / 60.0;
if (minutes <= 0.0) return 0.0f;
return static_cast<float>(completedItems_ / minutes);
}
int observedTotalItems() const {
std::set<std::string> itemIds;
for (const auto& e : timeline_) {
if (!e.itemId.empty()) itemIds.insert(e.itemId);
}
return static_cast<int>(itemIds.size());
}
void noteWorkerType(const std::string& itemId, const std::string& workerType) {
if (!itemId.empty() && !workerType.empty()) {
itemWorkerType_[itemId] = workerType;
}
}
std::string workerForItem(const std::string& itemId) const {
auto it = itemWorkerType_.find(itemId);
if (it == itemWorkerType_.end()) return "";
return it->second;
}
static int mapGet(const std::map<std::string, int>& m, const std::string& key) {
auto it = m.find(key);
if (it == m.end()) return 0;
return it->second;
}
static float mapGet(const std::map<std::string, float>& m, const std::string& key) {
auto it = m.find(key);
if (it == m.end()) return 0.0f;
return it->second;
}
void clearMatchingBlockers(const std::string& itemId) {
for (auto& blocker : blockers_) {
auto& ids = blocker.itemIds;
ids.erase(std::remove(ids.begin(), ids.end(), itemId), ids.end());
}
blockers_.erase(std::remove_if(blockers_.begin(), blockers_.end(),
[](const BlockerInfo& b) {
return b.itemIds.empty();
}),
blockers_.end());
}
void updateBlockersFromEvent(const json& detail) {
if (!detail.contains("blockers") || !detail["blockers"].is_array()) return;
blockers_.clear();
for (const auto& b : detail["blockers"]) {
BlockerInfo blocker;
blocker.type = b.value("type", "");
blocker.description = b.value("description", "");
if (b.contains("itemIds") && b["itemIds"].is_array()) {
blocker.itemIds = b["itemIds"].get<std::vector<std::string>>();
}
if (!blocker.type.empty()) blockers_.push_back(blocker);
}
}
static std::string inferPhase(const ProgressSnapshot& s) {
if (s.totalItems > 0 && s.completedItems >= s.totalItems) return "complete";
if (!s.blockers.empty()) return "blocked";
if (s.completedItems > 0) return "executing";
if (s.totalItems > 0) return "running";
return "idle";
}
};

196
editor/tests/step381_test.cpp Normal file
View File

@@ -0,0 +1,196 @@
// Step 381: Workflow progress + ETA tracking (12 tests)
#include <cassert>
#include <cmath>
#include <iostream>
#include <string>
#include "WorkflowProgress.h"
static OrchestratorEvent eventOf(const std::string& type,
const std::string& itemId,
const std::string& timestamp,
const json& detail = json::object()) {
OrchestratorEvent ev;
ev.type = type;
ev.itemId = itemId;
ev.timestamp = timestamp;
ev.detail = detail;
return ev;
}
int main() {
int passed = 0;
// Test 1: completion percent tracks completed/total
{
WorkflowProgress progress(4);
progress.recordEvent(eventOf("completed", "a", "2026-02-16T10:00:00Z"));
progress.recordEvent(eventOf("completed", "b", "2026-02-16T10:01:00Z"));
auto snap = progress.getSnapshot();
assert(snap.totalItems == 4);
assert(snap.completedItems == 2);
assert(std::fabs(snap.completionPercent - 50.0f) < 0.01f);
std::cout << "Test 1 PASSED: completion percent accurate\n";
passed++;
}
// Test 2: throughput computed from elapsed time and completions
{
WorkflowProgress progress(3);
progress.recordEvent(eventOf("completed", "a", "2026-02-16T10:00:00Z"));
progress.recordEvent(eventOf("completed", "b", "2026-02-16T10:02:00Z"));
auto snap = progress.getSnapshot();
assert(std::fabs(snap.itemsPerMinute - 1.0f) < 0.01f);
std::cout << "Test 2 PASSED: throughput computed\n";
passed++;
}
// Test 3: ETA is close for uniform completion cadence
{
WorkflowProgress progress(4);
progress.recordEvent(eventOf("completed", "a", "2026-02-16T10:00:00Z"));
progress.recordEvent(eventOf("completed", "b", "2026-02-16T10:02:00Z"));
auto snap = progress.getSnapshot();
// 2 done in 2 minutes => 1/min; remaining 2 => ~2 minutes.
assert(snap.estimatedRemainingMinutes >= 1.0f);
assert(snap.estimatedRemainingMinutes <= 4.0f);
std::cout << "Test 3 PASSED: ETA stays within reasonable bound\n";
passed++;
}
// Test 4: worker stats tracked per worker type
{
WorkflowProgress progress(2);
progress.recordEvent(eventOf("routed", "a", "2026-02-16T10:00:00Z",
json{{"workerType", "template"}}));
progress.recordEvent(eventOf("executed", "a", "2026-02-16T10:00:10Z",
json{{"workerType", "template"},
{"confidence", 0.9},
{"tokensGenerated", 40}}));
progress.recordEvent(eventOf("completed", "a", "2026-02-16T10:00:20Z"));
progress.recordEvent(eventOf("routed", "b", "2026-02-16T10:00:30Z",
json{{"workerType", "llm"}}));
progress.recordEvent(eventOf("executed", "b", "2026-02-16T10:01:00Z",
json{{"workerType", "llm"},
{"confidence", 0.6},
{"tokensGenerated", 200}}));
auto byWorker = progress.getWorkerStats();
assert(byWorker.count("template") == 1);
assert(byWorker.count("llm") == 1);
assert(byWorker["template"].completed == 1);
assert(byWorker["llm"].completed == 0);
std::cout << "Test 4 PASSED: worker stats grouped by worker type\n";
passed++;
}
// Test 5: rejection rate updates when rejected events are recorded
{
WorkflowProgress progress(1);
progress.recordEvent(eventOf("routed", "a", "2026-02-16T10:00:00Z",
json{{"workerType", "slm"}}));
progress.recordEvent(eventOf("executed", "a", "2026-02-16T10:00:20Z",
json{{"workerType", "slm"},
{"confidence", 0.7},
{"tokensGenerated", 80}}));
progress.recordEvent(eventOf("rejected", "a", "2026-02-16T10:00:25Z"));
auto stats = progress.getWorkerStats();
assert(stats.count("slm") == 1);
assert(std::fabs(stats["slm"].rejectionRate - 1.0f) < 0.01f);
std::cout << "Test 5 PASSED: rejection rate tracked\n";
passed++;
}
// Test 6: timeline records all events in order
{
WorkflowProgress progress(1);
progress.recordEvent(eventOf("routed", "a", "2026-02-16T10:00:00Z"));
progress.recordEvent(eventOf("executed", "a", "2026-02-16T10:00:10Z"));
progress.recordEvent(eventOf("completed", "a", "2026-02-16T10:00:20Z"));
auto timeline = progress.getTimeline();
assert(timeline.size() == 3);
assert(timeline[0].type == "routed");
assert(timeline[2].type == "completed");
std::cout << "Test 6 PASSED: timeline event log preserved\n";
passed++;
}
// Test 7: empty snapshot shows zero progress and unknown ETA
{
WorkflowProgress progress;
auto snap = progress.getSnapshot();
assert(snap.completionPercent == 0.0f);
assert(snap.estimatedRemainingMinutes < 0.0f);
std::cout << "Test 7 PASSED: empty workflow snapshot sane\n";
passed++;
}
// Test 8: blockers included from blocked event payload
{
WorkflowProgress progress(2);
progress.recordEvent(eventOf("blocked", "", "2026-02-16T10:00:00Z",
json{{"blockers", json::array({
json{{"type", "needs-human"},
{"itemIds", json::array({"x"})},
{"description", "manual"}}})}}));
auto snap = progress.getSnapshot();
assert(snap.blockers.size() == 1);
assert(snap.blockers[0].type == "needs-human");
std::cout << "Test 8 PASSED: blockers surfaced in snapshot\n";
passed++;
}
// Test 9: snapshot JSON serialization has expected fields
{
WorkflowProgress progress(1);
progress.recordEvent(eventOf("completed", "a", "2026-02-16T10:00:00Z"));
auto j = progress.getSnapshot().toJson();
assert(j.contains("totalItems"));
assert(j.contains("completionPercent"));
assert(j.contains("byWorkerType"));
assert(j.contains("blockers"));
std::cout << "Test 9 PASSED: snapshot serializes to JSON\n";
passed++;
}
// Test 10: completion percent stays monotonic under rejections
{
WorkflowProgress progress(3);
progress.recordEvent(eventOf("completed", "a", "2026-02-16T10:00:00Z"));
float p1 = progress.getSnapshot().completionPercent;
progress.recordEvent(eventOf("rejected", "b", "2026-02-16T10:00:10Z"));
float p2 = progress.getSnapshot().completionPercent;
progress.recordEvent(eventOf("completed", "b", "2026-02-16T10:01:00Z"));
float p3 = progress.getSnapshot().completionPercent;
assert(p2 >= p1);
assert(p3 >= p2);
std::cout << "Test 10 PASSED: completion stays monotonic\n";
passed++;
}
// Test 11: startedAt and lastActivityAt are tracked
{
WorkflowProgress progress(1);
progress.recordEvent(eventOf("routed", "a", "2026-02-16T10:00:00Z"));
progress.recordEvent(eventOf("completed", "a", "2026-02-16T10:02:00Z"));
auto snap = progress.getSnapshot();
assert(snap.startedAt == "2026-02-16T10:00:00Z");
assert(snap.lastActivityAt == "2026-02-16T10:02:00Z");
std::cout << "Test 11 PASSED: activity timestamps tracked\n";
passed++;
}
// Test 12: total items fallback to observed unique item ids
{
WorkflowProgress progress;
progress.recordEvent(eventOf("routed", "a", "2026-02-16T10:00:00Z"));
progress.recordEvent(eventOf("routed", "b", "2026-02-16T10:00:10Z"));
auto snap = progress.getSnapshot();
assert(snap.totalItems == 2);
std::cout << "Test 12 PASSED: observed-item fallback for total count\n";
passed++;
}
std::cout << "\nResults: " << passed << "/12\n";
assert(passed == 12);
return 0;
}

40
progress.md
View File

@@ -2879,6 +2879,46 @@ assembly for re-attempts.
- `editor/src/ContextAssembler.h` within header-size limit (`192` <= `600`)
- `editor/tests/step380_test.cpp` within test-file size guidance (`281` lines)
### Step 381: Progress Tracking + ETA
**Status:** PASS (12/12 tests)
Implemented a standalone workflow-progress tracker that consumes orchestrator
events and produces progress snapshots with completion %, throughput, ETA,
worker metrics, blockers, and timeline entries.
**Files created:**
- `editor/src/WorkflowProgress.h` — progress aggregation API:
- `WorkflowProgress::recordEvent(...)`
- `WorkflowProgress::getSnapshot()`
- `WorkflowProgress::getTimeline()`
- `WorkflowProgress::getWorkerStats()`
- `ProgressSnapshot`, `WorkerStats`, `TimelineEntry` with JSON serialization
- `editor/tests/step381_test.cpp` — 12 tests covering:
1. completion-percent accuracy
2. throughput (`itemsPerMinute`) calculation
3. ETA bound sanity for uniform completion cadence
4. per-worker metric aggregation
5. rejection-rate tracking
6. timeline event recording
7. empty-workflow snapshot behavior
8. blocker propagation into snapshots
9. snapshot JSON field coverage
10. completion monotonicity across rejection events
11. `startedAt`/`lastActivityAt` tracking
12. total-item fallback via observed item IDs
**Files modified:**
- `editor/CMakeLists.txt` — `step381_test` target
**Verification run:**
- `step381_test` — PASS (12/12) new step coverage
- `step380_test` — PASS (12/12) regression coverage
- `step379_test` — PASS (12/12) regression coverage
**Architecture gate check:**
- `editor/src/WorkflowProgress.h` within header-size limit (`303` <= `600`)
- `editor/tests/step381_test.cpp` within test-file size guidance (`196` lines)
# Roadmap Planning — Sprints 12-25+
## Status: Planning Complete (Sprints 12-19 detailed, 20-25 in roadmap.md)