From 5300be080ae35988c627ce58a228e425745181f0 Mon Sep 17 00:00:00 2001 From: Bill Date: Mon, 16 Feb 2026 12:39:37 -0700 Subject: [PATCH] Step 381: add workflow progress and ETA tracker --- editor/CMakeLists.txt | 9 + editor/src/WorkflowProgress.h | 303 ++++++++++++++++++++++++++++++++++ editor/tests/step381_test.cpp | 196 ++++++++++++++++++++++ progress.md | 40 +++++ 4 files changed, 548 insertions(+) create mode 100644 editor/src/WorkflowProgress.h create mode 100644 editor/tests/step381_test.cpp diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index c7fef6c..5089143 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -2308,4 +2308,13 @@ target_link_libraries(step380_test PRIVATE tree_sitter_javascript tree_sitter_typescript tree_sitter_java tree_sitter_rust tree_sitter_go) +add_executable(step381_test tests/step381_test.cpp) +target_include_directories(step381_test PRIVATE src) +target_link_libraries(step381_test PRIVATE + nlohmann_json::nlohmann_json + unofficial::tree-sitter::tree-sitter + tree_sitter_python tree_sitter_cpp tree_sitter_elisp + tree_sitter_javascript tree_sitter_typescript + tree_sitter_java tree_sitter_rust tree_sitter_go) + # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies) diff --git a/editor/src/WorkflowProgress.h b/editor/src/WorkflowProgress.h new file mode 100644 index 0000000..1bd7d55 --- /dev/null +++ b/editor/src/WorkflowProgress.h @@ -0,0 +1,303 @@ +#pragma once +// Step 381: Workflow progress + ETA tracking +// +// Aggregates orchestrator events into live workflow metrics for completion +// tracking, throughput, ETA, worker performance, and visualization timeline. + +#include "WorkflowOrchestrator.h" +#include +#include +#include +#include +#include +#include +#include + +struct WorkerStats { + int completed = 0; + float avgConfidence = 0.0f; + int avgTokensUsed = 0; + float rejectionRate = 0.0f; + + json toJson() const { + return json{ + {"completed", completed}, + {"avgConfidence", avgConfidence}, + {"avgTokensUsed", avgTokensUsed}, + {"rejectionRate", rejectionRate} + }; + } +}; + +struct TimelineEntry { + std::string type; + std::string itemId; + std::string timestamp; + json detail; + + json toJson() const { + return json{ + {"type", type}, + {"itemId", itemId}, + {"timestamp", timestamp}, + {"detail", detail} + }; + } +}; + +struct ProgressSnapshot { + int totalItems = 0; + int completedItems = 0; + float completionPercent = 0.0f; + float itemsPerMinute = 0.0f; + float estimatedRemainingMinutes = -1.0f; // -1 = unknown/unavailable + std::map byWorkerType; + std::string currentPhase; + std::vector blockers; + std::string startedAt; + std::string lastActivityAt; + + json toJson() const { + json workers = json::object(); + for (const auto& [k, v] : byWorkerType) { + workers[k] = v.toJson(); + } + + json blockerArr = json::array(); + for (const auto& b : blockers) { + blockerArr.push_back(json{ + {"type", b.type}, + {"itemIds", b.itemIds}, + {"description", b.description} + }); + } + + return json{ + {"totalItems", totalItems}, + {"completedItems", completedItems}, + {"completionPercent", completionPercent}, + {"itemsPerMinute", itemsPerMinute}, + {"estimatedRemainingMinutes", estimatedRemainingMinutes}, + {"byWorkerType", workers}, + {"currentPhase", currentPhase}, + {"blockers", blockerArr}, + {"startedAt", startedAt}, + {"lastActivityAt", lastActivityAt} + }; + } +}; + +class WorkflowProgress { +public: + explicit WorkflowProgress(int totalItems = 0) : totalItems_(std::max(0, totalItems)) {} + + void setTotalItems(int totalItems) { + totalItems_ = std::max(0, totalItems); + } + + void recordEvent(const OrchestratorEvent& event) { + TimelineEntry entry{event.type, event.itemId, event.timestamp, event.detail}; + timeline_.push_back(entry); + + if (startedAt_.empty()) startedAt_ = event.timestamp; + lastActivityAt_ = event.timestamp; + + if (event.type == "routed") { + noteWorkerType(event.itemId, event.detail.value("workerType", "")); + clearMatchingBlockers(event.itemId); + } else if (event.type == "executed") { + std::string worker = event.detail.value("workerType", ""); + noteWorkerType(event.itemId, worker); + if (!worker.empty()) { + workerAttempts_[worker]++; + workerConfidenceSum_[worker] += event.detail.value("confidence", 0.0f); + workerTokenSum_[worker] += event.detail.value("tokensGenerated", 0); + } + clearMatchingBlockers(event.itemId); + } else if (event.type == "completed") { + if (completedIds_.insert(event.itemId).second) { + completedItems_++; + std::string worker = workerForItem(event.itemId); + if (!worker.empty()) workerCompleted_[worker]++; + } + clearMatchingBlockers(event.itemId); + } else if (event.type == "rejected") { + std::string worker = workerForItem(event.itemId); + if (!worker.empty()) workerRejected_[worker]++; + } else if (event.type == "blocked") { + updateBlockersFromEvent(event.detail); + } + } + + ProgressSnapshot getSnapshot() const { + ProgressSnapshot s; + s.totalItems = totalItems_ > 0 ? totalItems_ : observedTotalItems(); + s.completedItems = completedItems_; + s.completionPercent = completionPercent(s.totalItems, s.completedItems); + s.itemsPerMinute = throughputPerMinute(); + int remaining = std::max(0, s.totalItems - s.completedItems); + if (s.itemsPerMinute > 0.0f && remaining > 0) { + s.estimatedRemainingMinutes = static_cast(remaining) / s.itemsPerMinute; + } else if (remaining == 0 && s.totalItems > 0) { + s.estimatedRemainingMinutes = 0.0f; + } + s.byWorkerType = getWorkerStats(); + s.currentPhase = inferPhase(s); + s.blockers = blockers_; + s.startedAt = startedAt_; + s.lastActivityAt = lastActivityAt_; + return s; + } + + std::vector getTimeline() const { + return timeline_; + } + + std::map getWorkerStats() const { + std::map out; + std::set workerKeys; + for (const auto& [k, _] : workerAttempts_) workerKeys.insert(k); + for (const auto& [k, _] : workerCompleted_) workerKeys.insert(k); + for (const auto& [k, _] : workerRejected_) workerKeys.insert(k); + + for (const auto& worker : workerKeys) { + WorkerStats ws; + ws.completed = mapGet(workerCompleted_, worker); + int attempts = mapGet(workerAttempts_, worker); + if (attempts > 0) { + ws.avgConfidence = mapGet(workerConfidenceSum_, worker) / + static_cast(attempts); + ws.avgTokensUsed = mapGet(workerTokenSum_, worker) / attempts; + ws.rejectionRate = static_cast(mapGet(workerRejected_, worker)) / + static_cast(attempts); + } + out[worker] = ws; + } + return out; + } + +private: + int totalItems_ = 0; + int completedItems_ = 0; + std::string startedAt_; + std::string lastActivityAt_; + std::vector timeline_; + std::vector blockers_; + std::map itemWorkerType_; + std::set completedIds_; + + std::map workerAttempts_; + std::map workerCompleted_; + std::map workerRejected_; + std::map workerConfidenceSum_; + std::map workerTokenSum_; + + static bool isDigits(const std::string& s) { + for (char c : s) { + if (!std::isdigit(static_cast(c))) return false; + } + return !s.empty(); + } + + static int toIntOrDefault(const std::string& s, int fallback) { + if (!isDigits(s)) return fallback; + return std::stoi(s); + } + + static std::time_t parseIso8601(const std::string& ts) { + // Expected format: YYYY-MM-DDTHH:MM:SSZ + if (ts.size() < 20) return 0; + std::tm t{}; + t.tm_year = toIntOrDefault(ts.substr(0, 4), 1970) - 1900; + t.tm_mon = toIntOrDefault(ts.substr(5, 2), 1) - 1; + t.tm_mday = toIntOrDefault(ts.substr(8, 2), 1); + t.tm_hour = toIntOrDefault(ts.substr(11, 2), 0); + t.tm_min = toIntOrDefault(ts.substr(14, 2), 0); + t.tm_sec = toIntOrDefault(ts.substr(17, 2), 0); + t.tm_isdst = 0; + return timegm(&t); + } + + static float completionPercent(int total, int completed) { + if (total <= 0) return 0.0f; + return 100.0f * static_cast(completed) / static_cast(total); + } + + float throughputPerMinute() const { + if (timeline_.size() < 2 || startedAt_.empty() || lastActivityAt_.empty()) { + return 0.0f; + } + std::time_t start = parseIso8601(startedAt_); + std::time_t end = parseIso8601(lastActivityAt_); + if (end <= start) return 0.0f; + double minutes = static_cast(end - start) / 60.0; + if (minutes <= 0.0) return 0.0f; + return static_cast(completedItems_ / minutes); + } + + int observedTotalItems() const { + std::set itemIds; + for (const auto& e : timeline_) { + if (!e.itemId.empty()) itemIds.insert(e.itemId); + } + return static_cast(itemIds.size()); + } + + void noteWorkerType(const std::string& itemId, const std::string& workerType) { + if (!itemId.empty() && !workerType.empty()) { + itemWorkerType_[itemId] = workerType; + } + } + + std::string workerForItem(const std::string& itemId) const { + auto it = itemWorkerType_.find(itemId); + if (it == itemWorkerType_.end()) return ""; + return it->second; + } + + static int mapGet(const std::map& m, const std::string& key) { + auto it = m.find(key); + if (it == m.end()) return 0; + return it->second; + } + + static float mapGet(const std::map& m, const std::string& key) { + auto it = m.find(key); + if (it == m.end()) return 0.0f; + return it->second; + } + + void clearMatchingBlockers(const std::string& itemId) { + for (auto& blocker : blockers_) { + auto& ids = blocker.itemIds; + ids.erase(std::remove(ids.begin(), ids.end(), itemId), ids.end()); + } + blockers_.erase(std::remove_if(blockers_.begin(), blockers_.end(), + [](const BlockerInfo& b) { + return b.itemIds.empty(); + }), + blockers_.end()); + } + + void updateBlockersFromEvent(const json& detail) { + if (!detail.contains("blockers") || !detail["blockers"].is_array()) return; + blockers_.clear(); + for (const auto& b : detail["blockers"]) { + BlockerInfo blocker; + blocker.type = b.value("type", ""); + blocker.description = b.value("description", ""); + if (b.contains("itemIds") && b["itemIds"].is_array()) { + blocker.itemIds = b["itemIds"].get>(); + } + if (!blocker.type.empty()) blockers_.push_back(blocker); + } + } + + static std::string inferPhase(const ProgressSnapshot& s) { + if (s.totalItems > 0 && s.completedItems >= s.totalItems) return "complete"; + if (!s.blockers.empty()) return "blocked"; + if (s.completedItems > 0) return "executing"; + if (s.totalItems > 0) return "running"; + return "idle"; + } +}; diff --git a/editor/tests/step381_test.cpp b/editor/tests/step381_test.cpp new file mode 100644 index 0000000..2b34599 --- /dev/null +++ b/editor/tests/step381_test.cpp @@ -0,0 +1,196 @@ +// Step 381: Workflow progress + ETA tracking (12 tests) + +#include +#include +#include +#include +#include "WorkflowProgress.h" + +static OrchestratorEvent eventOf(const std::string& type, + const std::string& itemId, + const std::string& timestamp, + const json& detail = json::object()) { + OrchestratorEvent ev; + ev.type = type; + ev.itemId = itemId; + ev.timestamp = timestamp; + ev.detail = detail; + return ev; +} + +int main() { + int passed = 0; + + // Test 1: completion percent tracks completed/total + { + WorkflowProgress progress(4); + progress.recordEvent(eventOf("completed", "a", "2026-02-16T10:00:00Z")); + progress.recordEvent(eventOf("completed", "b", "2026-02-16T10:01:00Z")); + auto snap = progress.getSnapshot(); + assert(snap.totalItems == 4); + assert(snap.completedItems == 2); + assert(std::fabs(snap.completionPercent - 50.0f) < 0.01f); + std::cout << "Test 1 PASSED: completion percent accurate\n"; + passed++; + } + + // Test 2: throughput computed from elapsed time and completions + { + WorkflowProgress progress(3); + progress.recordEvent(eventOf("completed", "a", "2026-02-16T10:00:00Z")); + progress.recordEvent(eventOf("completed", "b", "2026-02-16T10:02:00Z")); + auto snap = progress.getSnapshot(); + assert(std::fabs(snap.itemsPerMinute - 1.0f) < 0.01f); + std::cout << "Test 2 PASSED: throughput computed\n"; + passed++; + } + + // Test 3: ETA is close for uniform completion cadence + { + WorkflowProgress progress(4); + progress.recordEvent(eventOf("completed", "a", "2026-02-16T10:00:00Z")); + progress.recordEvent(eventOf("completed", "b", "2026-02-16T10:02:00Z")); + auto snap = progress.getSnapshot(); + // 2 done in 2 minutes => 1/min; remaining 2 => ~2 minutes. + assert(snap.estimatedRemainingMinutes >= 1.0f); + assert(snap.estimatedRemainingMinutes <= 4.0f); + std::cout << "Test 3 PASSED: ETA stays within reasonable bound\n"; + passed++; + } + + // Test 4: worker stats tracked per worker type + { + WorkflowProgress progress(2); + progress.recordEvent(eventOf("routed", "a", "2026-02-16T10:00:00Z", + json{{"workerType", "template"}})); + progress.recordEvent(eventOf("executed", "a", "2026-02-16T10:00:10Z", + json{{"workerType", "template"}, + {"confidence", 0.9}, + {"tokensGenerated", 40}})); + progress.recordEvent(eventOf("completed", "a", "2026-02-16T10:00:20Z")); + progress.recordEvent(eventOf("routed", "b", "2026-02-16T10:00:30Z", + json{{"workerType", "llm"}})); + progress.recordEvent(eventOf("executed", "b", "2026-02-16T10:01:00Z", + json{{"workerType", "llm"}, + {"confidence", 0.6}, + {"tokensGenerated", 200}})); + auto byWorker = progress.getWorkerStats(); + assert(byWorker.count("template") == 1); + assert(byWorker.count("llm") == 1); + assert(byWorker["template"].completed == 1); + assert(byWorker["llm"].completed == 0); + std::cout << "Test 4 PASSED: worker stats grouped by worker type\n"; + passed++; + } + + // Test 5: rejection rate updates when rejected events are recorded + { + WorkflowProgress progress(1); + progress.recordEvent(eventOf("routed", "a", "2026-02-16T10:00:00Z", + json{{"workerType", "slm"}})); + progress.recordEvent(eventOf("executed", "a", "2026-02-16T10:00:20Z", + json{{"workerType", "slm"}, + {"confidence", 0.7}, + {"tokensGenerated", 80}})); + progress.recordEvent(eventOf("rejected", "a", "2026-02-16T10:00:25Z")); + auto stats = progress.getWorkerStats(); + assert(stats.count("slm") == 1); + assert(std::fabs(stats["slm"].rejectionRate - 1.0f) < 0.01f); + std::cout << "Test 5 PASSED: rejection rate tracked\n"; + passed++; + } + + // Test 6: timeline records all events in order + { + WorkflowProgress progress(1); + progress.recordEvent(eventOf("routed", "a", "2026-02-16T10:00:00Z")); + progress.recordEvent(eventOf("executed", "a", "2026-02-16T10:00:10Z")); + progress.recordEvent(eventOf("completed", "a", "2026-02-16T10:00:20Z")); + auto timeline = progress.getTimeline(); + assert(timeline.size() == 3); + assert(timeline[0].type == "routed"); + assert(timeline[2].type == "completed"); + std::cout << "Test 6 PASSED: timeline event log preserved\n"; + passed++; + } + + // Test 7: empty snapshot shows zero progress and unknown ETA + { + WorkflowProgress progress; + auto snap = progress.getSnapshot(); + assert(snap.completionPercent == 0.0f); + assert(snap.estimatedRemainingMinutes < 0.0f); + std::cout << "Test 7 PASSED: empty workflow snapshot sane\n"; + passed++; + } + + // Test 8: blockers included from blocked event payload + { + WorkflowProgress progress(2); + progress.recordEvent(eventOf("blocked", "", "2026-02-16T10:00:00Z", + json{{"blockers", json::array({ + json{{"type", "needs-human"}, + {"itemIds", json::array({"x"})}, + {"description", "manual"}}})}})); + auto snap = progress.getSnapshot(); + assert(snap.blockers.size() == 1); + assert(snap.blockers[0].type == "needs-human"); + std::cout << "Test 8 PASSED: blockers surfaced in snapshot\n"; + passed++; + } + + // Test 9: snapshot JSON serialization has expected fields + { + WorkflowProgress progress(1); + progress.recordEvent(eventOf("completed", "a", "2026-02-16T10:00:00Z")); + auto j = progress.getSnapshot().toJson(); + assert(j.contains("totalItems")); + assert(j.contains("completionPercent")); + assert(j.contains("byWorkerType")); + assert(j.contains("blockers")); + std::cout << "Test 9 PASSED: snapshot serializes to JSON\n"; + passed++; + } + + // Test 10: completion percent stays monotonic under rejections + { + WorkflowProgress progress(3); + progress.recordEvent(eventOf("completed", "a", "2026-02-16T10:00:00Z")); + float p1 = progress.getSnapshot().completionPercent; + progress.recordEvent(eventOf("rejected", "b", "2026-02-16T10:00:10Z")); + float p2 = progress.getSnapshot().completionPercent; + progress.recordEvent(eventOf("completed", "b", "2026-02-16T10:01:00Z")); + float p3 = progress.getSnapshot().completionPercent; + assert(p2 >= p1); + assert(p3 >= p2); + std::cout << "Test 10 PASSED: completion stays monotonic\n"; + passed++; + } + + // Test 11: startedAt and lastActivityAt are tracked + { + WorkflowProgress progress(1); + progress.recordEvent(eventOf("routed", "a", "2026-02-16T10:00:00Z")); + progress.recordEvent(eventOf("completed", "a", "2026-02-16T10:02:00Z")); + auto snap = progress.getSnapshot(); + assert(snap.startedAt == "2026-02-16T10:00:00Z"); + assert(snap.lastActivityAt == "2026-02-16T10:02:00Z"); + std::cout << "Test 11 PASSED: activity timestamps tracked\n"; + passed++; + } + + // Test 12: total items fallback to observed unique item ids + { + WorkflowProgress progress; + progress.recordEvent(eventOf("routed", "a", "2026-02-16T10:00:00Z")); + progress.recordEvent(eventOf("routed", "b", "2026-02-16T10:00:10Z")); + auto snap = progress.getSnapshot(); + assert(snap.totalItems == 2); + std::cout << "Test 12 PASSED: observed-item fallback for total count\n"; + passed++; + } + + std::cout << "\nResults: " << passed << "/12\n"; + assert(passed == 12); + return 0; +} diff --git a/progress.md b/progress.md index 000930b..01e8f15 100644 --- a/progress.md +++ b/progress.md @@ -2879,6 +2879,46 @@ assembly for re-attempts. - `editor/src/ContextAssembler.h` within header-size limit (`192` <= `600`) - `editor/tests/step380_test.cpp` within test-file size guidance (`281` lines) +### Step 381: Progress Tracking + ETA +**Status:** PASS (12/12 tests) + +Implemented a standalone workflow-progress tracker that consumes orchestrator +events and produces progress snapshots with completion %, throughput, ETA, +worker metrics, blockers, and timeline entries. + +**Files created:** +- `editor/src/WorkflowProgress.h` — progress aggregation API: + - `WorkflowProgress::recordEvent(...)` + - `WorkflowProgress::getSnapshot()` + - `WorkflowProgress::getTimeline()` + - `WorkflowProgress::getWorkerStats()` + - `ProgressSnapshot`, `WorkerStats`, `TimelineEntry` with JSON serialization +- `editor/tests/step381_test.cpp` — 12 tests covering: + 1. completion-percent accuracy + 2. throughput (`itemsPerMinute`) calculation + 3. ETA bound sanity for uniform completion cadence + 4. per-worker metric aggregation + 5. rejection-rate tracking + 6. timeline event recording + 7. empty-workflow snapshot behavior + 8. blocker propagation into snapshots + 9. snapshot JSON field coverage + 10. completion monotonicity across rejection events + 11. `startedAt`/`lastActivityAt` tracking + 12. total-item fallback via observed item IDs + +**Files modified:** +- `editor/CMakeLists.txt` — `step381_test` target + +**Verification run:** +- `step381_test` — PASS (12/12) new step coverage +- `step380_test` — PASS (12/12) regression coverage +- `step379_test` — PASS (12/12) regression coverage + +**Architecture gate check:** +- `editor/src/WorkflowProgress.h` within header-size limit (`303` <= `600`) +- `editor/tests/step381_test.cpp` within test-file size guidance (`196` lines) + # Roadmap Planning — Sprints 12-25+ ## Status: Planning Complete (Sprints 12-19 detailed, 20-25 in roadmap.md)