Step 426: add workflow cost tracking and reporting

2026-02-16 17:02:39 -07:00
parent 9f4d360b89
commit fc950798f0
4 changed files with 377 additions and 0 deletions
--- a/editor/CMakeLists.txt
+++ b/editor/CMakeLists.txt
@@ -2713,4 +2713,13 @@ target_link_libraries(step425_test PRIVATE
    tree_sitter_javascript tree_sitter_typescript
    tree_sitter_java tree_sitter_rust tree_sitter_go)

+add_executable(step426_test tests/step426_test.cpp)
+target_include_directories(step426_test PRIVATE src)
+target_link_libraries(step426_test PRIVATE
+    nlohmann_json::nlohmann_json
+    unofficial::tree-sitter::tree-sitter
+    tree_sitter_python tree_sitter_cpp tree_sitter_elisp
+    tree_sitter_javascript tree_sitter_typescript
+    tree_sitter_java tree_sitter_rust tree_sitter_go)
+
 # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies)
--- a/editor/src/WorkflowCostTracker.h
+++ b/editor/src/WorkflowCostTracker.h
@@ -0,0 +1,150 @@
+#pragma once
+
+#include "ModelProfileRegistry.h"
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include <nlohmann/json.hpp>
+using json = nlohmann::json;
+
+struct CostRecord {
+    std::string itemId;
+    std::string workerType;   // slm|llm|deterministic|template|human
+    std::string profileName;  // resolved model profile name
+    int estimatedInputTokens = 0;
+    int estimatedOutputTokens = 0;
+    int actualInputTokens = 0;
+    int actualOutputTokens = 0;
+};
+
+struct CostSummary {
+    int estimatedInputTokens = 0;
+    int estimatedOutputTokens = 0;
+    int actualInputTokens = 0;
+    int actualOutputTokens = 0;
+    double estimatedCost = 0.0;
+    double actualCost = 0.0;
+    std::map<std::string, double> costByWorker;
+    std::map<std::string, double> costByProfile;
+
+    double varianceCost() const { return actualCost - estimatedCost; }
+};
+
+class WorkflowCostTracker {
+public:
+    explicit WorkflowCostTracker(const ModelProfileRegistry* registry = nullptr)
+        : registry_(registry) {}
+
+    void setRegistry(const ModelProfileRegistry* registry) {
+        registry_ = registry;
+    }
+
+    void recordEstimate(const std::string& itemId,
+                        const std::string& workerType,
+                        int inputTokens,
+                        int outputTokens) {
+        CostRecord& rec = records_[itemId];
+        rec.itemId = itemId;
+        rec.workerType = workerType;
+        rec.estimatedInputTokens = inputTokens;
+        rec.estimatedOutputTokens = outputTokens;
+        if (rec.profileName.empty()) rec.profileName = resolveProfile(workerType);
+    }
+
+    void recordActual(const std::string& itemId,
+                      const std::string& workerType,
+                      int inputTokens,
+                      int outputTokens,
+                      const std::string& explicitProfile = "") {
+        CostRecord& rec = records_[itemId];
+        rec.itemId = itemId;
+        rec.workerType = workerType;
+        rec.actualInputTokens = inputTokens;
+        rec.actualOutputTokens = outputTokens;
+        rec.profileName = !explicitProfile.empty() ? explicitProfile : resolveProfile(workerType);
+    }
+
+    bool hasRecord(const std::string& itemId) const {
+        return records_.find(itemId) != records_.end();
+    }
+
+    CostRecord getRecord(const std::string& itemId) const {
+        auto it = records_.find(itemId);
+        if (it == records_.end()) return {};
+        return it->second;
+    }
+
+    size_t count() const { return records_.size(); }
+
+    CostSummary summarize() const {
+        CostSummary out;
+        for (const auto& [_, rec] : records_) {
+            out.estimatedInputTokens += rec.estimatedInputTokens;
+            out.estimatedOutputTokens += rec.estimatedOutputTokens;
+            out.actualInputTokens += rec.actualInputTokens;
+            out.actualOutputTokens += rec.actualOutputTokens;
+
+            double est = estimateCost(rec.profileName, rec.estimatedInputTokens,
+                                      rec.estimatedOutputTokens);
+            double act = estimateCost(rec.profileName, rec.actualInputTokens,
+                                      rec.actualOutputTokens);
+            out.estimatedCost += est;
+            out.actualCost += act;
+            out.costByWorker[rec.workerType] += act;
+            out.costByProfile[rec.profileName] += act;
+        }
+        return out;
+    }
+
+    json reportJson() const {
+        auto s = summarize();
+        json recs = json::array();
+        for (const auto& [_, rec] : records_) {
+            recs.push_back({
+                {"itemId", rec.itemId},
+                {"workerType", rec.workerType},
+                {"profileName", rec.profileName},
+                {"estimatedInputTokens", rec.estimatedInputTokens},
+                {"estimatedOutputTokens", rec.estimatedOutputTokens},
+                {"actualInputTokens", rec.actualInputTokens},
+                {"actualOutputTokens", rec.actualOutputTokens}
+            });
+        }
+        return {
+            {"recordCount", (int)records_.size()},
+            {"estimatedInputTokens", s.estimatedInputTokens},
+            {"estimatedOutputTokens", s.estimatedOutputTokens},
+            {"actualInputTokens", s.actualInputTokens},
+            {"actualOutputTokens", s.actualOutputTokens},
+            {"estimatedCost", s.estimatedCost},
+            {"actualCost", s.actualCost},
+            {"costVariance", s.varianceCost()},
+            {"costByWorker", s.costByWorker},
+            {"costByProfile", s.costByProfile},
+            {"records", recs}
+        };
+    }
+
+private:
+    const ModelProfileRegistry* registry_ = nullptr;
+    std::map<std::string, CostRecord> records_;
+
+    std::string resolveProfile(const std::string& workerType) const {
+        if (!registry_) return "unknown";
+        std::string name = registry_->resolveProfileForWorker(workerType);
+        return name.empty() ? "unknown" : name;
+    }
+
+    double estimateCost(const std::string& profileName,
+                        int inputTokens,
+                        int outputTokens) const {
+        if (!registry_ || profileName.empty() || profileName == "unknown") return 0.0;
+        auto p = registry_->getProfile(profileName);
+        if (p.name.empty()) return 0.0;
+        double inCost = (double)inputTokens / 1000.0 * p.costPer1kInput;
+        double outCost = (double)outputTokens / 1000.0 * p.costPer1kOutput;
+        return inCost + outCost;
+    }
+};
--- a/editor/tests/step426_test.cpp
+++ b/editor/tests/step426_test.cpp
@@ -0,0 +1,174 @@
+// Step 426: Cost Tracking + Reporting Tests (12 tests)
+
+#include "WorkflowCostTracker.h"
+
+#include <iostream>
+
+static int passed = 0, failed = 0;
+#define TEST(name) { std::cout << "  " << #name << "... "; }
+#define PASS() { std::cout << "PASS\n"; ++passed; }
+#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; }
+#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {}
+
+static ModelProfileRegistry makeRegistry() {
+    ModelProfileRegistry reg;
+    reg.setWorkerProfileMapping("llm", "claude-sonnet");
+    reg.setWorkerProfileMapping("slm", "claude-haiku");
+    return reg;
+}
+
+void test_record_estimate_creates_record() {
+    TEST(record_estimate_creates_record);
+    auto reg = makeRegistry();
+    WorkflowCostTracker tracker(&reg);
+    tracker.recordEstimate("w1", "llm", 1000, 200);
+    CHECK(tracker.hasRecord("w1"), "record missing");
+    CHECK(tracker.getRecord("w1").estimatedInputTokens == 1000, "estimate not stored");
+    PASS();
+}
+
+void test_record_actual_updates_existing_record() {
+    TEST(record_actual_updates_existing_record);
+    auto reg = makeRegistry();
+    WorkflowCostTracker tracker(&reg);
+    tracker.recordEstimate("w1", "llm", 1000, 200);
+    tracker.recordActual("w1", "llm", 1200, 250);
+    auto rec = tracker.getRecord("w1");
+    CHECK(rec.actualInputTokens == 1200, "actual input mismatch");
+    CHECK(rec.actualOutputTokens == 250, "actual output mismatch");
+    PASS();
+}
+
+void test_record_actual_creates_new_record() {
+    TEST(record_actual_creates_new_record);
+    auto reg = makeRegistry();
+    WorkflowCostTracker tracker(&reg);
+    tracker.recordActual("w2", "slm", 300, 100);
+    CHECK(tracker.hasRecord("w2"), "actual-only record missing");
+    PASS();
+}
+
+void test_profile_resolves_from_worker_mapping() {
+    TEST(profile_resolves_from_worker_mapping);
+    auto reg = makeRegistry();
+    WorkflowCostTracker tracker(&reg);
+    tracker.recordEstimate("w1", "llm", 10, 10);
+    CHECK(tracker.getRecord("w1").profileName == "claude-sonnet",
+          "llm should map to sonnet");
+    PASS();
+}
+
+void test_explicit_profile_overrides_mapping() {
+    TEST(explicit_profile_overrides_mapping);
+    auto reg = makeRegistry();
+    WorkflowCostTracker tracker(&reg);
+    tracker.recordActual("w1", "llm", 100, 100, "claude-opus");
+    CHECK(tracker.getRecord("w1").profileName == "claude-opus",
+          "explicit profile should win");
+    PASS();
+}
+
+void test_summary_token_totals() {
+    TEST(summary_token_totals);
+    auto reg = makeRegistry();
+    WorkflowCostTracker tracker(&reg);
+    tracker.recordEstimate("w1", "llm", 100, 20);
+    tracker.recordEstimate("w2", "slm", 200, 30);
+    tracker.recordActual("w1", "llm", 110, 25);
+    tracker.recordActual("w2", "slm", 180, 28);
+    auto s = tracker.summarize();
+    CHECK(s.estimatedInputTokens == 300, "estimated input sum mismatch");
+    CHECK(s.actualOutputTokens == 53, "actual output sum mismatch");
+    PASS();
+}
+
+void test_summary_costs_non_negative() {
+    TEST(summary_costs_non_negative);
+    auto reg = makeRegistry();
+    WorkflowCostTracker tracker(&reg);
+    tracker.recordEstimate("w1", "llm", 1500, 500);
+    tracker.recordActual("w1", "llm", 1500, 500);
+    auto s = tracker.summarize();
+    CHECK(s.estimatedCost >= 0.0, "estimated cost negative");
+    CHECK(s.actualCost >= 0.0, "actual cost negative");
+    PASS();
+}
+
+void test_summary_cost_by_worker_present() {
+    TEST(summary_cost_by_worker_present);
+    auto reg = makeRegistry();
+    WorkflowCostTracker tracker(&reg);
+    tracker.recordActual("w1", "llm", 1000, 100);
+    tracker.recordActual("w2", "slm", 1000, 100);
+    auto s = tracker.summarize();
+    CHECK(s.costByWorker.count("llm") == 1, "missing llm worker bucket");
+    CHECK(s.costByWorker.count("slm") == 1, "missing slm worker bucket");
+    PASS();
+}
+
+void test_summary_cost_by_profile_present() {
+    TEST(summary_cost_by_profile_present);
+    auto reg = makeRegistry();
+    WorkflowCostTracker tracker(&reg);
+    tracker.recordActual("w1", "llm", 1000, 100);
+    auto s = tracker.summarize();
+    CHECK(s.costByProfile.count("claude-sonnet") == 1, "missing sonnet profile bucket");
+    PASS();
+}
+
+void test_variance_positive_when_actual_exceeds_estimate() {
+    TEST(variance_positive_when_actual_exceeds_estimate);
+    auto reg = makeRegistry();
+    WorkflowCostTracker tracker(&reg);
+    tracker.recordEstimate("w1", "llm", 100, 50);
+    tracker.recordActual("w1", "llm", 300, 200);
+    auto s = tracker.summarize();
+    CHECK(s.varianceCost() > 0.0, "expected positive variance");
+    PASS();
+}
+
+void test_report_json_contains_core_fields() {
+    TEST(report_json_contains_core_fields);
+    auto reg = makeRegistry();
+    WorkflowCostTracker tracker(&reg);
+    tracker.recordEstimate("w1", "llm", 100, 50);
+    tracker.recordActual("w1", "llm", 100, 50);
+    auto j = tracker.reportJson();
+    CHECK(j.contains("recordCount"), "missing recordCount");
+    CHECK(j.contains("estimatedCost"), "missing estimatedCost");
+    CHECK(j.contains("actualCost"), "missing actualCost");
+    CHECK(j.contains("records"), "missing records");
+    PASS();
+}
+
+void test_tracker_without_registry_uses_zero_costs() {
+    TEST(tracker_without_registry_uses_zero_costs);
+    WorkflowCostTracker tracker(nullptr);
+    tracker.recordEstimate("w1", "llm", 1000, 1000);
+    tracker.recordActual("w1", "llm", 1000, 1000);
+    auto s = tracker.summarize();
+    CHECK(s.estimatedCost == 0.0, "expected zero cost without registry");
+    CHECK(s.actualCost == 0.0, "expected zero cost without registry");
+    PASS();
+}
+
+int main() {
+    std::cout << "Step 426: Cost Tracking + Reporting Tests\n";
+
+    test_record_estimate_creates_record();                // 1
+    test_record_actual_updates_existing_record();         // 2
+    test_record_actual_creates_new_record();              // 3
+    test_profile_resolves_from_worker_mapping();          // 4
+    test_explicit_profile_overrides_mapping();            // 5
+    test_summary_token_totals();                          // 6
+    test_summary_costs_non_negative();                    // 7
+    test_summary_cost_by_worker_present();                // 8
+    test_summary_cost_by_profile_present();               // 9
+    test_variance_positive_when_actual_exceeds_estimate();// 10
+    test_report_json_contains_core_fields();              // 11
+    test_tracker_without_registry_uses_zero_costs();      // 12
+
+    std::cout << "\nResults: " << passed << "/" << (passed + failed)
+              << " passed\n";
+    return failed == 0 ? 0 : 1;
+}
--- a/progress.md
+++ b/progress.md
@@ -4658,6 +4658,50 @@ tokens by grouping requests with shared worker/language/file context.
  - `editor/src/MCPServer.h` (`1940` > `600`)
  - `editor/src/HeadlessAgentRPCHandler.h` (`2768` > `600`)

+### Step 426: Cost Tracking + Reporting
+**Status:** PASS (12/12 tests)
+
+Added workflow cost-accounting primitives to track estimated vs actual token
+usage and compute cost rollups by worker/profile for model-dispatch reporting.
+
+**Files created:**
+- `editor/src/WorkflowCostTracker.h` — cost tracking support:
+  - per-item `CostRecord` (estimated + actual token usage)
+  - model-profile resolution through `ModelProfileRegistry`
+  - estimate/actual recording API
+  - aggregated `CostSummary` (token totals, cost totals, variance)
+  - cost bucket rollups by worker and profile
+  - JSON report export
+- `editor/tests/step426_test.cpp` — 12 tests covering:
+  1. estimate record creation
+  2. actual record update path
+  3. actual-only record creation
+  4. worker-to-profile resolution
+  5. explicit profile override
+  6. summary token totals
+  7. non-negative cost totals
+  8. cost-by-worker buckets
+  9. cost-by-profile buckets
+  10. positive variance when actual > estimate
+  11. report JSON core fields
+  12. no-registry zero-cost edge case
+
+**Files modified:**
+- `editor/CMakeLists.txt` — `step426_test` target
+
+**Verification run:**
+- `step426_test` — PASS (12/12) new step coverage
+- `step425_test` — PASS (12/12) regression coverage
+- `step424_test` — PASS (12/12) regression coverage
+
+**Architecture gate check:**
+- `editor/src/WorkflowCostTracker.h` within header-size limit (`150` <= `600`)
+- `editor/tests/step426_test.cpp` within test-file size guidance (`174` lines)
+- Legacy oversized headers persist:
+  - `editor/src/ast/Serialization.h` (`1427` > `600`)
+  - `editor/src/MCPServer.h` (`1940` > `600`)
+  - `editor/src/HeadlessAgentRPCHandler.h` (`2768` > `600`)
+
 # Roadmap Planning — Sprints 12-25+

 ## Status: Planning Complete (Sprints 12-19 detailed, 20-25 in roadmap.md)