diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index e11d939..8074a6c 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -2713,4 +2713,13 @@ target_link_libraries(step425_test PRIVATE tree_sitter_javascript tree_sitter_typescript tree_sitter_java tree_sitter_rust tree_sitter_go) +add_executable(step426_test tests/step426_test.cpp) +target_include_directories(step426_test PRIVATE src) +target_link_libraries(step426_test PRIVATE + nlohmann_json::nlohmann_json + unofficial::tree-sitter::tree-sitter + tree_sitter_python tree_sitter_cpp tree_sitter_elisp + tree_sitter_javascript tree_sitter_typescript + tree_sitter_java tree_sitter_rust tree_sitter_go) + # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies) diff --git a/editor/src/WorkflowCostTracker.h b/editor/src/WorkflowCostTracker.h new file mode 100644 index 0000000..1a032c1 --- /dev/null +++ b/editor/src/WorkflowCostTracker.h @@ -0,0 +1,150 @@ +#pragma once + +#include "ModelProfileRegistry.h" + +#include +#include +#include + +#include +using json = nlohmann::json; + +struct CostRecord { + std::string itemId; + std::string workerType; // slm|llm|deterministic|template|human + std::string profileName; // resolved model profile name + int estimatedInputTokens = 0; + int estimatedOutputTokens = 0; + int actualInputTokens = 0; + int actualOutputTokens = 0; +}; + +struct CostSummary { + int estimatedInputTokens = 0; + int estimatedOutputTokens = 0; + int actualInputTokens = 0; + int actualOutputTokens = 0; + double estimatedCost = 0.0; + double actualCost = 0.0; + std::map costByWorker; + std::map costByProfile; + + double varianceCost() const { return actualCost - estimatedCost; } +}; + +class WorkflowCostTracker { +public: + explicit WorkflowCostTracker(const ModelProfileRegistry* registry = nullptr) + : registry_(registry) {} + + void setRegistry(const ModelProfileRegistry* registry) { + registry_ = registry; + } + + void recordEstimate(const std::string& itemId, + const std::string& workerType, + int inputTokens, + int outputTokens) { + CostRecord& rec = records_[itemId]; + rec.itemId = itemId; + rec.workerType = workerType; + rec.estimatedInputTokens = inputTokens; + rec.estimatedOutputTokens = outputTokens; + if (rec.profileName.empty()) rec.profileName = resolveProfile(workerType); + } + + void recordActual(const std::string& itemId, + const std::string& workerType, + int inputTokens, + int outputTokens, + const std::string& explicitProfile = "") { + CostRecord& rec = records_[itemId]; + rec.itemId = itemId; + rec.workerType = workerType; + rec.actualInputTokens = inputTokens; + rec.actualOutputTokens = outputTokens; + rec.profileName = !explicitProfile.empty() ? explicitProfile : resolveProfile(workerType); + } + + bool hasRecord(const std::string& itemId) const { + return records_.find(itemId) != records_.end(); + } + + CostRecord getRecord(const std::string& itemId) const { + auto it = records_.find(itemId); + if (it == records_.end()) return {}; + return it->second; + } + + size_t count() const { return records_.size(); } + + CostSummary summarize() const { + CostSummary out; + for (const auto& [_, rec] : records_) { + out.estimatedInputTokens += rec.estimatedInputTokens; + out.estimatedOutputTokens += rec.estimatedOutputTokens; + out.actualInputTokens += rec.actualInputTokens; + out.actualOutputTokens += rec.actualOutputTokens; + + double est = estimateCost(rec.profileName, rec.estimatedInputTokens, + rec.estimatedOutputTokens); + double act = estimateCost(rec.profileName, rec.actualInputTokens, + rec.actualOutputTokens); + out.estimatedCost += est; + out.actualCost += act; + out.costByWorker[rec.workerType] += act; + out.costByProfile[rec.profileName] += act; + } + return out; + } + + json reportJson() const { + auto s = summarize(); + json recs = json::array(); + for (const auto& [_, rec] : records_) { + recs.push_back({ + {"itemId", rec.itemId}, + {"workerType", rec.workerType}, + {"profileName", rec.profileName}, + {"estimatedInputTokens", rec.estimatedInputTokens}, + {"estimatedOutputTokens", rec.estimatedOutputTokens}, + {"actualInputTokens", rec.actualInputTokens}, + {"actualOutputTokens", rec.actualOutputTokens} + }); + } + return { + {"recordCount", (int)records_.size()}, + {"estimatedInputTokens", s.estimatedInputTokens}, + {"estimatedOutputTokens", s.estimatedOutputTokens}, + {"actualInputTokens", s.actualInputTokens}, + {"actualOutputTokens", s.actualOutputTokens}, + {"estimatedCost", s.estimatedCost}, + {"actualCost", s.actualCost}, + {"costVariance", s.varianceCost()}, + {"costByWorker", s.costByWorker}, + {"costByProfile", s.costByProfile}, + {"records", recs} + }; + } + +private: + const ModelProfileRegistry* registry_ = nullptr; + std::map records_; + + std::string resolveProfile(const std::string& workerType) const { + if (!registry_) return "unknown"; + std::string name = registry_->resolveProfileForWorker(workerType); + return name.empty() ? "unknown" : name; + } + + double estimateCost(const std::string& profileName, + int inputTokens, + int outputTokens) const { + if (!registry_ || profileName.empty() || profileName == "unknown") return 0.0; + auto p = registry_->getProfile(profileName); + if (p.name.empty()) return 0.0; + double inCost = (double)inputTokens / 1000.0 * p.costPer1kInput; + double outCost = (double)outputTokens / 1000.0 * p.costPer1kOutput; + return inCost + outCost; + } +}; diff --git a/editor/tests/step426_test.cpp b/editor/tests/step426_test.cpp new file mode 100644 index 0000000..66f7d39 --- /dev/null +++ b/editor/tests/step426_test.cpp @@ -0,0 +1,174 @@ +// Step 426: Cost Tracking + Reporting Tests (12 tests) + +#include "WorkflowCostTracker.h" + +#include + +static int passed = 0, failed = 0; +#define TEST(name) { std::cout << " " << #name << "... "; } +#define PASS() { std::cout << "PASS\n"; ++passed; } +#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; } +#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {} + +static ModelProfileRegistry makeRegistry() { + ModelProfileRegistry reg; + reg.setWorkerProfileMapping("llm", "claude-sonnet"); + reg.setWorkerProfileMapping("slm", "claude-haiku"); + return reg; +} + +void test_record_estimate_creates_record() { + TEST(record_estimate_creates_record); + auto reg = makeRegistry(); + WorkflowCostTracker tracker(®); + tracker.recordEstimate("w1", "llm", 1000, 200); + CHECK(tracker.hasRecord("w1"), "record missing"); + CHECK(tracker.getRecord("w1").estimatedInputTokens == 1000, "estimate not stored"); + PASS(); +} + +void test_record_actual_updates_existing_record() { + TEST(record_actual_updates_existing_record); + auto reg = makeRegistry(); + WorkflowCostTracker tracker(®); + tracker.recordEstimate("w1", "llm", 1000, 200); + tracker.recordActual("w1", "llm", 1200, 250); + auto rec = tracker.getRecord("w1"); + CHECK(rec.actualInputTokens == 1200, "actual input mismatch"); + CHECK(rec.actualOutputTokens == 250, "actual output mismatch"); + PASS(); +} + +void test_record_actual_creates_new_record() { + TEST(record_actual_creates_new_record); + auto reg = makeRegistry(); + WorkflowCostTracker tracker(®); + tracker.recordActual("w2", "slm", 300, 100); + CHECK(tracker.hasRecord("w2"), "actual-only record missing"); + PASS(); +} + +void test_profile_resolves_from_worker_mapping() { + TEST(profile_resolves_from_worker_mapping); + auto reg = makeRegistry(); + WorkflowCostTracker tracker(®); + tracker.recordEstimate("w1", "llm", 10, 10); + CHECK(tracker.getRecord("w1").profileName == "claude-sonnet", + "llm should map to sonnet"); + PASS(); +} + +void test_explicit_profile_overrides_mapping() { + TEST(explicit_profile_overrides_mapping); + auto reg = makeRegistry(); + WorkflowCostTracker tracker(®); + tracker.recordActual("w1", "llm", 100, 100, "claude-opus"); + CHECK(tracker.getRecord("w1").profileName == "claude-opus", + "explicit profile should win"); + PASS(); +} + +void test_summary_token_totals() { + TEST(summary_token_totals); + auto reg = makeRegistry(); + WorkflowCostTracker tracker(®); + tracker.recordEstimate("w1", "llm", 100, 20); + tracker.recordEstimate("w2", "slm", 200, 30); + tracker.recordActual("w1", "llm", 110, 25); + tracker.recordActual("w2", "slm", 180, 28); + auto s = tracker.summarize(); + CHECK(s.estimatedInputTokens == 300, "estimated input sum mismatch"); + CHECK(s.actualOutputTokens == 53, "actual output sum mismatch"); + PASS(); +} + +void test_summary_costs_non_negative() { + TEST(summary_costs_non_negative); + auto reg = makeRegistry(); + WorkflowCostTracker tracker(®); + tracker.recordEstimate("w1", "llm", 1500, 500); + tracker.recordActual("w1", "llm", 1500, 500); + auto s = tracker.summarize(); + CHECK(s.estimatedCost >= 0.0, "estimated cost negative"); + CHECK(s.actualCost >= 0.0, "actual cost negative"); + PASS(); +} + +void test_summary_cost_by_worker_present() { + TEST(summary_cost_by_worker_present); + auto reg = makeRegistry(); + WorkflowCostTracker tracker(®); + tracker.recordActual("w1", "llm", 1000, 100); + tracker.recordActual("w2", "slm", 1000, 100); + auto s = tracker.summarize(); + CHECK(s.costByWorker.count("llm") == 1, "missing llm worker bucket"); + CHECK(s.costByWorker.count("slm") == 1, "missing slm worker bucket"); + PASS(); +} + +void test_summary_cost_by_profile_present() { + TEST(summary_cost_by_profile_present); + auto reg = makeRegistry(); + WorkflowCostTracker tracker(®); + tracker.recordActual("w1", "llm", 1000, 100); + auto s = tracker.summarize(); + CHECK(s.costByProfile.count("claude-sonnet") == 1, "missing sonnet profile bucket"); + PASS(); +} + +void test_variance_positive_when_actual_exceeds_estimate() { + TEST(variance_positive_when_actual_exceeds_estimate); + auto reg = makeRegistry(); + WorkflowCostTracker tracker(®); + tracker.recordEstimate("w1", "llm", 100, 50); + tracker.recordActual("w1", "llm", 300, 200); + auto s = tracker.summarize(); + CHECK(s.varianceCost() > 0.0, "expected positive variance"); + PASS(); +} + +void test_report_json_contains_core_fields() { + TEST(report_json_contains_core_fields); + auto reg = makeRegistry(); + WorkflowCostTracker tracker(®); + tracker.recordEstimate("w1", "llm", 100, 50); + tracker.recordActual("w1", "llm", 100, 50); + auto j = tracker.reportJson(); + CHECK(j.contains("recordCount"), "missing recordCount"); + CHECK(j.contains("estimatedCost"), "missing estimatedCost"); + CHECK(j.contains("actualCost"), "missing actualCost"); + CHECK(j.contains("records"), "missing records"); + PASS(); +} + +void test_tracker_without_registry_uses_zero_costs() { + TEST(tracker_without_registry_uses_zero_costs); + WorkflowCostTracker tracker(nullptr); + tracker.recordEstimate("w1", "llm", 1000, 1000); + tracker.recordActual("w1", "llm", 1000, 1000); + auto s = tracker.summarize(); + CHECK(s.estimatedCost == 0.0, "expected zero cost without registry"); + CHECK(s.actualCost == 0.0, "expected zero cost without registry"); + PASS(); +} + +int main() { + std::cout << "Step 426: Cost Tracking + Reporting Tests\n"; + + test_record_estimate_creates_record(); // 1 + test_record_actual_updates_existing_record(); // 2 + test_record_actual_creates_new_record(); // 3 + test_profile_resolves_from_worker_mapping(); // 4 + test_explicit_profile_overrides_mapping(); // 5 + test_summary_token_totals(); // 6 + test_summary_costs_non_negative(); // 7 + test_summary_cost_by_worker_present(); // 8 + test_summary_cost_by_profile_present(); // 9 + test_variance_positive_when_actual_exceeds_estimate();// 10 + test_report_json_contains_core_fields(); // 11 + test_tracker_without_registry_uses_zero_costs(); // 12 + + std::cout << "\nResults: " << passed << "/" << (passed + failed) + << " passed\n"; + return failed == 0 ? 0 : 1; +} diff --git a/progress.md b/progress.md index 372b0c0..64ffa0e 100644 --- a/progress.md +++ b/progress.md @@ -4658,6 +4658,50 @@ tokens by grouping requests with shared worker/language/file context. - `editor/src/MCPServer.h` (`1940` > `600`) - `editor/src/HeadlessAgentRPCHandler.h` (`2768` > `600`) +### Step 426: Cost Tracking + Reporting +**Status:** PASS (12/12 tests) + +Added workflow cost-accounting primitives to track estimated vs actual token +usage and compute cost rollups by worker/profile for model-dispatch reporting. + +**Files created:** +- `editor/src/WorkflowCostTracker.h` — cost tracking support: + - per-item `CostRecord` (estimated + actual token usage) + - model-profile resolution through `ModelProfileRegistry` + - estimate/actual recording API + - aggregated `CostSummary` (token totals, cost totals, variance) + - cost bucket rollups by worker and profile + - JSON report export +- `editor/tests/step426_test.cpp` — 12 tests covering: + 1. estimate record creation + 2. actual record update path + 3. actual-only record creation + 4. worker-to-profile resolution + 5. explicit profile override + 6. summary token totals + 7. non-negative cost totals + 8. cost-by-worker buckets + 9. cost-by-profile buckets + 10. positive variance when actual > estimate + 11. report JSON core fields + 12. no-registry zero-cost edge case + +**Files modified:** +- `editor/CMakeLists.txt` — `step426_test` target + +**Verification run:** +- `step426_test` — PASS (12/12) new step coverage +- `step425_test` — PASS (12/12) regression coverage +- `step424_test` — PASS (12/12) regression coverage + +**Architecture gate check:** +- `editor/src/WorkflowCostTracker.h` within header-size limit (`150` <= `600`) +- `editor/tests/step426_test.cpp` within test-file size guidance (`174` lines) +- Legacy oversized headers persist: + - `editor/src/ast/Serialization.h` (`1427` > `600`) + - `editor/src/MCPServer.h` (`1940` > `600`) + - `editor/src/HeadlessAgentRPCHandler.h` (`2768` > `600`) + # Roadmap Planning — Sprints 12-25+ ## Status: Planning Complete (Sprints 12-19 detailed, 20-25 in roadmap.md)