Step 427: add Sprint 18b integration suite

2026-02-16 17:04:12 -07:00
parent fc950798f0
commit dd9c019e85
3 changed files with 228 additions and 0 deletions
--- a/editor/CMakeLists.txt
+++ b/editor/CMakeLists.txt
@@ -2722,4 +2722,13 @@ target_link_libraries(step426_test PRIVATE
    tree_sitter_javascript tree_sitter_typescript
    tree_sitter_java tree_sitter_rust tree_sitter_go)

+add_executable(step427_test tests/step427_test.cpp)
+target_include_directories(step427_test PRIVATE src)
+target_link_libraries(step427_test PRIVATE
+    nlohmann_json::nlohmann_json
+    unofficial::tree-sitter::tree-sitter
+    tree_sitter_python tree_sitter_cpp tree_sitter_elisp
+    tree_sitter_javascript tree_sitter_typescript
+    tree_sitter_java tree_sitter_rust tree_sitter_go)
+
 # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies)
--- a/editor/tests/step427_test.cpp
+++ b/editor/tests/step427_test.cpp
@@ -0,0 +1,164 @@
+// Step 427: Phase 18b Integration + Sprint Summary Tests (8 tests)
+
+#include "ModelProfileRegistry.h"
+#include "ContextWindowOptimizer.h"
+#include "BatchTaskSubmitter.h"
+#include "WorkflowCostTracker.h"
+
+#include <iostream>
+#include <string>
+#include <vector>
+
+static int passed = 0, failed = 0;
+#define TEST(name) { std::cout << "  " << #name << "... "; }
+#define PASS() { std::cout << "PASS\n"; ++passed; }
+#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; }
+#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {}
+
+static AgentTaskRequest makeTask(const std::string& id,
+                                 const std::string& worker,
+                                 const std::string& file,
+                                 const std::string& prompt,
+                                 const std::string& ctx) {
+    AgentTaskRequest t;
+    t.itemId = id;
+    t.workerType = worker;
+    t.language = file.find(".py") != std::string::npos ? "python" : "cpp";
+    t.bufferId = file;
+    t.prompt = prompt;
+    t.context = ctx;
+    return t;
+}
+
+void test_phase18b_profile_context_batch_cost_pipeline() {
+    TEST(phase18b_profile_context_batch_cost_pipeline);
+    ModelProfileRegistry reg;
+    ContextPack pack;
+    pack.instructions = "Implement safely.";
+    pack.localContext = "fn local(){}";
+    pack.fileContext = std::string(5000, 'f');
+    pack.projectContext = std::string(20000, 'p');
+    auto optimized = ContextWindowOptimizer::optimizeForWorker(pack, "llm", reg);
+    CHECK(!optimized.context.empty(), "optimized context missing");
+
+    std::vector<AgentTaskRequest> tasks = {
+        makeTask("w1", "llm", "a.cpp", "rewrite A", optimized.context),
+        makeTask("w2", "llm", "a.cpp", "rewrite B", optimized.context)
+    };
+    auto plan = BatchTaskSubmitter::planBatches(tasks, 4);
+    CHECK(plan.batches.size() == 1, "expected one batch");
+
+    WorkflowCostTracker costs(&reg);
+    costs.recordEstimate("w1", "llm", 1200, 300);
+    costs.recordEstimate("w2", "llm", 1100, 280);
+    costs.recordActual("w1", "llm", 1250, 320);
+    costs.recordActual("w2", "llm", 1000, 260);
+    auto summary = costs.summarize();
+    CHECK(summary.actualCost >= 0.0, "invalid actual cost");
+    PASS();
+}
+
+void test_profile_mapping_affects_context_scope_choice() {
+    TEST(profile_mapping_affects_context_scope_choice);
+    ModelProfileRegistry reg;
+    reg.setWorkerProfileMapping("llm", "local-slm");
+    ContextPack pack;
+    pack.instructions = "Do task";
+    pack.localContext = "local";
+    pack.fileContext = std::string(2000, 'f');
+    pack.projectContext = std::string(10000, 'p');
+    auto out = ContextWindowOptimizer::optimizeForWorker(pack, "llm", reg);
+    CHECK(out.scope == "local", "remapped llm should choose local scope");
+    PASS();
+}
+
+void test_batching_reduces_estimated_tokens_for_shared_context() {
+    TEST(batching_reduces_estimated_tokens_for_shared_context);
+    std::string shared = std::string(8000, 'x');
+    std::vector<AgentTaskRequest> tasks = {
+        makeTask("w1", "llm", "a.cpp", "task A", shared),
+        makeTask("w2", "llm", "a.cpp", "task B", shared),
+        makeTask("w3", "llm", "a.cpp", "task C", shared)
+    };
+    auto plan = BatchTaskSubmitter::planBatches(tasks, 8);
+    CHECK(plan.estimatedTokensBatched < plan.estimatedTokensIndividual,
+          "expected batching token savings");
+    PASS();
+}
+
+void test_cost_tracker_report_has_worker_and_profile_breakdown() {
+    TEST(cost_tracker_report_has_worker_and_profile_breakdown);
+    ModelProfileRegistry reg;
+    WorkflowCostTracker costs(&reg);
+    costs.recordActual("a", "llm", 1000, 100);
+    costs.recordActual("b", "slm", 1000, 100);
+    auto report = costs.reportJson();
+    CHECK(report["costByWorker"].contains("llm"), "missing llm worker bucket");
+    CHECK(report["costByProfile"].contains("claude-sonnet"), "missing sonnet profile bucket");
+    PASS();
+}
+
+void test_context_optimizer_budget_alignment_with_profile() {
+    TEST(context_optimizer_budget_alignment_with_profile);
+    ContextPack pack;
+    pack.instructions = "i";
+    pack.localContext = std::string(5000, 'l');
+    ModelProfile small{"small", 8000, 0, 0, "fast", {}};
+    ModelProfile large{"large", 200000, 0, 0, "slow", {}};
+    auto a = ContextWindowOptimizer::optimize(pack, small);
+    auto b = ContextWindowOptimizer::optimize(pack, large);
+    CHECK(a.budgetTokens < b.budgetTokens, "small model should have smaller budget");
+    PASS();
+}
+
+void test_cost_variance_tracks_over_under_runs() {
+    TEST(cost_variance_tracks_over_under_runs);
+    ModelProfileRegistry reg;
+    WorkflowCostTracker costs(&reg);
+    costs.recordEstimate("w1", "llm", 1000, 100);
+    costs.recordActual("w1", "llm", 1200, 150);
+    costs.recordEstimate("w2", "llm", 1000, 100);
+    costs.recordActual("w2", "llm", 800, 80);
+    auto s = costs.summarize();
+    CHECK(s.varianceCost() != 0.0, "variance should capture estimate drift");
+    PASS();
+}
+
+void test_multigroup_batch_plan_is_deterministic() {
+    TEST(multigroup_batch_plan_is_deterministic);
+    std::vector<AgentTaskRequest> tasks = {
+        makeTask("w3", "slm", "b.py", "p3", "ctx-b"),
+        makeTask("w1", "llm", "a.cpp", "p1", "ctx-a"),
+        makeTask("w2", "llm", "a.cpp", "p2", "ctx-a")
+    };
+    auto p1 = BatchTaskSubmitter::planBatches(tasks, 8);
+    auto p2 = BatchTaskSubmitter::planBatches(tasks, 8);
+    CHECK(p1.batches.size() == p2.batches.size(), "batch count should be deterministic");
+    CHECK(p1.batches[0].workerType == p2.batches[0].workerType, "batch ordering drift");
+    PASS();
+}
+
+void test_phase18b_expected_defaults_present() {
+    TEST(phase18b_expected_defaults_present);
+    ModelProfileRegistry reg;
+    CHECK(reg.resolveProfileForWorker("llm") == "claude-sonnet", "llm default mapping mismatch");
+    CHECK(reg.resolveProfileForWorker("slm") == "claude-haiku", "slm default mapping mismatch");
+    PASS();
+}
+
+int main() {
+    std::cout << "Step 427: Phase 18b Integration + Sprint Summary Tests\n";
+
+    test_phase18b_profile_context_batch_cost_pipeline();         // 1
+    test_profile_mapping_affects_context_scope_choice();         // 2
+    test_batching_reduces_estimated_tokens_for_shared_context(); // 3
+    test_cost_tracker_report_has_worker_and_profile_breakdown(); // 4
+    test_context_optimizer_budget_alignment_with_profile();       // 5
+    test_cost_variance_tracks_over_under_runs();                 // 6
+    test_multigroup_batch_plan_is_deterministic();               // 7
+    test_phase18b_expected_defaults_present();                   // 8
+
+    std::cout << "\nResults: " << passed << "/" << (passed + failed)
+              << " passed\n";
+    return failed == 0 ? 0 : 1;
+}
--- a/progress.md
+++ b/progress.md
@@ -4702,6 +4702,61 @@ usage and compute cost rollups by worker/profile for model-dispatch reporting.
  - `editor/src/MCPServer.h` (`1940` > `600`)
  - `editor/src/HeadlessAgentRPCHandler.h` (`2768` > `600`)

+### Step 427: Phase 18b Integration + Sprint Summary
+**Status:** PASS (8/8 tests)
+
+Added Phase 18b integration coverage validating that model-profile selection,
+context optimization, batch planning, and cost accounting compose into a
+coherent dispatch pipeline.
+
+**Files created:**
+- `editor/tests/step427_test.cpp` — 8 integration tests covering:
+  1. profile -> context -> batch -> cost pipeline composition
+  2. profile mapping impact on context scope selection
+  3. batching token-savings behavior
+  4. cost report worker/profile breakdowns
+  5. budget alignment across profile sizes
+  6. cost variance tracking for estimate drift
+  7. deterministic multi-group batching
+  8. expected default worker/profile mappings
+
+**Files modified:**
+- `editor/CMakeLists.txt` — `step427_test` target
+
+**Verification run:**
+- `step427_test` — PASS (8/8) new phase-integration coverage
+- `step426_test` — PASS (12/12) regression coverage
+- `step425_test` — PASS (12/12) regression coverage
+
+**Phase 18b completion snapshot (Steps 423-427):**
+- Step 423: model profile registry + configurable worker mapping
+- Step 424: context-window optimization by model budget
+- Step 425: batch task submission planning + savings estimates
+- Step 426: token/cost tracking and reporting
+- Step 427: end-to-end integration across 18b components
+
+**Sprint 18 completion snapshot (Steps 417-427):**
+- Phase 18a complete (Steps 417-422, 68 tests)
+- Phase 18b complete (Steps 423-427, 56 tests)
+- Sprint 18 total: 11 steps, 124 tests
+
+**Architecture gate check (end of Sprint 18):**
+- New Sprint 18b headers within header-size limit:
+  - `editor/src/ModelProfileRegistry.h` (`172` <= `600`)
+  - `editor/src/ContextWindowOptimizer.h` (`115` <= `600`)
+  - `editor/src/BatchTaskSubmitter.h` (`123` <= `600`)
+  - `editor/src/WorkflowCostTracker.h` (`150` <= `600`)
+- New Sprint 18b tests within file-size guidance:
+  - `editor/tests/step423_test.cpp` (`169` lines)
+  - `editor/tests/step424_test.cpp` (`156` lines)
+  - `editor/tests/step425_test.cpp` (`194` lines)
+  - `editor/tests/step426_test.cpp` (`174` lines)
+  - `editor/tests/step427_test.cpp` (`164` lines)
+- Legacy oversized headers persist:
+  - `editor/src/ast/Serialization.h` (`1427` > `600`)
+  - `editor/src/MCPServer.h` (`1940` > `600`)
+  - `editor/src/HeadlessAgentRPCHandler.h` (`2768` > `600`)
+
 # Roadmap Planning — Sprints 12-25+

 ## Status: Planning Complete (Sprints 12-19 detailed, 20-25 in roadmap.md)