From dd9c019e859953de6ee1d9028d00bad640c4d750 Mon Sep 17 00:00:00 2001 From: Bill Date: Mon, 16 Feb 2026 17:04:12 -0700 Subject: [PATCH] Step 427: add Sprint 18b integration suite --- editor/CMakeLists.txt | 9 ++ editor/tests/step427_test.cpp | 164 ++++++++++++++++++++++++++++++++++ progress.md | 55 ++++++++++++ 3 files changed, 228 insertions(+) create mode 100644 editor/tests/step427_test.cpp diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 8074a6c..451f7e5 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -2722,4 +2722,13 @@ target_link_libraries(step426_test PRIVATE tree_sitter_javascript tree_sitter_typescript tree_sitter_java tree_sitter_rust tree_sitter_go) +add_executable(step427_test tests/step427_test.cpp) +target_include_directories(step427_test PRIVATE src) +target_link_libraries(step427_test PRIVATE + nlohmann_json::nlohmann_json + unofficial::tree-sitter::tree-sitter + tree_sitter_python tree_sitter_cpp tree_sitter_elisp + tree_sitter_javascript tree_sitter_typescript + tree_sitter_java tree_sitter_rust tree_sitter_go) + # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies) diff --git a/editor/tests/step427_test.cpp b/editor/tests/step427_test.cpp new file mode 100644 index 0000000..ba810ac --- /dev/null +++ b/editor/tests/step427_test.cpp @@ -0,0 +1,164 @@ +// Step 427: Phase 18b Integration + Sprint Summary Tests (8 tests) + +#include "ModelProfileRegistry.h" +#include "ContextWindowOptimizer.h" +#include "BatchTaskSubmitter.h" +#include "WorkflowCostTracker.h" + +#include +#include +#include + +static int passed = 0, failed = 0; +#define TEST(name) { std::cout << " " << #name << "... "; } +#define PASS() { std::cout << "PASS\n"; ++passed; } +#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; } +#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {} + +static AgentTaskRequest makeTask(const std::string& id, + const std::string& worker, + const std::string& file, + const std::string& prompt, + const std::string& ctx) { + AgentTaskRequest t; + t.itemId = id; + t.workerType = worker; + t.language = file.find(".py") != std::string::npos ? "python" : "cpp"; + t.bufferId = file; + t.prompt = prompt; + t.context = ctx; + return t; +} + +void test_phase18b_profile_context_batch_cost_pipeline() { + TEST(phase18b_profile_context_batch_cost_pipeline); + ModelProfileRegistry reg; + ContextPack pack; + pack.instructions = "Implement safely."; + pack.localContext = "fn local(){}"; + pack.fileContext = std::string(5000, 'f'); + pack.projectContext = std::string(20000, 'p'); + auto optimized = ContextWindowOptimizer::optimizeForWorker(pack, "llm", reg); + CHECK(!optimized.context.empty(), "optimized context missing"); + + std::vector tasks = { + makeTask("w1", "llm", "a.cpp", "rewrite A", optimized.context), + makeTask("w2", "llm", "a.cpp", "rewrite B", optimized.context) + }; + auto plan = BatchTaskSubmitter::planBatches(tasks, 4); + CHECK(plan.batches.size() == 1, "expected one batch"); + + WorkflowCostTracker costs(®); + costs.recordEstimate("w1", "llm", 1200, 300); + costs.recordEstimate("w2", "llm", 1100, 280); + costs.recordActual("w1", "llm", 1250, 320); + costs.recordActual("w2", "llm", 1000, 260); + auto summary = costs.summarize(); + CHECK(summary.actualCost >= 0.0, "invalid actual cost"); + PASS(); +} + +void test_profile_mapping_affects_context_scope_choice() { + TEST(profile_mapping_affects_context_scope_choice); + ModelProfileRegistry reg; + reg.setWorkerProfileMapping("llm", "local-slm"); + ContextPack pack; + pack.instructions = "Do task"; + pack.localContext = "local"; + pack.fileContext = std::string(2000, 'f'); + pack.projectContext = std::string(10000, 'p'); + auto out = ContextWindowOptimizer::optimizeForWorker(pack, "llm", reg); + CHECK(out.scope == "local", "remapped llm should choose local scope"); + PASS(); +} + +void test_batching_reduces_estimated_tokens_for_shared_context() { + TEST(batching_reduces_estimated_tokens_for_shared_context); + std::string shared = std::string(8000, 'x'); + std::vector tasks = { + makeTask("w1", "llm", "a.cpp", "task A", shared), + makeTask("w2", "llm", "a.cpp", "task B", shared), + makeTask("w3", "llm", "a.cpp", "task C", shared) + }; + auto plan = BatchTaskSubmitter::planBatches(tasks, 8); + CHECK(plan.estimatedTokensBatched < plan.estimatedTokensIndividual, + "expected batching token savings"); + PASS(); +} + +void test_cost_tracker_report_has_worker_and_profile_breakdown() { + TEST(cost_tracker_report_has_worker_and_profile_breakdown); + ModelProfileRegistry reg; + WorkflowCostTracker costs(®); + costs.recordActual("a", "llm", 1000, 100); + costs.recordActual("b", "slm", 1000, 100); + auto report = costs.reportJson(); + CHECK(report["costByWorker"].contains("llm"), "missing llm worker bucket"); + CHECK(report["costByProfile"].contains("claude-sonnet"), "missing sonnet profile bucket"); + PASS(); +} + +void test_context_optimizer_budget_alignment_with_profile() { + TEST(context_optimizer_budget_alignment_with_profile); + ContextPack pack; + pack.instructions = "i"; + pack.localContext = std::string(5000, 'l'); + ModelProfile small{"small", 8000, 0, 0, "fast", {}}; + ModelProfile large{"large", 200000, 0, 0, "slow", {}}; + auto a = ContextWindowOptimizer::optimize(pack, small); + auto b = ContextWindowOptimizer::optimize(pack, large); + CHECK(a.budgetTokens < b.budgetTokens, "small model should have smaller budget"); + PASS(); +} + +void test_cost_variance_tracks_over_under_runs() { + TEST(cost_variance_tracks_over_under_runs); + ModelProfileRegistry reg; + WorkflowCostTracker costs(®); + costs.recordEstimate("w1", "llm", 1000, 100); + costs.recordActual("w1", "llm", 1200, 150); + costs.recordEstimate("w2", "llm", 1000, 100); + costs.recordActual("w2", "llm", 800, 80); + auto s = costs.summarize(); + CHECK(s.varianceCost() != 0.0, "variance should capture estimate drift"); + PASS(); +} + +void test_multigroup_batch_plan_is_deterministic() { + TEST(multigroup_batch_plan_is_deterministic); + std::vector tasks = { + makeTask("w3", "slm", "b.py", "p3", "ctx-b"), + makeTask("w1", "llm", "a.cpp", "p1", "ctx-a"), + makeTask("w2", "llm", "a.cpp", "p2", "ctx-a") + }; + auto p1 = BatchTaskSubmitter::planBatches(tasks, 8); + auto p2 = BatchTaskSubmitter::planBatches(tasks, 8); + CHECK(p1.batches.size() == p2.batches.size(), "batch count should be deterministic"); + CHECK(p1.batches[0].workerType == p2.batches[0].workerType, "batch ordering drift"); + PASS(); +} + +void test_phase18b_expected_defaults_present() { + TEST(phase18b_expected_defaults_present); + ModelProfileRegistry reg; + CHECK(reg.resolveProfileForWorker("llm") == "claude-sonnet", "llm default mapping mismatch"); + CHECK(reg.resolveProfileForWorker("slm") == "claude-haiku", "slm default mapping mismatch"); + PASS(); +} + +int main() { + std::cout << "Step 427: Phase 18b Integration + Sprint Summary Tests\n"; + + test_phase18b_profile_context_batch_cost_pipeline(); // 1 + test_profile_mapping_affects_context_scope_choice(); // 2 + test_batching_reduces_estimated_tokens_for_shared_context(); // 3 + test_cost_tracker_report_has_worker_and_profile_breakdown(); // 4 + test_context_optimizer_budget_alignment_with_profile(); // 5 + test_cost_variance_tracks_over_under_runs(); // 6 + test_multigroup_batch_plan_is_deterministic(); // 7 + test_phase18b_expected_defaults_present(); // 8 + + std::cout << "\nResults: " << passed << "/" << (passed + failed) + << " passed\n"; + return failed == 0 ? 0 : 1; +} diff --git a/progress.md b/progress.md index 64ffa0e..9e236b0 100644 --- a/progress.md +++ b/progress.md @@ -4702,6 +4702,61 @@ usage and compute cost rollups by worker/profile for model-dispatch reporting. - `editor/src/MCPServer.h` (`1940` > `600`) - `editor/src/HeadlessAgentRPCHandler.h` (`2768` > `600`) +### Step 427: Phase 18b Integration + Sprint Summary +**Status:** PASS (8/8 tests) + +Added Phase 18b integration coverage validating that model-profile selection, +context optimization, batch planning, and cost accounting compose into a +coherent dispatch pipeline. + +**Files created:** +- `editor/tests/step427_test.cpp` — 8 integration tests covering: + 1. profile -> context -> batch -> cost pipeline composition + 2. profile mapping impact on context scope selection + 3. batching token-savings behavior + 4. cost report worker/profile breakdowns + 5. budget alignment across profile sizes + 6. cost variance tracking for estimate drift + 7. deterministic multi-group batching + 8. expected default worker/profile mappings + +**Files modified:** +- `editor/CMakeLists.txt` — `step427_test` target + +**Verification run:** +- `step427_test` — PASS (8/8) new phase-integration coverage +- `step426_test` — PASS (12/12) regression coverage +- `step425_test` — PASS (12/12) regression coverage + +**Phase 18b completion snapshot (Steps 423-427):** +- Step 423: model profile registry + configurable worker mapping +- Step 424: context-window optimization by model budget +- Step 425: batch task submission planning + savings estimates +- Step 426: token/cost tracking and reporting +- Step 427: end-to-end integration across 18b components + +**Sprint 18 completion snapshot (Steps 417-427):** +- Phase 18a complete (Steps 417-422, 68 tests) +- Phase 18b complete (Steps 423-427, 56 tests) +- Sprint 18 total: 11 steps, 124 tests + +**Architecture gate check (end of Sprint 18):** +- New Sprint 18b headers within header-size limit: + - `editor/src/ModelProfileRegistry.h` (`172` <= `600`) + - `editor/src/ContextWindowOptimizer.h` (`115` <= `600`) + - `editor/src/BatchTaskSubmitter.h` (`123` <= `600`) + - `editor/src/WorkflowCostTracker.h` (`150` <= `600`) +- New Sprint 18b tests within file-size guidance: + - `editor/tests/step423_test.cpp` (`169` lines) + - `editor/tests/step424_test.cpp` (`156` lines) + - `editor/tests/step425_test.cpp` (`194` lines) + - `editor/tests/step426_test.cpp` (`174` lines) + - `editor/tests/step427_test.cpp` (`164` lines) +- Legacy oversized headers persist: + - `editor/src/ast/Serialization.h` (`1427` > `600`) + - `editor/src/MCPServer.h` (`1940` > `600`) + - `editor/src/HeadlessAgentRPCHandler.h` (`2768` > `600`) + # Roadmap Planning — Sprints 12-25+ ## Status: Planning Complete (Sprints 12-19 detailed, 20-25 in roadmap.md)