diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 49ea464..f48c32c 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -3397,4 +3397,13 @@ target_link_libraries(step501_test PRIVATE tree_sitter_javascript tree_sitter_typescript tree_sitter_java tree_sitter_rust tree_sitter_go) +add_executable(step502_test tests/step502_test.cpp) +target_include_directories(step502_test PRIVATE src) +target_link_libraries(step502_test PRIVATE + nlohmann_json::nlohmann_json + unofficial::tree-sitter::tree-sitter + tree_sitter_python tree_sitter_cpp tree_sitter_elisp + tree_sitter_javascript tree_sitter_typescript + tree_sitter_java tree_sitter_rust tree_sitter_go) + # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies) diff --git a/editor/src/MultiModelOrchestrationScenarioRunner.h b/editor/src/MultiModelOrchestrationScenarioRunner.h new file mode 100644 index 0000000..f52a597 --- /dev/null +++ b/editor/src/MultiModelOrchestrationScenarioRunner.h @@ -0,0 +1,177 @@ +#pragma once + +// Step 502: Scenario - Multi-Model Orchestration +// End-to-end orchestration scenario with deterministic/template/slm/llm/human workers. + +#include "RoutingEngine.h" + +#include +#include +#include + +enum class MultiModelTaskGroup { Deterministic, Template, SLM, LLM, Human }; + +struct MultiModelTaskResult { + std::string id; + std::string name; + MultiModelTaskGroup group = MultiModelTaskGroup::SLM; + WorkItem item; + RoutingDecision route; + bool autoCompleted = false; + bool completed = false; + bool reviewRequired = false; + int estimatedTokens = 0; + int actualTokens = 0; +}; + +struct MultiModelProgressPoint { + int completed = 0; + int total = 0; + float percent = 0.0f; +}; + +struct MultiModelOrchestrationScenarioResult { + std::vector tasks; + std::vector progress; + std::map routedCounts; + int estimatedTotalTokens = 0; + int actualTotalTokens = 0; + int reviewGateCount = 0; + bool reviewGatesExercised = false; + std::vector notes; + + int countGroup(MultiModelTaskGroup g) const { + int n = 0; + for (const auto& t : tasks) if (t.group == g) ++n; + return n; + } +}; + +class MultiModelOrchestrationScenarioRunner { +public: + static MultiModelOrchestrationScenarioResult run() { + MultiModelOrchestrationScenarioResult out; + RoutingEngine routing; + + auto items = buildScenarioItems(); + const int total = static_cast(items.size()); + int completed = 0; + + for (auto& task : items) { + task.route = routing.route(task.item); + task.estimatedTokens = task.route.contextBudgetTokens + estimatedOutputTokens(task.route.workerType); + out.estimatedTotalTokens += task.estimatedTokens; + out.routedCounts[task.route.workerType]++; + + executeTask(task, out.reviewGateCount); + out.actualTotalTokens += task.actualTokens; + if (task.completed) ++completed; + out.progress.push_back({completed, total, percent(completed, total)}); + } + + out.tasks = std::move(items); + out.reviewGatesExercised = out.reviewGateCount > 0; + out.notes.push_back("Multi-model orchestration scenario executed with 20 tasks"); + out.notes.push_back("Routing mix includes deterministic/template/slm/llm/human paths"); + return out; + } + +private: + static std::vector buildScenarioItems() { + std::vector out; + int idx = 0; + auto makeId = [&idx]() { return "mm-" + std::to_string(++idx); }; + + // 5 deterministic tasks + out.push_back(makeTask(makeId(), "normalize_id", MultiModelTaskGroup::Deterministic, "deterministic", "local", false)); + out.push_back(makeTask(makeId(), "sanitize_email", MultiModelTaskGroup::Deterministic, "deterministic", "local", false)); + out.push_back(makeTask(makeId(), "hash_password", MultiModelTaskGroup::Deterministic, "deterministic", "local", false)); + out.push_back(makeTask(makeId(), "build_slug", MultiModelTaskGroup::Deterministic, "deterministic", "local", false)); + out.push_back(makeTask(makeId(), "validate_url", MultiModelTaskGroup::Deterministic, "deterministic", "local", false)); + + // 5 template tasks (getter/setter/constructor style) + out.push_back(makeTask(makeId(), "get_user", MultiModelTaskGroup::Template, "", "local", false)); + out.push_back(makeTask(makeId(), "set_user", MultiModelTaskGroup::Template, "", "local", false)); + out.push_back(makeTask(makeId(), "get_short_link", MultiModelTaskGroup::Template, "", "local", false)); + out.push_back(makeTask(makeId(), "set_short_link", MultiModelTaskGroup::Template, "", "local", false)); + out.push_back(makeTask(makeId(), "is_active", MultiModelTaskGroup::Template, "", "local", false)); + + // 5 SLM tasks (narrow context) + out.push_back(makeTask(makeId(), "apply_rate_limit", MultiModelTaskGroup::SLM, "", "local", false)); + out.push_back(makeTask(makeId(), "merge_query_filters", MultiModelTaskGroup::SLM, "", "file", false)); + out.push_back(makeTask(makeId(), "serialize_session", MultiModelTaskGroup::SLM, "", "local", false)); + out.push_back(makeTask(makeId(), "parse_feature_flags", MultiModelTaskGroup::SLM, "", "file", false)); + out.push_back(makeTask(makeId(), "compute_expiry", MultiModelTaskGroup::SLM, "", "local", false)); + + // 3 LLM tasks (wide context + review gate) + out.push_back(makeTask(makeId(), "cross_module_cache_invalidation", MultiModelTaskGroup::LLM, "", "project", true)); + out.push_back(makeTask(makeId(), "multi_region_failover_strategy", MultiModelTaskGroup::LLM, "", "cross-project", true)); + out.push_back(makeTask(makeId(), "consistency_resolution_policy", MultiModelTaskGroup::LLM, "", "project", true)); + + // 2 human tasks (architectural decisions) + out.push_back(makeTask(makeId(), "select_sharding_strategy", MultiModelTaskGroup::Human, "human", "cross-project", true)); + out.push_back(makeTask(makeId(), "approve_security_boundary_model", MultiModelTaskGroup::Human, "human", "project", true)); + + return out; + } + + static MultiModelTaskResult makeTask(const std::string& id, + const std::string& name, + MultiModelTaskGroup group, + const std::string& workerType, + const std::string& contextWidth, + bool reviewRequired) { + MultiModelTaskResult t; + t.id = id; + t.name = name; + t.group = group; + t.item.id = id; + t.item.nodeName = name; + t.item.nodeType = "Function"; + t.item.bufferId = "scenario502"; + t.item.workerType = workerType; + t.item.contextWidth = contextWidth; + t.item.reviewRequired = reviewRequired; + t.item.status = WI_READY; + return t; + } + + static void executeTask(MultiModelTaskResult& t, int& reviewGateCount) { + t.reviewRequired = t.route.reviewRequired; + if (t.route.workerType == "deterministic" || t.route.workerType == "template") { + t.autoCompleted = true; + t.completed = true; + t.actualTokens = 0; + return; + } + + if (t.route.workerType == "slm") { + t.completed = true; + t.actualTokens = t.route.contextBudgetTokens / 3; + return; + } + + if (t.route.workerType == "llm") { + t.completed = true; + t.actualTokens = static_cast(t.route.contextBudgetTokens * 0.85f); + if (t.reviewRequired) ++reviewGateCount; + return; + } + + // Human path + t.completed = true; + t.actualTokens = 0; + ++reviewGateCount; + } + + static int estimatedOutputTokens(const std::string& workerType) { + if (workerType == "deterministic" || workerType == "template" || workerType == "human") return 0; + if (workerType == "slm") return 250; + return 900; + } + + static float percent(int done, int total) { + if (total <= 0) return 0.0f; + return 100.0f * static_cast(done) / static_cast(total); + } +}; diff --git a/editor/tests/step502_test.cpp b/editor/tests/step502_test.cpp new file mode 100644 index 0000000..875ee59 --- /dev/null +++ b/editor/tests/step502_test.cpp @@ -0,0 +1,152 @@ +// Step 502: Scenario - Multi-Model Orchestration Tests (12 tests) + +#include "MultiModelOrchestrationScenarioRunner.h" + +#include + +static int passed = 0, failed = 0; +#define TEST(name) { std::cout << " " << #name << "... "; } +#define PASS() { std::cout << "PASS\n"; ++passed; } +#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; } +#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {} + +static MultiModelOrchestrationScenarioResult runNow() { + return MultiModelOrchestrationScenarioRunner::run(); +} + +void test_scenario_builds_exactly_twenty_tasks() { + TEST(scenario_builds_exactly_twenty_tasks); + auto r = runNow(); + CHECK(r.tasks.size() == 20, "expected 20 tasks"); + PASS(); +} + +void test_task_group_distribution_matches_spec() { + TEST(task_group_distribution_matches_spec); + auto r = runNow(); + CHECK(r.countGroup(MultiModelTaskGroup::Deterministic) == 5, "expected 5 deterministic"); + CHECK(r.countGroup(MultiModelTaskGroup::Template) == 5, "expected 5 template"); + CHECK(r.countGroup(MultiModelTaskGroup::SLM) == 5, "expected 5 slm"); + CHECK(r.countGroup(MultiModelTaskGroup::LLM) == 3, "expected 3 llm"); + CHECK(r.countGroup(MultiModelTaskGroup::Human) == 2, "expected 2 human"); + PASS(); +} + +void test_routed_worker_distribution_exercises_all_worker_paths() { + TEST(routed_worker_distribution_exercises_all_worker_paths); + auto r = runNow(); + CHECK(r.routedCounts["deterministic"] == 5, "expected 5 deterministic routes"); + CHECK(r.routedCounts["template"] == 5, "expected 5 template routes"); + CHECK(r.routedCounts["slm"] == 5, "expected 5 slm routes"); + CHECK(r.routedCounts["llm"] == 3, "expected 3 llm routes"); + CHECK(r.routedCounts["human"] == 2, "expected 2 human routes"); + PASS(); +} + +void test_deterministic_and_template_tasks_auto_complete() { + TEST(deterministic_and_template_tasks_auto_complete); + auto r = runNow(); + int autoDone = 0; + for (const auto& t : r.tasks) { + if (t.autoCompleted) ++autoDone; + } + CHECK(autoDone == 10, "expected 10 auto-completed deterministic/template tasks"); + PASS(); +} + +void test_slm_tasks_use_narrow_context_widths() { + TEST(slm_tasks_use_narrow_context_widths); + auto r = runNow(); + for (const auto& t : r.tasks) { + if (t.group != MultiModelTaskGroup::SLM) continue; + CHECK(t.route.contextWidth == "local" || t.route.contextWidth == "file", + "slm tasks should use local/file context"); + } + PASS(); +} + +void test_llm_tasks_use_wide_context_widths() { + TEST(llm_tasks_use_wide_context_widths); + auto r = runNow(); + for (const auto& t : r.tasks) { + if (t.group != MultiModelTaskGroup::LLM) continue; + CHECK(t.route.contextWidth == "project" || t.route.contextWidth == "cross-project", + "llm tasks should use project/cross-project context"); + } + PASS(); +} + +void test_human_tasks_are_flagged_for_review() { + TEST(human_tasks_are_flagged_for_review); + auto r = runNow(); + int humanReview = 0; + for (const auto& t : r.tasks) { + if (t.group == MultiModelTaskGroup::Human && t.reviewRequired) ++humanReview; + } + CHECK(humanReview == 2, "expected both human tasks to require review"); + PASS(); +} + +void test_progress_tracking_reaches_one_hundred_percent() { + TEST(progress_tracking_reaches_one_hundred_percent); + auto r = runNow(); + CHECK(!r.progress.empty(), "expected progress points"); + CHECK(r.progress.back().completed == 20, "expected all tasks completed"); + CHECK(r.progress.back().percent >= 100.0f, "expected 100% completion"); + PASS(); +} + +void test_estimated_cost_and_actual_cost_are_computed() { + TEST(estimated_cost_and_actual_cost_are_computed); + auto r = runNow(); + CHECK(r.estimatedTotalTokens > 0, "missing estimated token total"); + CHECK(r.actualTotalTokens >= 0, "missing actual token total"); + CHECK(r.actualTotalTokens < r.estimatedTotalTokens, "expected actual token use below estimate"); + PASS(); +} + +void test_review_gates_are_exercised() { + TEST(review_gates_are_exercised); + auto r = runNow(); + CHECK(r.reviewGatesExercised, "review gates should be exercised"); + CHECK(r.reviewGateCount >= 5, "expected llm + human review gates"); + PASS(); +} + +void test_all_tasks_finish_in_completed_state() { + TEST(all_tasks_finish_in_completed_state); + auto r = runNow(); + for (const auto& t : r.tasks) { + CHECK(t.completed, "all scenario tasks should be completed"); + } + PASS(); +} + +void test_scenario_notes_capture_multi_model_summary() { + TEST(scenario_notes_capture_multi_model_summary); + auto r = runNow(); + CHECK(r.notes.size() >= 2, "expected scenario notes"); + CHECK(r.notes[0].find("20 tasks") != std::string::npos, "missing task-count summary"); + PASS(); +} + +int main() { + std::cout << "Step 502: Scenario - Multi-Model Orchestration Tests\n"; + + test_scenario_builds_exactly_twenty_tasks(); // 1 + test_task_group_distribution_matches_spec(); // 2 + test_routed_worker_distribution_exercises_all_worker_paths(); // 3 + test_deterministic_and_template_tasks_auto_complete(); // 4 + test_slm_tasks_use_narrow_context_widths(); // 5 + test_llm_tasks_use_wide_context_widths(); // 6 + test_human_tasks_are_flagged_for_review(); // 7 + test_progress_tracking_reaches_one_hundred_percent(); // 8 + test_estimated_cost_and_actual_cost_are_computed(); // 9 + test_review_gates_are_exercised(); // 10 + test_all_tasks_finish_in_completed_state(); // 11 + test_scenario_notes_capture_multi_model_summary(); // 12 + + std::cout << "\nResults: " << passed << "/" << (passed + failed) + << " passed\n"; + return failed == 0 ? 0 : 1; +} diff --git a/progress.md b/progress.md index e245079..72aef55 100644 --- a/progress.md +++ b/progress.md @@ -7712,3 +7712,36 @@ pipeline to Rust with confidence and review gating. - `editor/src/CrossLanguagePortScenarioRunner.h` within header-size limit (`127` <= `600`) - `editor/tests/step501_test.cpp` within test-file size guidance (`165` lines) - Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md` + +### Step 502: Scenario - Multi-Model Orchestration +**Status:** PASS (12/12 tests) + +Implements the fourth Phase 25b end-to-end scenario for a 20-function project +that exercises all worker classes and review gates. + +**Files added:** +- `editor/src/MultiModelOrchestrationScenarioRunner.h` - scenario orchestration across: + - 20-task synthetic project generation + - routing through deterministic/template/slm/llm/human workers + - deterministic/template auto-completion behavior + - per-task token estimate vs actual token tracking + - progress snapshots across full completion lifecycle + - review gate tracking for LLM + human paths +- `editor/tests/step502_test.cpp` - 12 scenario tests covering: + - exact routing distribution and task-group counts + - narrow vs wide context routing semantics + - auto-completion expectations + - review gate exercise and cost/progress accounting + - completion and summary notes +- `editor/CMakeLists.txt` - `step502_test` target + +**Verification run:** +- `cmake -S editor -B editor/build-native` - PASS +- `cmake --build editor/build-native --target step502_test step501_test` - PASS +- `./editor/build-native/step502_test` - PASS (12/12) +- `./editor/build-native/step501_test` - PASS (12/12) regression coverage + +**Architecture gate check:** +- `editor/src/MultiModelOrchestrationScenarioRunner.h` within header-size limit (`177` <= `600`) +- `editor/tests/step502_test.cpp` within test-file size guidance (`152` lines) +- Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md`