bill/whetstone_DSL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Step 502: add multi-model orchestration scenario and tests

Browse Source
This commit is contained in:
Bill
2026-02-16 22:39:55 -07:00
parent e6ac78e311
commit 7ec9147fb7
4 changed files with 371 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
editor/CMakeLists.txt
View File

@@ -3397,4 +3397,13 @@ target_link_libraries(step501_test PRIVATE
tree_sitter_javascript tree_sitter_typescript
tree_sitter_java tree_sitter_rust tree_sitter_go)
add_executable(step502_test tests/step502_test.cpp)
target_include_directories(step502_test PRIVATE src)
target_link_libraries(step502_test PRIVATE
nlohmann_json::nlohmann_json
unofficial::tree-sitter::tree-sitter
tree_sitter_python tree_sitter_cpp tree_sitter_elisp
tree_sitter_javascript tree_sitter_typescript
tree_sitter_java tree_sitter_rust tree_sitter_go)
# Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies)

177
editor/src/MultiModelOrchestrationScenarioRunner.h Normal file
View File

@@ -0,0 +1,177 @@
#pragma once
// Step 502: Scenario - Multi-Model Orchestration
// End-to-end orchestration scenario with deterministic/template/slm/llm/human workers.
#include "RoutingEngine.h"
#include <map>
#include <string>
#include <vector>
enum class MultiModelTaskGroup { Deterministic, Template, SLM, LLM, Human };
struct MultiModelTaskResult {
std::string id;
std::string name;
MultiModelTaskGroup group = MultiModelTaskGroup::SLM;
WorkItem item;
RoutingDecision route;
bool autoCompleted = false;
bool completed = false;
bool reviewRequired = false;
int estimatedTokens = 0;
int actualTokens = 0;
};
struct MultiModelProgressPoint {
int completed = 0;
int total = 0;
float percent = 0.0f;
};
struct MultiModelOrchestrationScenarioResult {
std::vector<MultiModelTaskResult> tasks;
std::vector<MultiModelProgressPoint> progress;
std::map<std::string, int> routedCounts;
int estimatedTotalTokens = 0;
int actualTotalTokens = 0;
int reviewGateCount = 0;
bool reviewGatesExercised = false;
std::vector<std::string> notes;
int countGroup(MultiModelTaskGroup g) const {
int n = 0;
for (const auto& t : tasks) if (t.group == g) ++n;
return n;
}
};
class MultiModelOrchestrationScenarioRunner {
public:
static MultiModelOrchestrationScenarioResult run() {
MultiModelOrchestrationScenarioResult out;
RoutingEngine routing;
auto items = buildScenarioItems();
const int total = static_cast<int>(items.size());
int completed = 0;
for (auto& task : items) {
task.route = routing.route(task.item);
task.estimatedTokens = task.route.contextBudgetTokens + estimatedOutputTokens(task.route.workerType);
out.estimatedTotalTokens += task.estimatedTokens;
out.routedCounts[task.route.workerType]++;
executeTask(task, out.reviewGateCount);
out.actualTotalTokens += task.actualTokens;
if (task.completed) ++completed;
out.progress.push_back({completed, total, percent(completed, total)});
}
out.tasks = std::move(items);
out.reviewGatesExercised = out.reviewGateCount > 0;
out.notes.push_back("Multi-model orchestration scenario executed with 20 tasks");
out.notes.push_back("Routing mix includes deterministic/template/slm/llm/human paths");
return out;
}
private:
static std::vector<MultiModelTaskResult> buildScenarioItems() {
std::vector<MultiModelTaskResult> out;
int idx = 0;
auto makeId = [&idx]() { return "mm-" + std::to_string(++idx); };
// 5 deterministic tasks
out.push_back(makeTask(makeId(), "normalize_id", MultiModelTaskGroup::Deterministic, "deterministic", "local", false));
out.push_back(makeTask(makeId(), "sanitize_email", MultiModelTaskGroup::Deterministic, "deterministic", "local", false));
out.push_back(makeTask(makeId(), "hash_password", MultiModelTaskGroup::Deterministic, "deterministic", "local", false));
out.push_back(makeTask(makeId(), "build_slug", MultiModelTaskGroup::Deterministic, "deterministic", "local", false));
out.push_back(makeTask(makeId(), "validate_url", MultiModelTaskGroup::Deterministic, "deterministic", "local", false));
// 5 template tasks (getter/setter/constructor style)
out.push_back(makeTask(makeId(), "get_user", MultiModelTaskGroup::Template, "", "local", false));
out.push_back(makeTask(makeId(), "set_user", MultiModelTaskGroup::Template, "", "local", false));
out.push_back(makeTask(makeId(), "get_short_link", MultiModelTaskGroup::Template, "", "local", false));
out.push_back(makeTask(makeId(), "set_short_link", MultiModelTaskGroup::Template, "", "local", false));
out.push_back(makeTask(makeId(), "is_active", MultiModelTaskGroup::Template, "", "local", false));
// 5 SLM tasks (narrow context)
out.push_back(makeTask(makeId(), "apply_rate_limit", MultiModelTaskGroup::SLM, "", "local", false));
out.push_back(makeTask(makeId(), "merge_query_filters", MultiModelTaskGroup::SLM, "", "file", false));
out.push_back(makeTask(makeId(), "serialize_session", MultiModelTaskGroup::SLM, "", "local", false));
out.push_back(makeTask(makeId(), "parse_feature_flags", MultiModelTaskGroup::SLM, "", "file", false));
out.push_back(makeTask(makeId(), "compute_expiry", MultiModelTaskGroup::SLM, "", "local", false));
// 3 LLM tasks (wide context + review gate)
out.push_back(makeTask(makeId(), "cross_module_cache_invalidation", MultiModelTaskGroup::LLM, "", "project", true));
out.push_back(makeTask(makeId(), "multi_region_failover_strategy", MultiModelTaskGroup::LLM, "", "cross-project", true));
out.push_back(makeTask(makeId(), "consistency_resolution_policy", MultiModelTaskGroup::LLM, "", "project", true));
// 2 human tasks (architectural decisions)
out.push_back(makeTask(makeId(), "select_sharding_strategy", MultiModelTaskGroup::Human, "human", "cross-project", true));
out.push_back(makeTask(makeId(), "approve_security_boundary_model", MultiModelTaskGroup::Human, "human", "project", true));
return out;
}
static MultiModelTaskResult makeTask(const std::string& id,
const std::string& name,
MultiModelTaskGroup group,
const std::string& workerType,
const std::string& contextWidth,
bool reviewRequired) {
MultiModelTaskResult t;
t.id = id;
t.name = name;
t.group = group;
t.item.id = id;
t.item.nodeName = name;
t.item.nodeType = "Function";
t.item.bufferId = "scenario502";
t.item.workerType = workerType;
t.item.contextWidth = contextWidth;
t.item.reviewRequired = reviewRequired;
t.item.status = WI_READY;
return t;
}
static void executeTask(MultiModelTaskResult& t, int& reviewGateCount) {
t.reviewRequired = t.route.reviewRequired;
if (t.route.workerType == "deterministic" || t.route.workerType == "template") {
t.autoCompleted = true;
t.completed = true;
t.actualTokens = 0;
return;
}
if (t.route.workerType == "slm") {
t.completed = true;
t.actualTokens = t.route.contextBudgetTokens / 3;
return;
}
if (t.route.workerType == "llm") {
t.completed = true;
t.actualTokens = static_cast<int>(t.route.contextBudgetTokens * 0.85f);
if (t.reviewRequired) ++reviewGateCount;
return;
}
// Human path
t.completed = true;
t.actualTokens = 0;
++reviewGateCount;
}
static int estimatedOutputTokens(const std::string& workerType) {
if (workerType == "deterministic" || workerType == "template" || workerType == "human") return 0;
if (workerType == "slm") return 250;
return 900;
}
static float percent(int done, int total) {
if (total <= 0) return 0.0f;
return 100.0f * static_cast<float>(done) / static_cast<float>(total);
}
};

152
editor/tests/step502_test.cpp Normal file
View File

@@ -0,0 +1,152 @@
// Step 502: Scenario - Multi-Model Orchestration Tests (12 tests)
#include "MultiModelOrchestrationScenarioRunner.h"
#include <iostream>
static int passed = 0, failed = 0;
#define TEST(name) { std::cout << " " << #name << "... "; }
#define PASS() { std::cout << "PASS\n"; ++passed; }
#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; }
#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {}
static MultiModelOrchestrationScenarioResult runNow() {
return MultiModelOrchestrationScenarioRunner::run();
}
void test_scenario_builds_exactly_twenty_tasks() {
TEST(scenario_builds_exactly_twenty_tasks);
auto r = runNow();
CHECK(r.tasks.size() == 20, "expected 20 tasks");
PASS();
}
void test_task_group_distribution_matches_spec() {
TEST(task_group_distribution_matches_spec);
auto r = runNow();
CHECK(r.countGroup(MultiModelTaskGroup::Deterministic) == 5, "expected 5 deterministic");
CHECK(r.countGroup(MultiModelTaskGroup::Template) == 5, "expected 5 template");
CHECK(r.countGroup(MultiModelTaskGroup::SLM) == 5, "expected 5 slm");
CHECK(r.countGroup(MultiModelTaskGroup::LLM) == 3, "expected 3 llm");
CHECK(r.countGroup(MultiModelTaskGroup::Human) == 2, "expected 2 human");
PASS();
}
void test_routed_worker_distribution_exercises_all_worker_paths() {
TEST(routed_worker_distribution_exercises_all_worker_paths);
auto r = runNow();
CHECK(r.routedCounts["deterministic"] == 5, "expected 5 deterministic routes");
CHECK(r.routedCounts["template"] == 5, "expected 5 template routes");
CHECK(r.routedCounts["slm"] == 5, "expected 5 slm routes");
CHECK(r.routedCounts["llm"] == 3, "expected 3 llm routes");
CHECK(r.routedCounts["human"] == 2, "expected 2 human routes");
PASS();
}
void test_deterministic_and_template_tasks_auto_complete() {
TEST(deterministic_and_template_tasks_auto_complete);
auto r = runNow();
int autoDone = 0;
for (const auto& t : r.tasks) {
if (t.autoCompleted) ++autoDone;
}
CHECK(autoDone == 10, "expected 10 auto-completed deterministic/template tasks");
PASS();
}
void test_slm_tasks_use_narrow_context_widths() {
TEST(slm_tasks_use_narrow_context_widths);
auto r = runNow();
for (const auto& t : r.tasks) {
if (t.group != MultiModelTaskGroup::SLM) continue;
CHECK(t.route.contextWidth == "local" || t.route.contextWidth == "file",
"slm tasks should use local/file context");
}
PASS();
}
void test_llm_tasks_use_wide_context_widths() {
TEST(llm_tasks_use_wide_context_widths);
auto r = runNow();
for (const auto& t : r.tasks) {
if (t.group != MultiModelTaskGroup::LLM) continue;
CHECK(t.route.contextWidth == "project" || t.route.contextWidth == "cross-project",
"llm tasks should use project/cross-project context");
}
PASS();
}
void test_human_tasks_are_flagged_for_review() {
TEST(human_tasks_are_flagged_for_review);
auto r = runNow();
int humanReview = 0;
for (const auto& t : r.tasks) {
if (t.group == MultiModelTaskGroup::Human && t.reviewRequired) ++humanReview;
}
CHECK(humanReview == 2, "expected both human tasks to require review");
PASS();
}
void test_progress_tracking_reaches_one_hundred_percent() {
TEST(progress_tracking_reaches_one_hundred_percent);
auto r = runNow();
CHECK(!r.progress.empty(), "expected progress points");
CHECK(r.progress.back().completed == 20, "expected all tasks completed");
CHECK(r.progress.back().percent >= 100.0f, "expected 100% completion");
PASS();
}
void test_estimated_cost_and_actual_cost_are_computed() {
TEST(estimated_cost_and_actual_cost_are_computed);
auto r = runNow();
CHECK(r.estimatedTotalTokens > 0, "missing estimated token total");
CHECK(r.actualTotalTokens >= 0, "missing actual token total");
CHECK(r.actualTotalTokens < r.estimatedTotalTokens, "expected actual token use below estimate");
PASS();
}
void test_review_gates_are_exercised() {
TEST(review_gates_are_exercised);
auto r = runNow();
CHECK(r.reviewGatesExercised, "review gates should be exercised");
CHECK(r.reviewGateCount >= 5, "expected llm + human review gates");
PASS();
}
void test_all_tasks_finish_in_completed_state() {
TEST(all_tasks_finish_in_completed_state);
auto r = runNow();
for (const auto& t : r.tasks) {
CHECK(t.completed, "all scenario tasks should be completed");
}
PASS();
}
void test_scenario_notes_capture_multi_model_summary() {
TEST(scenario_notes_capture_multi_model_summary);
auto r = runNow();
CHECK(r.notes.size() >= 2, "expected scenario notes");
CHECK(r.notes[0].find("20 tasks") != std::string::npos, "missing task-count summary");
PASS();
}
int main() {
std::cout << "Step 502: Scenario - Multi-Model Orchestration Tests\n";
test_scenario_builds_exactly_twenty_tasks(); // 1
test_task_group_distribution_matches_spec(); // 2
test_routed_worker_distribution_exercises_all_worker_paths(); // 3
test_deterministic_and_template_tasks_auto_complete(); // 4
test_slm_tasks_use_narrow_context_widths(); // 5
test_llm_tasks_use_wide_context_widths(); // 6
test_human_tasks_are_flagged_for_review(); // 7
test_progress_tracking_reaches_one_hundred_percent(); // 8
test_estimated_cost_and_actual_cost_are_computed(); // 9
test_review_gates_are_exercised(); // 10
test_all_tasks_finish_in_completed_state(); // 11
test_scenario_notes_capture_multi_model_summary(); // 12
std::cout << "\nResults: " << passed << "/" << (passed + failed)
<< " passed\n";
return failed == 0 ? 0 : 1;
}

33
progress.md
View File

@@ -7712,3 +7712,36 @@ pipeline to Rust with confidence and review gating.
- `editor/src/CrossLanguagePortScenarioRunner.h` within header-size limit (`127` <= `600`)
- `editor/tests/step501_test.cpp` within test-file size guidance (`165` lines)
- Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md`
### Step 502: Scenario - Multi-Model Orchestration
**Status:** PASS (12/12 tests)
Implements the fourth Phase 25b end-to-end scenario for a 20-function project
that exercises all worker classes and review gates.
**Files added:**
- `editor/src/MultiModelOrchestrationScenarioRunner.h` - scenario orchestration across:
- 20-task synthetic project generation
- routing through deterministic/template/slm/llm/human workers
- deterministic/template auto-completion behavior
- per-task token estimate vs actual token tracking
- progress snapshots across full completion lifecycle
- review gate tracking for LLM + human paths
- `editor/tests/step502_test.cpp` - 12 scenario tests covering:
- exact routing distribution and task-group counts
- narrow vs wide context routing semantics
- auto-completion expectations
- review gate exercise and cost/progress accounting
- completion and summary notes
- `editor/CMakeLists.txt` - `step502_test` target
**Verification run:**
- `cmake -S editor -B editor/build-native` - PASS
- `cmake --build editor/build-native --target step502_test step501_test` - PASS
- `./editor/build-native/step502_test` - PASS (12/12)
- `./editor/build-native/step501_test` - PASS (12/12) regression coverage
**Architecture gate check:**
- `editor/src/MultiModelOrchestrationScenarioRunner.h` within header-size limit (`177` <= `600`)
- `editor/tests/step502_test.cpp` within test-file size guidance (`152` lines)
- Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md`