Step 423: add model profile registry with overrides

2026-02-16 16:56:10 -07:00
parent 428d7191c5
commit 01951bac03
4 changed files with 405 additions and 0 deletions
--- a/editor/CMakeLists.txt
+++ b/editor/CMakeLists.txt
@@ -2686,4 +2686,13 @@ target_link_libraries(step422_test PRIVATE
    tree_sitter_javascript tree_sitter_typescript
    tree_sitter_java tree_sitter_rust tree_sitter_go)

+add_executable(step423_test tests/step423_test.cpp)
+target_include_directories(step423_test PRIVATE src)
+target_link_libraries(step423_test PRIVATE
+    nlohmann_json::nlohmann_json
+    unofficial::tree-sitter::tree-sitter
+    tree_sitter_python tree_sitter_cpp tree_sitter_elisp
+    tree_sitter_javascript tree_sitter_typescript
+    tree_sitter_java tree_sitter_rust tree_sitter_go)
+
 # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies)
--- a/editor/src/ModelProfileRegistry.h
+++ b/editor/src/ModelProfileRegistry.h
@@ -0,0 +1,172 @@
+#pragma once
+
+#include <algorithm>
+#include <cctype>
+#include <fstream>
+#include <map>
+#include <string>
+#include <vector>
+
+#include <nlohmann/json.hpp>
+using json = nlohmann::json;
+
+struct ModelProfile {
+    std::string name;
+    int contextWindow = 0;
+    double costPer1kInput = 0.0;
+    double costPer1kOutput = 0.0;
+    std::string speedClass;
+    std::vector<std::string> capabilities;
+
+    json toJson() const {
+        return {
+            {"name", name},
+            {"contextWindow", contextWindow},
+            {"costPer1kInput", costPer1kInput},
+            {"costPer1kOutput", costPer1kOutput},
+            {"speedClass", speedClass},
+            {"capabilities", capabilities}
+        };
+    }
+};
+
+class ModelProfileRegistry {
+public:
+    ModelProfileRegistry() {
+        registerDefaults();
+    }
+
+    void registerProfile(const ModelProfile& profile) {
+        if (profile.name.empty()) return;
+        profiles_[profile.name] = profile;
+    }
+
+    bool hasProfile(const std::string& name) const {
+        return profiles_.find(name) != profiles_.end();
+    }
+
+    ModelProfile getProfile(const std::string& name) const {
+        auto it = profiles_.find(name);
+        if (it != profiles_.end()) return it->second;
+        return {};
+    }
+
+    std::vector<std::string> listProfileNames() const {
+        std::vector<std::string> names;
+        for (const auto& [k, _] : profiles_) names.push_back(k);
+        std::sort(names.begin(), names.end());
+        return names;
+    }
+
+    std::string resolveProfileForWorker(const std::string& workerType) const {
+        std::string lower = toLower(workerType);
+        auto it = workerToProfile_.find(lower);
+        if (it != workerToProfile_.end()) return it->second;
+        return "";
+    }
+
+    ModelProfile resolveForWorker(const std::string& workerType) const {
+        std::string name = resolveProfileForWorker(workerType);
+        if (name.empty()) return {};
+        return getProfile(name);
+    }
+
+    void setWorkerProfileMapping(const std::string& workerType,
+                                 const std::string& profileName) {
+        std::string w = toLower(workerType);
+        if (w.empty() || profileName.empty()) return;
+        if (!hasProfile(profileName)) return;
+        workerToProfile_[w] = profileName;
+    }
+
+    bool loadOverridesFromFile(const std::string& configPath,
+                               std::string& error) {
+        std::ifstream in(configPath);
+        if (!in.is_open()) {
+            error = "Cannot open config: " + configPath;
+            return false;
+        }
+        json cfg;
+        try {
+            in >> cfg;
+        } catch (...) {
+            error = "Invalid JSON in config";
+            return false;
+        }
+        if (!cfg.is_object()) {
+            error = "Config must be object";
+            return false;
+        }
+        applyOverrides(cfg);
+        return true;
+    }
+
+    void applyOverrides(const json& cfg) {
+        if (cfg.contains("profiles") && cfg["profiles"].is_array()) {
+            for (const auto& p : cfg["profiles"]) {
+                std::string name = p.value("name", "");
+                if (name.empty()) continue;
+                ModelProfile prof = getProfile(name);
+                if (prof.name.empty()) prof.name = name;
+                prof.contextWindow = p.value("contextWindow", prof.contextWindow);
+                prof.costPer1kInput = p.value("costPer1kInput", prof.costPer1kInput);
+                prof.costPer1kOutput = p.value("costPer1kOutput", prof.costPer1kOutput);
+                prof.speedClass = p.value("speedClass", prof.speedClass);
+                if (p.contains("capabilities") && p["capabilities"].is_array()) {
+                    prof.capabilities.clear();
+                    for (const auto& c : p["capabilities"]) {
+                        if (c.is_string()) prof.capabilities.push_back(c.get<std::string>());
+                    }
+                }
+                registerProfile(prof);
+            }
+        }
+
+        if (cfg.contains("workerMapping") && cfg["workerMapping"].is_object()) {
+            for (auto it = cfg["workerMapping"].begin();
+                 it != cfg["workerMapping"].end(); ++it) {
+                if (!it.value().is_string()) continue;
+                setWorkerProfileMapping(it.key(), it.value().get<std::string>());
+            }
+        }
+    }
+
+    json toJson() const {
+        json profiles = json::array();
+        for (const auto& name : listProfileNames()) {
+            profiles.push_back(getProfile(name).toJson());
+        }
+        json mapping = json::object();
+        for (const auto& [worker, profile] : workerToProfile_) {
+            mapping[worker] = profile;
+        }
+        return {{"profiles", profiles}, {"workerMapping", mapping}};
+    }
+
+private:
+    std::map<std::string, ModelProfile> profiles_;
+    std::map<std::string, std::string> workerToProfile_;
+
+    static std::string toLower(const std::string& in) {
+        std::string out = in;
+        std::transform(out.begin(), out.end(), out.begin(),
+                       [](unsigned char c) { return (char)std::tolower(c); });
+        return out;
+    }
+
+    void registerDefaults() {
+        registerProfile({"claude-opus", 200000, 0.015, 0.075, "slow",
+                         {"reasoning", "architecture", "long-context"}});
+        registerProfile({"claude-sonnet", 200000, 0.003, 0.015, "medium",
+                         {"coding", "reasoning", "long-context"}});
+        registerProfile({"claude-haiku", 200000, 0.0008, 0.004, "fast",
+                         {"coding", "quick-edits"}});
+        registerProfile({"local-slm", 8192, 0.0, 0.0, "fast",
+                         {"offline", "low-latency"}});
+
+        workerToProfile_["llm"] = "claude-sonnet";
+        workerToProfile_["slm"] = "claude-haiku";
+        workerToProfile_["deterministic"] = "local-slm";
+        workerToProfile_["template"] = "local-slm";
+    }
+};
--- a/editor/tests/step423_test.cpp
+++ b/editor/tests/step423_test.cpp
@@ -0,0 +1,169 @@
+// Step 423: Model Profile Registry Tests (12 tests)
+
+#include "ModelProfileRegistry.h"
+
+#include <filesystem>
+#include <fstream>
+#include <iostream>
+#include <string>
+
+static int passed = 0, failed = 0;
+#define TEST(name) { std::cout << "  " << #name << "... "; }
+#define PASS() { std::cout << "PASS\n"; ++passed; }
+#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; }
+#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {}
+
+static std::string makeTempDir(const std::string& name) {
+    auto dir = std::filesystem::temp_directory_path() / ("whetstone_step423_" + name);
+    std::filesystem::create_directories(dir);
+    return dir.string();
+}
+
+void test_default_profiles_exist() {
+    TEST(default_profiles_exist);
+    ModelProfileRegistry reg;
+    CHECK(reg.hasProfile("claude-opus"), "missing claude-opus");
+    CHECK(reg.hasProfile("claude-sonnet"), "missing claude-sonnet");
+    CHECK(reg.hasProfile("claude-haiku"), "missing claude-haiku");
+    CHECK(reg.hasProfile("local-slm"), "missing local-slm");
+    PASS();
+}
+
+void test_default_llm_mapping_to_sonnet() {
+    TEST(default_llm_mapping_to_sonnet);
+    ModelProfileRegistry reg;
+    CHECK(reg.resolveProfileForWorker("llm") == "claude-sonnet",
+          "llm should default to claude-sonnet");
+    PASS();
+}
+
+void test_default_slm_mapping_to_haiku() {
+    TEST(default_slm_mapping_to_haiku);
+    ModelProfileRegistry reg;
+    CHECK(reg.resolveProfileForWorker("slm") == "claude-haiku",
+          "slm should default to claude-haiku");
+    PASS();
+}
+
+void test_resolve_for_worker_returns_profile() {
+    TEST(resolve_for_worker_returns_profile);
+    ModelProfileRegistry reg;
+    auto p = reg.resolveForWorker("llm");
+    CHECK(p.name == "claude-sonnet", "expected sonnet profile");
+    CHECK(p.contextWindow >= 100000, "expected large context window");
+    PASS();
+}
+
+void test_register_custom_profile() {
+    TEST(register_custom_profile);
+    ModelProfileRegistry reg;
+    reg.registerProfile({"my-model", 32000, 0.001, 0.002, "fast", {"coding"}});
+    CHECK(reg.hasProfile("my-model"), "custom profile not registered");
+    CHECK(reg.getProfile("my-model").contextWindow == 32000, "custom context mismatch");
+    PASS();
+}
+
+void test_set_worker_mapping_requires_existing_profile() {
+    TEST(set_worker_mapping_requires_existing_profile);
+    ModelProfileRegistry reg;
+    reg.setWorkerProfileMapping("llm", "missing-profile");
+    CHECK(reg.resolveProfileForWorker("llm") == "claude-sonnet",
+          "should not override with missing profile");
+    PASS();
+}
+
+void test_set_worker_mapping_overrides_default() {
+    TEST(set_worker_mapping_overrides_default);
+    ModelProfileRegistry reg;
+    reg.setWorkerProfileMapping("llm", "claude-opus");
+    CHECK(reg.resolveProfileForWorker("llm") == "claude-opus",
+          "llm override should map to opus");
+    PASS();
+}
+
+void test_apply_overrides_from_json_profiles() {
+    TEST(apply_overrides_from_json_profiles);
+    ModelProfileRegistry reg;
+    json cfg = {
+        {"profiles", json::array({
+            {{"name", "claude-sonnet"}, {"contextWindow", 120000}, {"speedClass", "medium-fast"}},
+            {{"name", "custom-x"}, {"contextWindow", 64000}, {"costPer1kInput", 0.002}}
+        })}
+    };
+    reg.applyOverrides(cfg);
+    CHECK(reg.getProfile("claude-sonnet").contextWindow == 120000, "sonnet override failed");
+    CHECK(reg.hasProfile("custom-x"), "custom profile from json missing");
+    PASS();
+}
+
+void test_apply_overrides_worker_mapping() {
+    TEST(apply_overrides_worker_mapping);
+    ModelProfileRegistry reg;
+    json cfg = {{"workerMapping", {{"llm", "claude-opus"}, {"slm", "local-slm"}}}};
+    reg.applyOverrides(cfg);
+    CHECK(reg.resolveProfileForWorker("llm") == "claude-opus", "llm mapping override failed");
+    CHECK(reg.resolveProfileForWorker("slm") == "local-slm", "slm mapping override failed");
+    PASS();
+}
+
+void test_load_overrides_from_file() {
+    TEST(load_overrides_from_file);
+    ModelProfileRegistry reg;
+    auto dir = std::filesystem::path(makeTempDir("cfg_file"));
+    auto path = dir / "config.json";
+    std::ofstream out(path);
+    out << R"({
+  "profiles": [{"name":"claude-haiku","contextWindow":16000}],
+  "workerMapping": {"slm":"claude-haiku"}
+})";
+    out.close();
+
+    std::string err;
+    CHECK(reg.loadOverridesFromFile(path.string(), err), "load override failed");
+    CHECK(reg.getProfile("claude-haiku").contextWindow == 16000, "haiku override missing");
+    CHECK(reg.resolveProfileForWorker("slm") == "claude-haiku", "mapping not applied");
+    PASS();
+}
+
+void test_load_overrides_invalid_json_fails() {
+    TEST(load_overrides_invalid_json_fails);
+    ModelProfileRegistry reg;
+    auto dir = std::filesystem::path(makeTempDir("bad_cfg"));
+    auto path = dir / "config.json";
+    std::ofstream(path) << "{ bad json";
+    std::string err;
+    CHECK(!reg.loadOverridesFromFile(path.string(), err), "invalid json should fail");
+    CHECK(!err.empty(), "error message should be present");
+    PASS();
+}
+
+void test_registry_json_export_contains_mapping() {
+    TEST(registry_json_export_contains_mapping);
+    ModelProfileRegistry reg;
+    auto j = reg.toJson();
+    CHECK(j.contains("profiles"), "missing profiles");
+    CHECK(j.contains("workerMapping"), "missing workerMapping");
+    CHECK(j["workerMapping"].value("llm", "") == "claude-sonnet", "llm mapping missing");
+    PASS();
+}
+
+int main() {
+    std::cout << "Step 423: Model Profile Registry Tests\n";
+
+    test_default_profiles_exist();                    // 1
+    test_default_llm_mapping_to_sonnet();            // 2
+    test_default_slm_mapping_to_haiku();             // 3
+    test_resolve_for_worker_returns_profile();       // 4
+    test_register_custom_profile();                  // 5
+    test_set_worker_mapping_requires_existing_profile(); // 6
+    test_set_worker_mapping_overrides_default();     // 7
+    test_apply_overrides_from_json_profiles();       // 8
+    test_apply_overrides_worker_mapping();           // 9
+    test_load_overrides_from_file();                 // 10
+    test_load_overrides_invalid_json_fails();        // 11
+    test_registry_json_export_contains_mapping();    // 12
+
+    std::cout << "\nResults: " << passed << "/" << (passed + failed)
+              << " passed\n";
+    return failed == 0 ? 0 : 1;
+}
--- a/progress.md
+++ b/progress.md
@@ -4512,6 +4512,61 @@ orchestration handoff, external-result submission, human review, and completion.
  - `editor/src/MCPServer.h` (`1940` > `600`)
  - `editor/src/HeadlessAgentRPCHandler.h` (`2768` > `600`)

+## Phase 18b: Multi-Model Dispatch
+
+### Step 423: Model Profile Registry
+**Status:** PASS (12/12 tests)
+
+Added a model-profile registry for mapping workflow worker classes (`slm`, `llm`,
+deterministic/template) to concrete model profiles with context-window and cost
+metadata, including `.whetstone/config.json`-style override support.
+
+**Files created:**
+- `editor/src/ModelProfileRegistry.h` — model profile support:
+  - `ModelProfile` schema (`name`, `contextWindow`, cost rates, speed class, capabilities)
+  - default profile set:
+    - `claude-opus`
+    - `claude-sonnet`
+    - `claude-haiku`
+    - `local-slm`
+  - default worker mappings:
+    - `llm` -> `claude-sonnet`
+    - `slm` -> `claude-haiku`
+    - `deterministic`/`template` -> `local-slm`
+  - profile registration/listing/lookup
+  - worker mapping override API
+  - JSON/file override loading for profiles and worker mappings
+  - manifest export (`toJson`)
+- `editor/tests/step423_test.cpp` — 12 tests covering:
+  1. default profile presence
+  2. default `llm` mapping
+  3. default `slm` mapping
+  4. worker-to-profile resolution
+  5. custom profile registration
+  6. mapping validation for missing profiles
+  7. mapping override behavior
+  8. profile override via JSON payload
+  9. worker-mapping override via JSON payload
+  10. file-based override loading
+  11. invalid JSON error path
+  12. registry JSON export shape
+
+**Files modified:**
+- `editor/CMakeLists.txt` — `step423_test` target
+
+**Verification run:**
+- `step423_test` — PASS (12/12) new step coverage
+- `step422_test` — PASS (8/8) regression coverage
+- `step421_test` — PASS (12/12) regression coverage
+
+**Architecture gate check:**
+- `editor/src/ModelProfileRegistry.h` within header-size limit (`172` <= `600`)
+- `editor/tests/step423_test.cpp` within test-file size guidance (`169` lines)
+- Legacy oversized headers persist:
+  - `editor/src/ast/Serialization.h` (`1427` > `600`)
+  - `editor/src/MCPServer.h` (`1940` > `600`)
+  - `editor/src/HeadlessAgentRPCHandler.h` (`2768` > `600`)
+
 # Roadmap Planning — Sprints 12-25+

 ## Status: Planning Complete (Sprints 12-19 detailed, 20-25 in roadmap.md)