From 82bc20b5c8c3c4d03919be6884fc3c088bae904a Mon Sep 17 00:00:00 2001
From: Bill <wholcombe@cnm.edu>
Date: Tue, 10 Feb 2026 08:06:04 -0700
Subject: [PATCH] Step 224: evaluation harness tests

---
 PROGRESS.md                   |  1 +
 editor/CMakeLists.txt         |  4 ++++
 editor/tests/step224_test.cpp | 40 +++++++++++++++++++++++++++++++++++
 sprint7_plan.md               |  2 +-
 4 files changed, 46 insertions(+), 1 deletion(-)
 create mode 100644 editor/tests/step224_test.cpp

diff --git a/PROGRESS.md b/PROGRESS.md
index ddeb779..05a06d3 100644
--- a/PROGRESS.md
+++ b/PROGRESS.md
@@ -739,6 +739,7 @@ vcpkg's imgui 1.91.9 removed the `sdl2-binding` feature (only `sdl3-binding` exi
 | 2026-02-10 | Codex | Step 221: Evaluation basic task suite (30 tasks across core tool flows). 1/1 tests pass (step221_test). |
 | 2026-02-10 | Codex | Step 222: Evaluation workflow task suite (20 multi-step tasks). 1/1 tests pass (step222_test). |
 | 2026-02-10 | Codex | Step 223: Evaluation runner CLI (whetstone_eval, task/trace/report pipeline). 1/1 tests pass (step223_test). |
+| 2026-02-10 | Codex | Step 224: Evaluation harness tests (good/bad/partial scoring). 1/1 tests pass (step224_test). |
 | 2026-02-10 | Codex | Step 166: Extract main.cpp into panel headers marked complete (already implemented). step166_test passes; file_limits_test 4/4 passes. |
 | 2026-02-10 | Claude Opus 4.6 | Sprint 7 Phase 7a (Steps 202–206): API docs (AGENT_API.md, 23 methods), JSON schemas (20 files), exposed ContextAPI/BatchMutationAPI/Pipeline via RPC, permission policy updates. 51/51 tests pass. |
 | 2026-02-10 | Claude Opus 4.6 | Sprint 7 Phase 7b (Steps 207–213): MCPServer.h (10 tools, 5 resources, 4 prompts, JSON-RPC 2.0 initialize handshake), MCPBridge.h (stdio transport). 90/90 tests pass. |
diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt
index 00cb8d3..50fcd80 100644
--- a/editor/CMakeLists.txt
+++ b/editor/CMakeLists.txt
@@ -1154,6 +1154,10 @@ target_include_directories(step222_test PRIVATE src)
 
 add_executable(step223_test tests/step223_test.cpp)
 target_include_directories(step223_test PRIVATE src)
+
+add_executable(step224_test tests/step224_test.cpp)
+target_include_directories(step224_test PRIVATE src)
+target_link_libraries(step224_test PRIVATE nlohmann_json::nlohmann_json)
 add_executable(step213_test tests/step213_test.cpp)
 target_include_directories(step213_test PRIVATE src)
 target_link_libraries(step213_test PRIVATE nlohmann_json::nlohmann_json)
diff --git a/editor/tests/step224_test.cpp b/editor/tests/step224_test.cpp
new file mode 100644
index 0000000..eac32de
--- /dev/null
+++ b/editor/tests/step224_test.cpp
@@ -0,0 +1,40 @@
+// Step 224: Evaluation harness tests.
+
+#include <cassert>
+#include <string>
+#include "EvalHarness.h"
+
+int main() {
+    EvalTask task;
+    task.id = "t1";
+    task.expectedOutcome = {{"toolCalls", {"whetstone_get_ast"}}};
+
+    Trace good;
+    good.id = "t1";
+    good.steps.push_back({"tool_call", "", "whetstone_get_ast", {}, {}});
+    good.toolCallCount = 1;
+
+    EvalResult r1 = EvalRunner::evaluateTraceForTask(task, good);
+    assert(r1.score == 1.0);
+    assert(r1.passed);
+
+    Trace bad;
+    bad.id = "t1";
+    EvalResult r2 = EvalRunner::evaluateTraceForTask(task, bad);
+    assert(r2.score == 0.0);
+    assert(!r2.passed);
+
+    EvalTask partial;
+    partial.id = "t2";
+    partial.allowPartial = true;
+    partial.expectedOutcome = {{"toolCalls", {"a", "b"}}};
+    Trace partialTrace;
+    partialTrace.id = "t2";
+    partialTrace.steps.push_back({"tool_call", "", "a", {}, {}});
+    EvalResult r3 = EvalRunner::evaluateTraceForTask(partial, partialTrace);
+    assert(r3.score > 0.0 && r3.score < 1.0);
+    assert(r3.passed);
+
+    printf("step224_test: all assertions passed\n");
+    return 0;
+}
diff --git a/sprint7_plan.md b/sprint7_plan.md
index dc4643d..a9906b7 100644
--- a/sprint7_plan.md
+++ b/sprint7_plan.md
@@ -331,7 +331,7 @@ Test suites that measure how accurately an LLM can use Whetstone tools.
   - Leaderboard output: rank agents by accuracy, efficiency (fewer tool calls = better)
   *New:* `editor/src/eval_main.cpp`
 
-- [ ] **Step 224: Evaluation harness tests**
+- [x] **Step 224: Evaluation harness tests**
   Tests for the evaluation framework itself:
   1. Known-good traces score 100% on basic tasks
   2. Known-bad traces (wrong mutations) score 0%