From d6941da7931313fef776aba19d7f10dad95c75a1 Mon Sep 17 00:00:00 2001 From: Bill Date: Wed, 11 Feb 2026 15:48:06 +0000 Subject: [PATCH] Step 253: diagnostic pipeline integration tests (Phase 9b complete) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit End-to-end tests: valid code → zero diags, annotation errors with nodeIds, quick fix application clearing diagnostics, delta showing removals/additions, combined parser+annotation streams, severity filtering. Phase 9b: 44/44 tests across steps 250–253. Co-Authored-By: Claude Opus 4.6 --- editor/CMakeLists.txt | 11 ++ editor/tests/step253_test.cpp | 321 ++++++++++++++++++++++++++++++++++ progress.md | 26 +++ 3 files changed, 358 insertions(+) create mode 100644 editor/tests/step253_test.cpp diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 3e403e5..f3d84ee 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -1457,4 +1457,15 @@ target_link_libraries(step252_test PRIVATE tree_sitter_java tree_sitter_rust tree_sitter_go tree_sitter_org) +# Step 253: Diagnostic and delta integration tests +add_executable(step253_test tests/step253_test.cpp) +target_include_directories(step253_test PRIVATE src) +target_link_libraries(step253_test PRIVATE + nlohmann_json::nlohmann_json + unofficial::tree-sitter::tree-sitter + tree_sitter_python tree_sitter_cpp tree_sitter_elisp + tree_sitter_javascript tree_sitter_typescript + tree_sitter_java tree_sitter_rust tree_sitter_go + tree_sitter_org) + # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies) diff --git a/editor/tests/step253_test.cpp b/editor/tests/step253_test.cpp new file mode 100644 index 0000000..49fc3a3 --- /dev/null +++ b/editor/tests/step253_test.cpp @@ -0,0 +1,321 @@ +// Step 253 TDD Test: Diagnostic and Delta Integration Tests +// +// Comprehensive end-to-end tests for the full diagnostic pipeline: +// structured diagnostics, quick fixes, delta streaming, and filtering. +// Exercises the complete flow: introduce error → diagnose → fix → verify. +#include "HeadlessEditorState.h" +#include "MCPBridge.h" + +#include +#include + +static void expect(bool cond, const std::string& name, + int& passed, int& failed) { + if (cond) { + std::cout << "Test " << (passed + failed + 1) + << " PASS: " << name << "\n"; + ++passed; + } else { + std::cout << "Test " << (passed + failed + 1) + << " FAIL: " << name << "\n"; + ++failed; + } +} + +// Helper: send RPC +static json rpc(HeadlessEditorState& state, const std::string& session, + const std::string& method, + json params = json::object()) { + json request = {{"jsonrpc", "2.0"}, {"id", 1}, + {"method", method}, {"params", params}}; + return state.processAgentRequest(request, session); +} + +int main() { + int passed = 0; + int failed = 0; + + // ================================================================= + // Test 1: Parse valid code → zero diagnostics + // ================================================================= + { + HeadlessEditorState state; + state.defaultLanguage = "python"; + state.openBuffer("valid.py", + "def add(a, b):\n return a + b\n\n" + "def mul(x, y):\n return x * y\n", "python"); + state.setAgentRole("s1", AgentRole::Refactor); + + json resp = rpc(state, "s1", "getDiagnostics"); + int count = resp["result"].value("count", -1); + expect(count == 0, + "Valid Python code → zero diagnostics", + passed, failed); + } + + // ================================================================= + // Test 2: Invalid code → structured diagnostics with nodeIds + // ================================================================= + { + HeadlessEditorState state; + state.defaultLanguage = "python"; + state.openBuffer("anno.py", + "def cleanup(ptr):\n x = ptr\n", "python"); + state.setAgentRole("s1", AgentRole::Refactor); + + // Add annotation to create a diagnostic with nodeId + Module* ast = state.activeAST(); + for (auto* c : ast->allChildren()) { + if (c->conceptType == "Function") { + c->addChild("annotations", new DeallocateAnnotation()); + break; + } + } + + json resp = rpc(state, "s1", "getDiagnostics"); + int count = resp["result"].value("count", 0); + bool hasNodeId = false; + bool hasCode = false; + for (const auto& d : resp["result"]["diagnostics"]) { + if (!d.value("nodeId", "").empty()) hasNodeId = true; + if (!d.value("code", "").empty()) hasCode = true; + } + expect(count > 0 && hasNodeId && hasCode, + "Annotation error has structured diagnostics with nodeId " + + std::string("and code (count=") + std::to_string(count) + ")", + passed, failed); + } + + // ================================================================= + // Test 3: Diagnostics include fix suggestions where applicable + // ================================================================= + { + HeadlessEditorState state; + state.defaultLanguage = "python"; + state.openBuffer("fix.py", + "def cleanup(ptr):\n x = ptr\n", "python"); + state.setAgentRole("s1", AgentRole::Refactor); + + for (auto* c : state.activeAST()->allChildren()) { + if (c->conceptType == "Function") { + c->addChild("annotations", new DeallocateAnnotation()); + break; + } + } + + json resp = rpc(state, "s1", "getQuickFixes"); + int fixCount = resp["result"].value("count", 0); + bool hasMutation = false; + if (!resp["result"]["fixes"].empty()) { + hasMutation = resp["result"]["fixes"][0].contains("mutation") && + resp["result"]["fixes"][0]["mutation"].contains("type"); + } + expect(fixCount > 0 && hasMutation, + "Quick fixes include concrete mutation objects (fixes=" + + std::to_string(fixCount) + ")", + passed, failed); + } + + // ================================================================= + // Test 4: applyQuickFix resolves the diagnostic + // ================================================================= + { + HeadlessEditorState state; + state.defaultLanguage = "python"; + state.openBuffer("resolve.py", + "def cleanup(ptr):\n x = ptr\n", "python"); + state.setAgentRole("s1", AgentRole::Refactor); + + std::string funcId; + for (auto* c : state.activeAST()->allChildren()) { + if (c->conceptType == "Function") { + funcId = c->id; + c->addChild("annotations", new DeallocateAnnotation()); + break; + } + } + + // Verify diagnostic exists before fix + json before = rpc(state, "s1", "getDiagnostics"); + int beforeCount = before["result"].value("count", 0); + + // Apply quick fix + json fixResp = rpc(state, "s1", "applyQuickFix", + {{"diagCode", "E0200"}, {"nodeId", funcId}}); + bool cleared = fixResp.contains("result") && + fixResp["result"].value("diagnosticCleared", false); + int remaining = 0; + if (fixResp.contains("result")) + remaining = fixResp["result"].value("remainingDiagnostics", -1); + + expect(beforeCount > 0 && cleared && remaining == 0, + "applyQuickFix resolves diagnostic (before=" + + std::to_string(beforeCount) + " cleared=" + + (cleared ? "true" : "false") + " remaining=" + + std::to_string(remaining) + ")", + passed, failed); + } + + // ================================================================= + // Test 5: Delta after fix shows removal + // ================================================================= + { + HeadlessEditorState state; + state.defaultLanguage = "python"; + state.openBuffer("delta1.py", + "def cleanup(ptr):\n x = ptr\n", "python"); + state.setAgentRole("s1", AgentRole::Refactor); + + std::string funcId; + for (auto* c : state.activeAST()->allChildren()) { + if (c->conceptType == "Function") { + funcId = c->id; + c->addChild("annotations", new DeallocateAnnotation()); + break; + } + } + + // Get diagnostics (establishes version with error) + json diagResp = rpc(state, "s1", "getDiagnostics"); + int ver = diagResp["result"].value("version", 0); + + // Remove the annotation (fix the error manually) + Module* ast = state.activeAST(); + for (auto* c : ast->allChildren()) { + if (c->conceptType == "Function") { + auto annos = c->getChildren("annotations"); + for (auto* a : annos) { + if (a->conceptType == "DeallocateAnnotation") { + c->removeChild(a); + delete a; + break; + } + } + break; + } + } + + // Get delta → should show removal + json delta = rpc(state, "s1", "getDiagnosticsDelta", + {{"sinceVersion", ver}}); + int removed = delta["result"].value("removedCount", 0); + int added = delta["result"].value("addedCount", 0); + expect(removed > 0 && added == 0, + "Delta after fix: removed=" + std::to_string(removed) + + " added=" + std::to_string(added), + passed, failed); + } + + // ================================================================= + // Test 6: Delta after introducing new error shows addition + // ================================================================= + { + HeadlessEditorState state; + state.defaultLanguage = "python"; + state.openBuffer("delta2.py", + "def foo():\n return 1\n", "python"); + state.setAgentRole("s1", AgentRole::Refactor); + + // Get diagnostics (clean baseline) + json diagResp = rpc(state, "s1", "getDiagnostics"); + int ver = diagResp["result"].value("version", 0); + int initialCount = diagResp["result"].value("count", 0); + + // Introduce an error + for (auto* c : state.activeAST()->allChildren()) { + if (c->conceptType == "Function") { + c->addChild("annotations", new DeallocateAnnotation()); + break; + } + } + + // Get delta → should show addition + json delta = rpc(state, "s1", "getDiagnosticsDelta", + {{"sinceVersion", ver}}); + int added = delta["result"].value("addedCount", 0); + int removed = delta["result"].value("removedCount", 0); + expect(initialCount == 0 && added > 0 && removed == 0, + "Delta after new error: added=" + std::to_string(added) + + " removed=" + std::to_string(removed), + passed, failed); + } + + // ================================================================= + // Test 7: Combined tree-sitter + Whetstone diagnostics in one stream + // ================================================================= + { + // Use the pipeline to get both parse and validation diagnostics + Pipeline pipeline; + auto pr = pipeline.run("def broken(\n", "python", "cpp"); + auto allDiags = collectPipelineDiagnostics(pr); + + bool hasParser = false; + for (const auto& d : allDiags) { + if (d.source == "parser") hasParser = true; + } + + // Also test annotation diagnostics in a separate AST + HeadlessEditorState state; + state.defaultLanguage = "python"; + state.openBuffer("combined.py", + "def cleanup(ptr):\n x = ptr\n", "python"); + for (auto* c : state.activeAST()->allChildren()) { + if (c->conceptType == "Function") { + c->addChild("annotations", new DeallocateAnnotation()); + break; + } + } + auto astDiags = collectAllDiagnostics(state.activeAST()); + bool hasAnnotation = false; + for (const auto& d : astDiags) { + if (d.source == "annotation") hasAnnotation = true; + } + + expect(hasParser && hasAnnotation, + "Both parser and annotation diagnostics available in streams", + passed, failed); + } + + // ================================================================= + // Test 8: Severity filtering works across the diagnostic pipeline + // ================================================================= + { + HeadlessEditorState state; + state.defaultLanguage = "python"; + state.openBuffer("sev.py", + "def cleanup(ptr):\n x = ptr\n", "python"); + state.setAgentRole("s1", AgentRole::Refactor); + + for (auto* c : state.activeAST()->allChildren()) { + if (c->conceptType == "Function") { + c->addChild("annotations", new DeallocateAnnotation()); + break; + } + } + + // All diagnostics + json all = rpc(state, "s1", "getDiagnostics"); + int allCount = all["result"].value("count", 0); + + // Errors only + json errOnly = rpc(state, "s1", "getDiagnostics", + {{"severity", "error"}}); + int errCount = errOnly["result"].value("count", 0); + + // All diagnostics with hint level (should include everything) + json hint = rpc(state, "s1", "getDiagnostics", + {{"severity", "hint"}}); + int hintCount = hint["result"].value("count", 0); + + expect(allCount > 0 && errCount <= allCount && + hintCount >= errCount, + "Severity filtering: all=" + std::to_string(allCount) + + " error=" + std::to_string(errCount) + + " hint=" + std::to_string(hintCount), + passed, failed); + } + + std::cout << "\n=== Step 253 Results: " << passed << " passed, " + << failed << " failed ===\n"; + return failed == 0 ? 0 : 1; +} diff --git a/progress.md b/progress.md index feb2372..5295bdd 100644 --- a/progress.md +++ b/progress.md @@ -252,3 +252,29 @@ based on a version number, enabling fast mutate → check loops. - Delta computed by set difference between previous and current snapshots - Response includes addedCount/removedCount for quick checks without parsing - tools/list now returns 21 tools (was 20): +whetstone_get_diagnostics_delta + +### Step 253: Diagnostic and Delta Integration Tests +**Status:** PASS (8/8 tests) + +Comprehensive end-to-end tests for the full diagnostic pipeline: structured +diagnostics, quick fixes, delta streaming, and filtering. Exercises the +complete flow: introduce error → diagnose → fix → verify clearing. + +**Files created:** +- `editor/tests/step253_test.cpp` — 8 integration test cases: + 1. Valid Python → zero diagnostics + 2. Annotation error → structured diagnostics with nodeIds and codes + 3. Quick fixes include concrete mutation objects + 4. applyQuickFix resolves diagnostic (cleared=true, remaining=0) + 5. Delta after fix shows removal (removed=1, added=0) + 6. Delta after new error shows addition (added=1, removed=0) + 7. Combined parser + annotation diagnostics in streams + 8. Severity filtering across the pipeline + +**Files modified:** +- `editor/CMakeLists.txt` — step253_test target + +**Key results:** +- Phase 9b complete: all 4 steps pass (44/44 tests across steps 250–253) +- Full diagnostic pipeline validated: diagnose → fix → verify → delta +- 21 MCP tools, structured error codes (E01xx/E02xx/E03xx), delta streaming