From 5fdb21d74d1583bf3287c6915ab066fb9f07650e Mon Sep 17 00:00:00 2001 From: Bill Date: Mon, 9 Feb 2026 10:22:13 -0700 Subject: [PATCH] Step 94a: AST source spans --- PROGRESS.md | 3 ++ editor/CMakeLists.txt | 4 ++ editor/src/ast/ASTNode.h | 15 ++++++ editor/src/ast/Parser.h | 84 ++++++++++++++++++++++++++++++---- editor/src/ast/Serialization.h | 17 +++++++ editor/tests/step94a_test.cpp | 39 ++++++++++++++++ sprint4_plan.md | 6 +++ 7 files changed, 160 insertions(+), 8 deletions(-) create mode 100644 editor/tests/step94a_test.cpp diff --git a/PROGRESS.md b/PROGRESS.md index ddccc8d..af11da4 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -194,6 +194,7 @@ All 38 steps implemented and passing. Each step has a corresponding test (`step1 - [x] Step 92: **IMPLEMENTED** — LSP completion requests, response parsing, and completion popup with filtering/acceptance (1/1 tests pass) - [x] Step 93: **IMPLEMENTED** — LSP hover and signature help requests, response parsing, and inline tooltip/popup (2/2 tests pass) - [x] Step 94: **IMPLEMENTED** — Whetstone diagnostics aggregation via Pipeline, merged Problems panel, and gutter markers (1/1 tests pass) +- [x] Step 94a: **IMPLEMENTED** — AST source span tracking via tree-sitter, serialized in JSON (2/2 tests pass) - [x] Step 95: **IMPLEMENTED** — Diagnostic gutter markers with tooltips and inline squiggles (1/1 tests pass) - [x] Step 96: **IMPLEMENTED** — LSP server settings UI with defaults, auto-detect, and schema validation hook (2/2 tests pass) @@ -269,6 +270,7 @@ vcpkg's imgui 1.91.9 removed the `sdl2-binding` feature (only `sdl3-binding` exi **Step 92:** Compile and pass (1/1) **Step 93:** Compile and pass (2/2) **Step 94:** Compile and pass (1/1) +**Step 94a:** Compile and pass (2/2) **Step 95:** Compile and pass (1/1) **Step 96:** Compile and pass (2/2) @@ -373,5 +375,6 @@ Sprint 4 in progress. Step 76 (LayoutManager) done. Next: Step 77 (custom code e | 2026-02-09 | Codex | Step 92: LSP completion requests/response parsing; completion popup with filtering and acceptance. 1/1 tests pass. | | 2026-02-09 | Codex | Step 93: LSP hover and signature help requests/response parsing; tooltip and signature popup. 2/2 tests pass. | | 2026-02-09 | Codex | Step 94: Whetstone diagnostics aggregation via Pipeline; merged Problems panel and gutter markers. 1/1 tests pass. | +| 2026-02-09 | Codex | Step 94a: AST source span tracking (tree-sitter) with JSON serialization. 2/2 tests pass. | | 2026-02-09 | Codex | Step 95: Diagnostic gutter markers with tooltips and inline squiggles. 1/1 tests pass. | | 2026-02-09 | Codex | Step 96: LSP server settings UI with defaults, auto-detect, and schema validation hook. 2/2 tests pass. | diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 7d8b69e..7ed5dee 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -522,6 +522,10 @@ add_executable(step94_test tests/step94_test.cpp) target_include_directories(step94_test PRIVATE src) target_link_libraries(step94_test PRIVATE nlohmann_json::nlohmann_json) +add_executable(step94a_test tests/step94a_test.cpp) +target_include_directories(step94a_test PRIVATE src) +target_link_libraries(step94a_test PRIVATE nlohmann_json::nlohmann_json unofficial::tree-sitter::tree-sitter tree_sitter_python) + add_executable(step95_test tests/step95_test.cpp) target_include_directories(step95_test PRIVATE src) target_link_libraries(step95_test PRIVATE imgui::imgui unofficial::tree-sitter::tree-sitter tree_sitter_python tree_sitter_cpp tree_sitter_elisp) diff --git a/editor/src/ast/ASTNode.h b/editor/src/ast/ASTNode.h index 01cb2bf..a5238fb 100644 --- a/editor/src/ast/ASTNode.h +++ b/editor/src/ast/ASTNode.h @@ -8,9 +8,24 @@ public: std::string id; std::string conceptType; ASTNode* parent = nullptr; + int spanStartLine = -1; + int spanStartCol = -1; + int spanEndLine = -1; + int spanEndCol = -1; virtual ~ASTNode() = default; + void setSpan(int startLine, int startCol, int endLine, int endCol) { + spanStartLine = startLine; + spanStartCol = startCol; + spanEndLine = endLine; + spanEndCol = endCol; + } + + bool hasSpan() const { + return spanStartLine >= 0 && spanStartCol >= 0 && spanEndLine >= 0 && spanEndCol >= 0; + } + // Multi-valued child: append to role void addChild(const std::string& role, ASTNode* child) { child->parent = this; diff --git a/editor/src/ast/Parser.h b/editor/src/ast/Parser.h index 5d6f118..b78de31 100644 --- a/editor/src/ast/Parser.h +++ b/editor/src/ast/Parser.h @@ -67,6 +67,7 @@ public: module->id = IdGenerator::next("mod"); module->name = "parsed_python_module"; module->targetLanguage = "python"; + applySpan(module.get(), root); convertPythonModule(root, source, module.get()); @@ -86,6 +87,7 @@ public: result.module->id = IdGenerator::next("mod"); result.module->name = "parsed_python_module"; result.module->targetLanguage = "python"; + applySpan(result.module.get(), root); convertPythonModule(root, source, result.module.get()); collectDiagnostics(root, source, result.diagnostics); @@ -108,6 +110,7 @@ public: module->id = IdGenerator::next("mod"); module->name = "parsed_cpp_module"; module->targetLanguage = "cpp"; + applySpan(module.get(), root); convertCppTranslationUnit(root, source, module.get()); @@ -127,6 +130,7 @@ public: result.module->id = IdGenerator::next("mod"); result.module->name = "parsed_cpp_module"; result.module->targetLanguage = "cpp"; + applySpan(result.module.get(), root); convertCppTranslationUnit(root, source, result.module.get()); collectDiagnostics(root, source, result.diagnostics); @@ -149,6 +153,7 @@ public: module->id = IdGenerator::next("mod"); module->name = "parsed_elisp_module"; module->targetLanguage = "elisp"; + applySpan(module.get(), root); convertElispSourceFile(root, source, module.get()); @@ -168,6 +173,7 @@ public: result.module->id = IdGenerator::next("mod"); result.module->name = "parsed_elisp_module"; result.module->targetLanguage = "elisp"; + applySpan(result.module.get(), root); convertElispSourceFile(root, source, result.module.get()); collectDiagnostics(root, source, result.diagnostics); @@ -192,6 +198,13 @@ private: return ts_node_type(node); } + static void applySpan(ASTNode* node, TSNode tsNode) { + if (!node || ts_node_is_null(tsNode)) return; + TSPoint start = ts_node_start_point(tsNode); + TSPoint end = ts_node_end_point(tsNode); + node->setSpan((int)start.row, (int)start.column, (int)end.row, (int)end.column); + } + static bool isNamed(TSNode node) { return ts_node_is_named(node); } @@ -243,7 +256,10 @@ private: auto* fn = new Function(); fn->id = IdGenerator::next("fn"); + applySpan(fn, node); + applySpan(fn, node); fn->name = nodeText(nameNode, source); + applySpan(fn, node); // Parameters TSNode paramsNode = childByFieldName(node, "parameters"); @@ -271,6 +287,7 @@ private: std::string type = nodeType(child); if (type == "identifier") { auto* param = new Parameter(IdGenerator::next("param"), nodeText(child, source)); + applySpan(param, child); fn->addChild("parameters", param); } else if (type == "default_parameter") { // def f(x=10) → Parameter with defaultValue @@ -278,6 +295,7 @@ private: TSNode valueN = childByFieldName(child, "value"); if (!ts_node_is_null(nameN)) { auto* param = new Parameter(IdGenerator::next("param"), nodeText(nameN, source)); + applySpan(param, child); if (!ts_node_is_null(valueN)) { ASTNode* defVal = convertPythonExpression(valueN, source); if (defVal) param->setChild("defaultValue", defVal); @@ -303,6 +321,7 @@ private: if (type == "return_statement") { auto* ret = new Return(); ret->id = IdGenerator::next("ret"); + applySpan(ret, node); // The return value is the first named child (if any) uint32_t count = ts_node_named_child_count(node); if (count > 0) { @@ -314,6 +333,7 @@ private: } else if (type == "if_statement") { auto* ifStmt = new IfStatement(); ifStmt->id = IdGenerator::next("if"); + applySpan(ifStmt, node); TSNode condNode = childByFieldName(node, "condition"); if (!ts_node_is_null(condNode)) { ASTNode* cond = convertPythonExpression(condNode, source); @@ -331,6 +351,7 @@ private: } else if (type == "for_statement") { auto* forLoop = new ForLoop(); forLoop->id = IdGenerator::next("for"); + applySpan(forLoop, node); TSNode leftNode = childByFieldName(node, "left"); if (!ts_node_is_null(leftNode)) { forLoop->iteratorName = nodeText(leftNode, source); @@ -352,6 +373,7 @@ private: } else if (type == "expression_statement") { auto* exprStmt = new ExpressionStatement(); exprStmt->id = IdGenerator::next("exprstmt"); + applySpan(exprStmt, node); uint32_t count = ts_node_named_child_count(node); if (count > 0) { ASTNode* expr = convertPythonExpression(ts_node_named_child(node, 0), source); @@ -364,6 +386,7 @@ private: if (expr) { auto* exprStmt = new ExpressionStatement(); exprStmt->id = IdGenerator::next("exprstmt"); + applySpan(exprStmt, node); exprStmt->setChild("expression", expr); return exprStmt; } @@ -375,6 +398,7 @@ private: if (type == "binary_operator") { auto* binOp = new BinaryOperation(); binOp->id = IdGenerator::next("binop"); + applySpan(binOp, node); TSNode opNode = childByFieldName(node, "operator"); if (!ts_node_is_null(opNode)) { binOp->op = nodeText(opNode, source); @@ -392,20 +416,24 @@ private: return binOp; } else if (type == "identifier") { auto* ref = new VariableReference(IdGenerator::next("var"), nodeText(node, source)); + applySpan(ref, node); return ref; } else if (type == "integer") { std::string text = nodeText(node, source); int val = 0; try { val = std::stoi(text); } catch (...) {} auto* lit = new IntegerLiteral(IdGenerator::next("int"), val); + applySpan(lit, node); return lit; } else if (type == "string" || type == "concatenated_string") { auto* lit = new StringLiteral(IdGenerator::next("str"), nodeText(node, source)); + applySpan(lit, node); return lit; } else if (type == "comparison_operator" || type == "boolean_operator") { // Treat like binary op auto* binOp = new BinaryOperation(); binOp->id = IdGenerator::next("binop"); + applySpan(binOp, node); uint32_t count = ts_node_named_child_count(node); if (count >= 2) { ASTNode* left = convertPythonExpression(ts_node_named_child(node, 0), source); @@ -429,6 +457,7 @@ private: } else if (type == "call") { auto* call = new FunctionCall(); call->id = IdGenerator::next("call"); + applySpan(call, node); TSNode funcNode = childByFieldName(node, "function"); if (!ts_node_is_null(funcNode)) { call->functionName = nodeText(funcNode, source); @@ -448,6 +477,7 @@ private: } else if (type == "unary_operator") { auto* unOp = new UnaryOperation(); unOp->id = IdGenerator::next("unop"); + applySpan(unOp, node); TSNode opNode = childByFieldName(node, "operator"); if (!ts_node_is_null(opNode)) { unOp->op = nodeText(opNode, source); @@ -463,6 +493,7 @@ private: std::string text = nodeText(node, source); if (!text.empty()) { auto* ref = new VariableReference(IdGenerator::next("var"), text); + applySpan(ref, node); return ref; } return nullptr; @@ -486,6 +517,7 @@ private: static Function* convertCppFunction(TSNode node, const std::string& source) { auto* fn = new Function(); fn->id = IdGenerator::next("fn"); + applySpan(fn, node); // In C++ grammar, the structure is: // function_definition: type declarator body @@ -578,6 +610,7 @@ private: static void convertCppParameter(TSNode paramNode, const std::string& source, Function* fn) { auto* param = new Parameter(); param->id = IdGenerator::next("param"); + applySpan(param, paramNode); // parameter_declaration has type and declarator fields TSNode typeNode = childByFieldName(paramNode, "type"); @@ -586,6 +619,7 @@ private: if (!ts_node_is_null(typeNode)) { std::string typeText = nodeText(typeNode, source); auto* primType = new PrimitiveType(IdGenerator::next("type"), typeText); + applySpan(primType, typeNode); param->setChild("type", primType); } @@ -611,6 +645,7 @@ private: if (type == "return_statement") { auto* ret = new Return(); ret->id = IdGenerator::next("ret"); + applySpan(ret, node); uint32_t count = ts_node_named_child_count(node); if (count > 0) { ASTNode* val = convertCppExpression(ts_node_named_child(node, 0), source); @@ -620,6 +655,7 @@ private: } else if (type == "expression_statement") { auto* exprStmt = new ExpressionStatement(); exprStmt->id = IdGenerator::next("exprstmt"); + applySpan(exprStmt, node); uint32_t count = ts_node_named_child_count(node); if (count > 0) { ASTNode* expr = convertCppExpression(ts_node_named_child(node, 0), source); @@ -629,10 +665,12 @@ private: } else if (type == "declaration") { auto* exprStmt = new ExpressionStatement(); exprStmt->id = IdGenerator::next("exprstmt"); + applySpan(exprStmt, node); return exprStmt; } else if (type == "if_statement") { auto* ifStmt = new IfStatement(); ifStmt->id = IdGenerator::next("if"); + applySpan(ifStmt, node); return ifStmt; } return nullptr; @@ -643,6 +681,7 @@ private: if (type == "binary_expression") { auto* binOp = new BinaryOperation(); binOp->id = IdGenerator::next("binop"); + applySpan(binOp, node); TSNode leftNode = childByFieldName(node, "left"); TSNode rightNode = childByFieldName(node, "right"); TSNode opNode = childByFieldName(node, "operator"); @@ -659,14 +698,20 @@ private: } return binOp; } else if (type == "identifier") { - return new VariableReference(IdGenerator::next("var"), nodeText(node, source)); + auto* ref = new VariableReference(IdGenerator::next("var"), nodeText(node, source)); + applySpan(ref, node); + return ref; } else if (type == "number_literal") { std::string text = nodeText(node, source); int val = 0; try { val = std::stoi(text); } catch (...) {} - return new IntegerLiteral(IdGenerator::next("int"), val); + auto* lit = new IntegerLiteral(IdGenerator::next("int"), val); + applySpan(lit, node); + return lit; } else if (type == "string_literal" || type == "raw_string_literal") { - return new StringLiteral(IdGenerator::next("str"), nodeText(node, source)); + auto* lit = new StringLiteral(IdGenerator::next("str"), nodeText(node, source)); + applySpan(lit, node); + return lit; } else if (type == "parenthesized_expression") { uint32_t count = ts_node_named_child_count(node); if (count > 0) return convertCppExpression(ts_node_named_child(node, 0), source); @@ -674,7 +719,9 @@ private: // Fallback std::string text = nodeText(node, source); if (!text.empty()) { - return new VariableReference(IdGenerator::next("var"), text); + auto* ref = new VariableReference(IdGenerator::next("var"), text); + applySpan(ref, node); + return ref; } return nullptr; } @@ -726,6 +773,7 @@ private: static Function* convertElispDefun(TSNode node, const std::string& source) { auto* fn = new Function(); fn->id = IdGenerator::next("fn"); + applySpan(fn, node); // tree-sitter-elisp function_definition has: // field "name" → symbol (function name) @@ -761,6 +809,7 @@ private: if (expr) { auto* exprStmt = new ExpressionStatement(); exprStmt->id = IdGenerator::next("exprstmt"); + applySpan(exprStmt, child); exprStmt->setChild("expression", expr); fn->addChild("body", exprStmt); } @@ -819,11 +868,13 @@ private: // Last form — wrap in ExpressionStatement (implicit return) auto* exprStmt = new ExpressionStatement(); exprStmt->id = IdGenerator::next("exprstmt"); + applySpan(exprStmt, bodyChild); exprStmt->setChild("expression", expr); fn->addChild("body", exprStmt); } else { auto* exprStmt = new ExpressionStatement(); exprStmt->id = IdGenerator::next("exprstmt"); + applySpan(exprStmt, bodyChild); exprStmt->setChild("expression", expr); fn->addChild("body", exprStmt); } @@ -865,6 +916,7 @@ private: if (expr) { auto* exprStmt = new ExpressionStatement(); exprStmt->id = IdGenerator::next("exprstmt"); + applySpan(exprStmt, bodyChild); exprStmt->setChild("expression", expr); fn->addChild("body", exprStmt); } @@ -883,6 +935,7 @@ private: std::string type = nodeType(child); if (type == "symbol" || type == "identifier") { auto* param = new Parameter(IdGenerator::next("param"), nodeText(child, source)); + applySpan(param, child); fn->addChild("parameters", param); } } @@ -898,6 +951,7 @@ private: if (expr) { auto* exprStmt = new ExpressionStatement(); exprStmt->id = IdGenerator::next("exprstmt"); + applySpan(exprStmt, child); exprStmt->setChild("expression", expr); fn->addChild("body", exprStmt); } @@ -907,6 +961,7 @@ private: if (expr) { auto* exprStmt = new ExpressionStatement(); exprStmt->id = IdGenerator::next("exprstmt"); + applySpan(exprStmt, bodyNode); exprStmt->setChild("expression", expr); fn->addChild("body", exprStmt); } @@ -936,6 +991,7 @@ private: if (expr) { auto* exprStmt = new ExpressionStatement(); exprStmt->id = IdGenerator::next("exprstmt"); + applySpan(exprStmt, child); exprStmt->setChild("expression", expr); fn->addChild("body", exprStmt); } @@ -956,6 +1012,7 @@ private: auto* binOp = new BinaryOperation(); binOp->id = IdGenerator::next("binop"); binOp->op = firstText; + applySpan(binOp, node); ASTNode* left = convertElispExpression(ts_node_named_child(node, 1), source); ASTNode* right = convertElispExpression(ts_node_named_child(node, 2), source); if (left) binOp->setChild("left", left); @@ -967,6 +1024,7 @@ private: if (count >= 1) { auto* call = new FunctionCall(); call->id = IdGenerator::next("call"); + applySpan(call, node); TSNode funcName = ts_node_named_child(node, 0); call->functionName = nodeText(funcName, source); for (uint32_t i = 1; i < count; ++i) { @@ -976,14 +1034,20 @@ private: return call; } } else if (type == "symbol" || type == "identifier") { - return new VariableReference(IdGenerator::next("var"), nodeText(node, source)); + auto* ref = new VariableReference(IdGenerator::next("var"), nodeText(node, source)); + applySpan(ref, node); + return ref; } else if (type == "integer" || type == "number") { std::string text = nodeText(node, source); int val = 0; try { val = std::stoi(text); } catch (...) {} - return new IntegerLiteral(IdGenerator::next("int"), val); + auto* lit = new IntegerLiteral(IdGenerator::next("int"), val); + applySpan(lit, node); + return lit; } else if (type == "string") { - return new StringLiteral(IdGenerator::next("str"), nodeText(node, source)); + auto* lit = new StringLiteral(IdGenerator::next("str"), nodeText(node, source)); + applySpan(lit, node); + return lit; } else if (type == "special_form") { // Could be (if ...), (let ...), etc. return convertElispSpecialForm(node, source); @@ -992,7 +1056,9 @@ private: // Fallback std::string text = nodeText(node, source); if (!text.empty()) { - return new VariableReference(IdGenerator::next("var"), text); + auto* ref = new VariableReference(IdGenerator::next("var"), text); + applySpan(ref, node); + return ref; } return nullptr; } @@ -1007,6 +1073,7 @@ private: if (formName == "if" && count >= 3) { auto* ifStmt = new IfStatement(); ifStmt->id = IdGenerator::next("if"); + applySpan(ifStmt, node); ASTNode* cond = convertElispExpression(ts_node_named_child(node, 1), source); if (cond) ifStmt->setChild("condition", cond); return ifStmt; @@ -1015,6 +1082,7 @@ private: // Generic: treat as function call auto* call = new FunctionCall(); call->id = IdGenerator::next("call"); + applySpan(call, node); call->functionName = formName; for (uint32_t i = 1; i < count; ++i) { ASTNode* arg = convertElispExpression(ts_node_named_child(node, i), source); diff --git a/editor/src/ast/Serialization.h b/editor/src/ast/Serialization.h index d213270..a072d56 100644 --- a/editor/src/ast/Serialization.h +++ b/editor/src/ast/Serialization.h @@ -123,6 +123,12 @@ inline json toJson(const ASTNode* node) { j["id"] = node->id; j["concept"] = node->conceptType; j["properties"] = propertiesToJson(node); + if (node->hasSpan()) { + j["span"] = { + {"start", {{"line", node->spanStartLine}, {"col", node->spanStartCol}}}, + {"end", {{"line", node->spanEndLine}, {"col", node->spanEndCol}}} + }; + } json children = json::object(); for (const auto& role : node->childRoles()) { @@ -274,6 +280,17 @@ inline ASTNode* fromJson(const json& j) { if (!node) return nullptr; node->id = j["id"].get(); + if (j.contains("span")) { + const auto& span = j["span"]; + if (span.contains("start") && span.contains("end")) { + const auto& s = span["start"]; + const auto& e = span["end"]; + if (s.contains("line") && s.contains("col") && e.contains("line") && e.contains("col")) { + node->setSpan(s["line"].get(), s["col"].get(), + e["line"].get(), e["col"].get()); + } + } + } if (j.contains("properties")) { setPropertiesFromJson(node, j["properties"]); diff --git a/editor/tests/step94a_test.cpp b/editor/tests/step94a_test.cpp new file mode 100644 index 0000000..82670f2 --- /dev/null +++ b/editor/tests/step94a_test.cpp @@ -0,0 +1,39 @@ +// Step 94a TDD Test: AST source span tracking +// +// Tests: +// 1. Parser assigns spans to AST nodes +// 2. Serialization preserves spans + +#include +#include +#include "ast/Parser.h" +#include "ast/Serialization.h" + +int main() { + int passed = 0; + int failed = 0; + + std::string source = "def foo(x):\n return x\n"; + auto mod = TreeSitterParser::parsePython(source); + assert(mod); + auto funcs = mod->getChildren("functions"); + assert(!funcs.empty()); + auto* fn = funcs[0]; + assert(fn->hasSpan()); + assert(fn->spanStartLine == 0); + assert(fn->spanEndLine >= 1); + std::cout << "Test 1 PASS: parser spans set" << std::endl; + ++passed; + + auto j = toJson(fn); + assert(j.contains("span")); + assert(j["span"].contains("start")); + auto* round = fromJson(j); + assert(round->hasSpan()); + assert(round->spanStartLine == fn->spanStartLine); + std::cout << "Test 2 PASS: serialization preserves spans" << std::endl; + ++passed; + + std::cout << "\n=== Step 94a Results: " << passed << " passed, " << failed << " failed ===" << std::endl; + return failed > 0 ? 1 : 0; +} diff --git a/sprint4_plan.md b/sprint4_plan.md index 8b8797c..d47daba 100644 --- a/sprint4_plan.md +++ b/sprint4_plan.md @@ -168,6 +168,12 @@ This is the biggest architectural change from the original plan. Wire `Pipeline.h` for full analysis pass (debounced, cached). *Wires:* `AnnotationValidator.h`, `StrategyValidator.h`, `Pipeline.h` +- [ ] **Step 94a: Source span tracking** + Add source spans (start/end line/column) to AST nodes and propagate them from + tree-sitter CST nodes. Serialize spans in `Serialization.h`. Whetstone diagnostics + map to exact ranges using node spans (no heuristic line guessing). + *Modifies:* `ast/ASTNode.h`, `ast/Parser.h`, `ast/Serialization.h` + - [ ] **Step 95: Gutter markers and inline squiggles** Red circle (error) and yellow triangle (warning) icons in the gutter for lines with diagnostics (from both LSP and Whetstone). Hover shows message