From 93d84fd29bb4b23f79d0596aa06dcf56be9d2305 Mon Sep 17 00:00:00 2001 From: Bill Date: Mon, 16 Feb 2026 00:42:26 -0700 Subject: [PATCH] =?UTF-8?q?Step=20339:=20C++=20Parser=20=E2=80=94=20Prepro?= =?UTF-8?q?cessor,=20Enum,=20Namespace=20(12/12=20tests)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extended CppParser to parse #include, #pragma, #define, enum class/enum, namespace, using alias, and typedef from tree-sitter CST. Handles all preprocessor directive types and recursive namespace body parsing. Co-Authored-By: Claude Opus 4.6 --- editor/CMakeLists.txt | 9 ++ editor/src/ast/CppParser.h | 206 +++++++++++++++++++++++++ editor/src/ast/Parser.h | 2 + editor/tests/step339_test.cpp | 283 ++++++++++++++++++++++++++++++++++ progress.md | 21 +++ 5 files changed, 521 insertions(+) create mode 100644 editor/tests/step339_test.cpp diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index cef6cdf..74a546c 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -2035,4 +2035,13 @@ add_executable(step338_test tests/step338_test.cpp) target_include_directories(step338_test PRIVATE src) target_link_libraries(step338_test PRIVATE nlohmann_json::nlohmann_json) +add_executable(step339_test tests/step339_test.cpp) +target_include_directories(step339_test PRIVATE src) +target_link_libraries(step339_test PRIVATE + nlohmann_json::nlohmann_json + unofficial::tree-sitter::tree-sitter + tree_sitter_python tree_sitter_cpp tree_sitter_elisp + tree_sitter_javascript tree_sitter_typescript + tree_sitter_java tree_sitter_rust tree_sitter_go) + # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies) diff --git a/editor/src/ast/CppParser.h b/editor/src/ast/CppParser.h index e4074d8..3a9ab20 100644 --- a/editor/src/ast/CppParser.h +++ b/editor/src/ast/CppParser.h @@ -59,6 +59,42 @@ private: convertCppClassOrStruct(child, source, module, type == "struct_specifier"); } else if (type == "template_declaration") { convertCppTemplateDecl(child, source, module); + } else if (type == "preproc_include") { + auto* node = convertCppInclude(child, source); + if (node) module->addChild("statements", node); + } else if (type == "preproc_def") { + auto* node = convertCppMacroDef(child, source, false); + if (node) module->addChild("statements", node); + } else if (type == "preproc_function_def") { + auto* node = convertCppMacroDef(child, source, true); + if (node) module->addChild("statements", node); + } else if (type == "preproc_call") { + auto* node = convertCppPragma(child, source); + if (node) module->addChild("statements", node); + } else if (type == "enum_specifier") { + auto* node = convertCppEnum(child, source); + if (node) module->addChild("statements", node); + } else if (type == "namespace_definition") { + auto* node = convertCppNamespace(child, source); + if (node) module->addChild("statements", node); + } else if (type == "alias_declaration") { + auto* node = convertCppTypeAlias(child, source, true); + if (node) module->addChild("statements", node); + } else if (type == "type_definition") { + auto* node = convertCppTypeAlias(child, source, false); + if (node) module->addChild("statements", node); + } else if (type == "declaration") { + // Could be a using declaration or other top-level decl + // Check for alias_declaration child + uint32_t nc = ts_node_named_child_count(child); + for (uint32_t j = 0; j < nc; ++j) { + TSNode sub = ts_node_named_child(child, j); + std::string subType = nodeType(sub); + if (subType == "alias_declaration") { + auto* node = convertCppTypeAlias(sub, source, true); + if (node) module->addChild("statements", node); + } + } } } } @@ -569,4 +605,174 @@ private: } } + // --- Preprocessor, Enum, Namespace converters (Step 339) --- + + static IncludeDirective* convertCppInclude(TSNode node, const std::string& source) { + TSNode pathNode = childByFieldName(node, "path"); + if (ts_node_is_null(pathNode)) return nullptr; + std::string pathText = nodeText(pathNode, source); + bool isSystem = false; + if (!pathText.empty() && pathText[0] == '<') { + isSystem = true; + pathText = pathText.substr(1, pathText.size() - 2); + } else if (!pathText.empty() && pathText[0] == '"') { + pathText = pathText.substr(1, pathText.size() - 2); + } + auto* inc = new IncludeDirective(IdGenerator::next("inc"), pathText, isSystem); + applySpan(inc, node); + return inc; + } + + static MacroDefinition* convertCppMacroDef(TSNode node, const std::string& source, bool isFunctionLike) { + TSNode nameNode = childByFieldName(node, "name"); + if (ts_node_is_null(nameNode)) return nullptr; + std::string name = nodeText(nameNode, source); + auto* mac = new MacroDefinition(IdGenerator::next("mac"), name); + mac->isFunctionLike = isFunctionLike; + applySpan(mac, node); + + TSNode valueNode = childByFieldName(node, "value"); + if (!ts_node_is_null(valueNode)) { + mac->body = nodeText(valueNode, source); + } + + if (isFunctionLike) { + TSNode paramsNode = childByFieldName(node, "parameters"); + if (!ts_node_is_null(paramsNode)) { + uint32_t pc = ts_node_named_child_count(paramsNode); + for (uint32_t i = 0; i < pc; ++i) { + TSNode p = ts_node_named_child(paramsNode, i); + mac->parameters.push_back(nodeText(p, source)); + } + } + } + return mac; + } + + static PragmaDirective* convertCppPragma(TSNode node, const std::string& source) { + TSNode dirNode = childByFieldName(node, "directive"); + if (ts_node_is_null(dirNode)) return nullptr; + std::string dirText = nodeText(dirNode, source); + if (dirText != "#pragma") return nullptr; + TSNode argNode = childByFieldName(node, "argument"); + std::string argText; + if (!ts_node_is_null(argNode)) { + argText = nodeText(argNode, source); + } + auto* prag = new PragmaDirective(IdGenerator::next("prag"), argText); + applySpan(prag, node); + return prag; + } + + static EnumDeclaration* convertCppEnum(TSNode node, const std::string& source) { + TSNode nameNode = childByFieldName(node, "name"); + std::string name; + if (!ts_node_is_null(nameNode)) name = nodeText(nameNode, source); + if (name.empty()) return nullptr; + + std::string fullText = nodeText(node, source); + bool isScoped = (fullText.find("enum class") != std::string::npos || + fullText.find("enum struct") != std::string::npos); + + auto* enumDecl = new EnumDeclaration(IdGenerator::next("enum"), name, isScoped); + applySpan(enumDecl, node); + + // Check for underlying type + TSNode underlyingNode = childByFieldName(node, "underlying_type"); + if (!ts_node_is_null(underlyingNode)) { + enumDecl->underlyingType = nodeText(underlyingNode, source); + } else { + // Parse from text: "enum class Name : uint8_t {" + auto colonPos = fullText.find(':'); + auto bracePos = fullText.find('{'); + if (colonPos != std::string::npos && bracePos != std::string::npos && colonPos < bracePos) { + std::string afterColon = fullText.substr(colonPos + 1, bracePos - colonPos - 1); + while (!afterColon.empty() && afterColon[0] == ' ') afterColon.erase(0, 1); + while (!afterColon.empty() && afterColon.back() == ' ') afterColon.pop_back(); + if (!afterColon.empty()) enumDecl->underlyingType = afterColon; + } + } + + TSNode bodyNode = childByFieldName(node, "body"); + if (!ts_node_is_null(bodyNode)) { + uint32_t bc = ts_node_named_child_count(bodyNode); + for (uint32_t i = 0; i < bc; ++i) { + TSNode member = ts_node_named_child(bodyNode, i); + std::string mtype = nodeType(member); + if (mtype == "enumerator") { + TSNode mNameNode = childByFieldName(member, "name"); + std::string mName; + if (!ts_node_is_null(mNameNode)) mName = nodeText(mNameNode, source); + std::string mValue; + TSNode mValueNode = childByFieldName(member, "value"); + if (!ts_node_is_null(mValueNode)) mValue = nodeText(mValueNode, source); + if (!mName.empty()) { + auto* em = new EnumMember(IdGenerator::next("em"), mName, mValue); + applySpan(em, member); + enumDecl->addChild("members", em); + } + } + } + } + return enumDecl; + } + + static NamespaceDeclaration* convertCppNamespace(TSNode node, const std::string& source) { + TSNode nameNode = childByFieldName(node, "name"); + std::string name; + if (!ts_node_is_null(nameNode)) name = nodeText(nameNode, source); + + auto* ns = new NamespaceDeclaration(IdGenerator::next("ns"), name); + applySpan(ns, node); + + TSNode bodyNode = childByFieldName(node, "body"); + if (!ts_node_is_null(bodyNode)) { + uint32_t bc = ts_node_named_child_count(bodyNode); + for (uint32_t i = 0; i < bc; ++i) { + TSNode child = ts_node_named_child(bodyNode, i); + std::string ctype = nodeType(child); + if (ctype == "function_definition") { + auto* fn = convertCppFunction(child, source); + if (fn) ns->addChild("body", fn); + } else if (ctype == "class_specifier" || ctype == "struct_specifier") { + auto tmpMod = std::make_unique(); + convertCppClassOrStruct(child, source, tmpMod.get(), ctype == "struct_specifier"); + for (auto* cls : tmpMod->getChildren("classes")) + ns->addChild("body", cls); + } else if (ctype == "enum_specifier") { + auto* e = convertCppEnum(child, source); + if (e) ns->addChild("body", e); + } else if (ctype == "namespace_definition") { + auto* inner = convertCppNamespace(child, source); + if (inner) ns->addChild("body", inner); + } + } + } + return ns; + } + + static TypeAlias* convertCppTypeAlias(TSNode node, const std::string& source, bool isUsing) { + if (isUsing) { + TSNode nameNode = childByFieldName(node, "name"); + TSNode typeNode = childByFieldName(node, "type"); + std::string name, target; + if (!ts_node_is_null(nameNode)) name = nodeText(nameNode, source); + if (!ts_node_is_null(typeNode)) target = nodeText(typeNode, source); + if (name.empty()) return nullptr; + auto* ta = new TypeAlias(IdGenerator::next("ta"), name, target, true); + applySpan(ta, node); + return ta; + } else { + TSNode typeNode = childByFieldName(node, "type"); + TSNode declNode = childByFieldName(node, "declarator"); + std::string target, name; + if (!ts_node_is_null(typeNode)) target = nodeText(typeNode, source); + if (!ts_node_is_null(declNode)) name = nodeText(declNode, source); + if (name.empty()) return nullptr; + auto* ta = new TypeAlias(IdGenerator::next("ta"), name, target, false); + applySpan(ta, node); + return ta; + } + } + // --------------------------------------------------------------- diff --git a/editor/src/ast/Parser.h b/editor/src/ast/Parser.h index 0b55bb7..b6ae73d 100644 --- a/editor/src/ast/Parser.h +++ b/editor/src/ast/Parser.h @@ -12,6 +12,8 @@ #include "ClassDeclaration.h" #include "GenericType.h" #include "AsyncNodes.h" +#include "PreprocessorNodes.h" +#include "EnumNamespaceNodes.h" #include #include #include diff --git a/editor/tests/step339_test.cpp b/editor/tests/step339_test.cpp new file mode 100644 index 0000000..21cc2cb --- /dev/null +++ b/editor/tests/step339_test.cpp @@ -0,0 +1,283 @@ +// Step 339: C++ Parser — Preprocessor, Enum, Namespace (12 tests) +// Tests tree-sitter C++ parsing of preprocessor directives, enums, namespaces, type aliases + +#include +#include +#include +#include "ast/PreprocessorNodes.h" +#include "ast/EnumNamespaceNodes.h" +#include "ast/ClassDeclaration.h" +#include "ast/AsyncNodes.h" +#include "ast/Serialization.h" +#include "ast/Parser.h" + +int main() { + int passed = 0; + + // Test 1: #include system parsed + { + std::string src = "#include \n"; + auto mod = TreeSitterParser::parseCpp(src); + auto stmts = mod->getChildren("statements"); + assert(!stmts.empty()); + bool found = false; + for (auto* s : stmts) { + if (s->conceptType == "IncludeDirective") { + auto* inc = static_cast(s); + assert(inc->isSystem); + assert(inc->path == "vector"); + found = true; + } + } + assert(found); + std::cout << "Test 1 PASSED: #include parsed\n"; + passed++; + } + + // Test 2: #include local parsed + { + std::string src = "#include \"ast/ASTNode.h\"\n"; + auto mod = TreeSitterParser::parseCpp(src); + auto stmts = mod->getChildren("statements"); + bool found = false; + for (auto* s : stmts) { + if (s->conceptType == "IncludeDirective") { + auto* inc = static_cast(s); + assert(!inc->isSystem); + assert(inc->path == "ast/ASTNode.h"); + found = true; + } + } + assert(found); + std::cout << "Test 2 PASSED: #include \"ast/ASTNode.h\" parsed\n"; + passed++; + } + + // Test 3: #pragma once + { + std::string src = "#pragma once\n"; + auto mod = TreeSitterParser::parseCpp(src); + auto stmts = mod->getChildren("statements"); + bool found = false; + for (auto* s : stmts) { + if (s->conceptType == "PragmaDirective") { + auto* prag = static_cast(s); + assert(prag->directive == "once"); + found = true; + } + } + assert(found); + std::cout << "Test 3 PASSED: #pragma once parsed\n"; + passed++; + } + + // Test 4: #define object-like macro + { + std::string src = "#define MAX_SIZE 1024\n"; + auto mod = TreeSitterParser::parseCpp(src); + auto stmts = mod->getChildren("statements"); + bool found = false; + for (auto* s : stmts) { + if (s->conceptType == "MacroDefinition") { + auto* mac = static_cast(s); + assert(mac->name == "MAX_SIZE"); + assert(!mac->isFunctionLike); + assert(mac->body.find("1024") != std::string::npos); + found = true; + } + } + assert(found); + std::cout << "Test 4 PASSED: #define object-like macro\n"; + passed++; + } + + // Test 5: #define function-like macro + { + std::string src = "#define MIN(a, b) ((a) < (b) ? (a) : (b))\n"; + auto mod = TreeSitterParser::parseCpp(src); + auto stmts = mod->getChildren("statements"); + bool found = false; + for (auto* s : stmts) { + if (s->conceptType == "MacroDefinition") { + auto* mac = static_cast(s); + assert(mac->name == "MIN"); + assert(mac->isFunctionLike); + assert(mac->parameters.size() >= 2); + found = true; + } + } + assert(found); + std::cout << "Test 5 PASSED: #define function-like macro\n"; + passed++; + } + + // Test 6: enum class with values + { + std::string src = "enum class Color { Red = 0, Green = 1, Blue = 2 };\n"; + auto mod = TreeSitterParser::parseCpp(src); + auto stmts = mod->getChildren("statements"); + bool found = false; + for (auto* s : stmts) { + if (s->conceptType == "EnumDeclaration") { + auto* e = static_cast(s); + assert(e->name == "Color"); + assert(e->isScoped); + auto members = e->getChildren("members"); + assert(members.size() == 3); + assert(static_cast(members[0])->name == "Red"); + assert(static_cast(members[0])->value == "0"); + assert(static_cast(members[2])->name == "Blue"); + found = true; + } + } + assert(found); + std::cout << "Test 6 PASSED: enum class with values\n"; + passed++; + } + + // Test 7: plain enum + { + std::string src = "enum Flags { A, B, C };\n"; + auto mod = TreeSitterParser::parseCpp(src); + auto stmts = mod->getChildren("statements"); + bool found = false; + for (auto* s : stmts) { + if (s->conceptType == "EnumDeclaration") { + auto* e = static_cast(s); + assert(e->name == "Flags"); + assert(!e->isScoped); + auto members = e->getChildren("members"); + assert(members.size() == 3); + found = true; + } + } + assert(found); + std::cout << "Test 7 PASSED: plain enum parsed\n"; + passed++; + } + + // Test 8: namespace with contents + { + std::string src = R"( +namespace whetstone { + void helper() {} +} +)"; + auto mod = TreeSitterParser::parseCpp(src); + auto stmts = mod->getChildren("statements"); + bool found = false; + for (auto* s : stmts) { + if (s->conceptType == "NamespaceDeclaration") { + auto* ns = static_cast(s); + assert(ns->name == "whetstone"); + auto body = ns->getChildren("body"); + assert(!body.empty()); + assert(body[0]->conceptType == "Function"); + found = true; + } + } + assert(found); + std::cout << "Test 8 PASSED: namespace with contents\n"; + passed++; + } + + // Test 9: using alias + { + std::string src = "using StringVec = std::vector;\n"; + auto mod = TreeSitterParser::parseCpp(src); + auto stmts = mod->getChildren("statements"); + bool found = false; + for (auto* s : stmts) { + if (s->conceptType == "TypeAlias") { + auto* ta = static_cast(s); + assert(ta->aliasName == "StringVec"); + assert(ta->isUsing); + assert(ta->targetType.find("vector") != std::string::npos); + found = true; + } + } + assert(found); + std::cout << "Test 9 PASSED: using alias parsed\n"; + passed++; + } + + // Test 10: typedef + { + std::string src = "typedef unsigned int UINT;\n"; + auto mod = TreeSitterParser::parseCpp(src); + auto stmts = mod->getChildren("statements"); + bool found = false; + for (auto* s : stmts) { + if (s->conceptType == "TypeAlias") { + auto* ta = static_cast(s); + assert(ta->aliasName == "UINT"); + assert(!ta->isUsing); + found = true; + } + } + assert(found); + std::cout << "Test 10 PASSED: typedef parsed\n"; + passed++; + } + + // Test 11: Mixed file with preprocessor + namespace + enum + functions + { + std::string src = R"( +#pragma once +#include +#define VERSION 1 + +namespace ast { + enum class NodeType { Expression, Statement, Declaration }; + void process() {} +} +)"; + auto mod = TreeSitterParser::parseCpp(src); + auto stmts = mod->getChildren("statements"); + int pragmaCount = 0, includeCount = 0, macroCount = 0, nsCount = 0; + for (auto* s : stmts) { + if (s->conceptType == "PragmaDirective") pragmaCount++; + if (s->conceptType == "IncludeDirective") includeCount++; + if (s->conceptType == "MacroDefinition") macroCount++; + if (s->conceptType == "NamespaceDeclaration") nsCount++; + } + assert(pragmaCount >= 1); + assert(includeCount >= 1); + assert(macroCount >= 1); + assert(nsCount >= 1); + std::cout << "Test 11 PASSED: Mixed file parsed\n"; + passed++; + } + + // Test 12: Backward compat — existing function parsing still works + { + std::string src = R"( +#include + +class Widget : public Base { +public: + void render() {} +}; + +void standalone() {} +)"; + auto mod = TreeSitterParser::parseCpp(src); + auto classes = mod->getChildren("classes"); + auto funcs = mod->getChildren("functions"); + auto stmts = mod->getChildren("statements"); + assert(!classes.empty()); + assert(static_cast(classes[0])->name == "Widget"); + assert(!funcs.empty()); + // Should also have the include + bool hasInclude = false; + for (auto* s : stmts) { + if (s->conceptType == "IncludeDirective") hasInclude = true; + } + assert(hasInclude); + std::cout << "Test 12 PASSED: Backward compat\n"; + passed++; + } + + std::cout << "\nResults: " << passed << "/12 tests passed\n"; + return (passed == 12) ? 0 : 1; +} diff --git a/progress.md b/progress.md index 6a8c857..89ec6c1 100644 --- a/progress.md +++ b/progress.md @@ -1605,6 +1605,27 @@ vs typedef). Full JSON serialization roundtrip and CompactAST names for all 4 ty NamespaceDeclaration, TypeAlias - `editor/CMakeLists.txt` — step338_test target +### Step 339: C++ Parser — Preprocessor, Enum, Namespace +**Status:** PASS (12/12 tests) + +Extended CppParser to parse preprocessor directives (#include, #pragma, #define), +enums (scoped/unscoped with values), namespaces (with recursive body parsing), and +type aliases (using/typedef) from tree-sitter CST. Handles preproc_include, +preproc_def, preproc_function_def, preproc_call, enum_specifier, namespace_definition, +alias_declaration, and type_definition node types. + +**Files modified:** +- `editor/src/ast/Parser.h` — Added PreprocessorNodes.h, EnumNamespaceNodes.h includes +- `editor/src/ast/CppParser.h` — Extended convertCppTranslationUnit dispatch, added + convertCppInclude, convertCppMacroDef, convertCppPragma, convertCppEnum, + convertCppNamespace, convertCppTypeAlias converter functions +- `editor/CMakeLists.txt` — step339_test target with tree-sitter libraries + +**Files created:** +- `editor/tests/step339_test.cpp` — 12 tests: system/local include, pragma once, + object-like/function-like macros, scoped/plain enums, namespace with body, using alias, + typedef, mixed file, backward compat + --- # Roadmap Planning — Sprints 12-25+