From 8c1eeae89754ad752e49b70d950072bb552508b8 Mon Sep 17 00:00:00 2001 From: Bill Date: Mon, 16 Feb 2026 11:08:20 -0700 Subject: [PATCH] Step 370: Add Common Lisp parser and routing --- editor/CMakeLists.txt | 9 + editor/src/Pipeline.h | 5 + editor/src/ast/CommonLispParser.h | 598 ++++++++++++++++++++++++++++++ editor/src/ast/Parser.h | 1 + editor/tests/step370_test.cpp | 219 +++++++++++ progress.md | 46 +++ 6 files changed, 878 insertions(+) create mode 100644 editor/src/ast/CommonLispParser.h create mode 100644 editor/tests/step370_test.cpp diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 80fd7d3..1e27f1a 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -2209,4 +2209,13 @@ target_link_libraries(step369_test PRIVATE tree_sitter_javascript tree_sitter_typescript tree_sitter_java tree_sitter_rust tree_sitter_go) +add_executable(step370_test tests/step370_test.cpp) +target_include_directories(step370_test PRIVATE src) +target_link_libraries(step370_test PRIVATE + nlohmann_json::nlohmann_json + unofficial::tree-sitter::tree-sitter + tree_sitter_python tree_sitter_cpp tree_sitter_elisp + tree_sitter_javascript tree_sitter_typescript + tree_sitter_java tree_sitter_rust tree_sitter_go) + # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies) diff --git a/editor/src/Pipeline.h b/editor/src/Pipeline.h index 0510c83..120acc4 100644 --- a/editor/src/Pipeline.h +++ b/editor/src/Pipeline.h @@ -126,6 +126,11 @@ public: auto pr = WatParser::parseWatWithDiagnostics(source); diags = std::move(pr.diagnostics); return std::move(pr.module); + } else if (language == "common-lisp" || language == "commonlisp" || + language == "lisp" || language == "cl") { + auto pr = CommonLispParser::parseCommonLispWithDiagnostics(source); + diags = std::move(pr.diagnostics); + return std::move(pr.module); } return nullptr; } diff --git a/editor/src/ast/CommonLispParser.h b/editor/src/ast/CommonLispParser.h new file mode 100644 index 0000000..9744f0b --- /dev/null +++ b/editor/src/ast/CommonLispParser.h @@ -0,0 +1,598 @@ +#pragma once + +#include "ASTNode.h" +#include "Module.h" +#include "Function.h" +#include "Variable.h" +#include "Parameter.h" +#include "Type.h" +#include "ClassDeclaration.h" +#include "Statement.h" +#include "Expression.h" +#include "AsyncNodes.h" +#include "PreprocessorNodes.h" +#include "Annotation.h" +#include "../ast/Parser.h" +#include +#include +#include +#include +#include + +class CommonLispParser { +public: + static std::unique_ptr parseCommonLisp(const std::string& source) { + ParseResult result = parseCommonLispWithDiagnostics(source); + return std::move(result.module); + } + + static ParseResult parseCommonLispWithDiagnostics(const std::string& source) { + ParseResult result; + result.module = std::make_unique(); + result.module->id = IdGenerator::next("mod"); + result.module->name = "parsed_common_lisp_module"; + result.module->targetLanguage = "common-lisp"; + + std::vector forms = parseForms(source, result.diagnostics); + for (const auto& form : forms) { + convertTopLevelForm(form, result.module.get()); + } + return result; + } + +private: + struct SExpr { + bool isList = false; + std::string atom; + std::vector list; + }; + + static bool isWhitespace(char c) { + return c == ' ' || c == '\t' || c == '\r' || c == '\n'; + } + + static bool isDelimiter(char c) { + return isWhitespace(c) || c == '(' || c == ')' || c == '\'' || c == '`' || c == ';'; + } + + static std::string canonicalHead(const std::string& symbol) { + size_t pos = symbol.rfind(':'); + if (pos == std::string::npos) return symbol; + return symbol.substr(pos + 1); + } + + static bool isInteger(const std::string& s) { + if (s.empty()) return false; + size_t i = (s[0] == '-' || s[0] == '+') ? 1 : 0; + if (i >= s.size()) return false; + for (; i < s.size(); ++i) { + if (!std::isdigit(static_cast(s[i]))) return false; + } + return true; + } + + static bool isFloat(const std::string& s) { + if (s.empty()) return false; + bool seenDigit = false; + bool seenDot = false; + size_t i = (s[0] == '-' || s[0] == '+') ? 1 : 0; + if (i >= s.size()) return false; + for (; i < s.size(); ++i) { + char c = s[i]; + if (std::isdigit(static_cast(c))) { + seenDigit = true; + } else if (c == '.' && !seenDot) { + seenDot = true; + } else { + return false; + } + } + return seenDigit && seenDot; + } + + static bool isStringToken(const std::string& s) { + return s.size() >= 2 && s.front() == '"' && s.back() == '"'; + } + + static bool isEarmuff(const std::string& symbol) { + return symbol.size() > 2 && symbol.front() == '*' && symbol.back() == '*'; + } + + static PrimitiveType* primitiveType(const std::string& name) { + return new PrimitiveType(IdGenerator::next("type"), name); + } + + static ASTNode* mapTypeName(const std::string& tname) { + std::string t = canonicalHead(tname); + if (t == "fixnum" || t == "integer") return primitiveType("int"); + if (t == "single-float" || t == "double-float" || t == "float") return primitiveType("float"); + if (t == "string") return primitiveType("string"); + if (t == "t") return primitiveType("any"); + if (t == "nil") return primitiveType("nil"); + return new CustomType(IdGenerator::next("type"), t); + } + + static void skipWhitespaceAndComments(const std::string& source, size_t& pos) { + while (pos < source.size()) { + if (isWhitespace(source[pos])) { + ++pos; + continue; + } + if (source[pos] == ';') { + while (pos < source.size() && source[pos] != '\n') ++pos; + continue; + } + break; + } + } + + static std::string parseStringToken(const std::string& source, size_t& pos, + std::vector& diags) { + std::string out; + out.push_back('"'); + ++pos; + bool escaped = false; + while (pos < source.size()) { + char c = source[pos++]; + out.push_back(c); + if (escaped) { + escaped = false; + continue; + } + if (c == '\\') { + escaped = true; + } else if (c == '"') { + return out; + } + } + diags.push_back({1, 1, "Unterminated string in Common Lisp source", "error"}); + return out; + } + + static std::string parseAtomToken(const std::string& source, size_t& pos) { + size_t start = pos; + while (pos < source.size() && !isDelimiter(source[pos])) ++pos; + return source.substr(start, pos - start); + } + + static SExpr makeQuotedForm(const SExpr& value) { + SExpr quoted; + quoted.isList = true; + SExpr quoteSymbol; + quoteSymbol.atom = "quote"; + quoted.list.push_back(quoteSymbol); + quoted.list.push_back(value); + return quoted; + } + + static SExpr parseSingleForm(const std::string& source, size_t& pos, + std::vector& diags) { + skipWhitespaceAndComments(source, pos); + if (pos >= source.size()) return SExpr{}; + + if (source[pos] == '(') { + ++pos; + SExpr list; + list.isList = true; + for (;;) { + skipWhitespaceAndComments(source, pos); + if (pos >= source.size()) { + diags.push_back({1, 1, "Unbalanced parentheses in Common Lisp source", "error"}); + return list; + } + if (source[pos] == ')') { + ++pos; + return list; + } + list.list.push_back(parseSingleForm(source, pos, diags)); + } + } + + if (source[pos] == ')') { + diags.push_back({1, 1, "Unmatched closing parenthesis in Common Lisp source", "error"}); + ++pos; + return SExpr{}; + } + + if (source[pos] == '\'' || source[pos] == '`') { + ++pos; + SExpr inner = parseSingleForm(source, pos, diags); + return makeQuotedForm(inner); + } + + if (source[pos] == '"') { + SExpr atom; + atom.atom = parseStringToken(source, pos, diags); + return atom; + } + + SExpr atom; + atom.atom = parseAtomToken(source, pos); + return atom; + } + + static std::vector parseForms(const std::string& source, + std::vector& diags) { + std::vector out; + size_t pos = 0; + while (pos < source.size()) { + skipWhitespaceAndComments(source, pos); + if (pos >= source.size()) break; + out.push_back(parseSingleForm(source, pos, diags)); + } + return out; + } + + static std::string atomText(const SExpr& expr) { + return expr.isList ? "" : expr.atom; + } + + static std::string headSymbol(const SExpr& expr) { + if (!expr.isList || expr.list.empty() || expr.list[0].isList) return ""; + return canonicalHead(expr.list[0].atom); + } + + static void appendExpressionStatement(ASTNode* owner, const std::string& role, ASTNode* expr) { + if (!expr) return; + auto* stmt = new ExpressionStatement(); + stmt->id = IdGenerator::next("exprstmt"); + stmt->setChild("expression", expr); + owner->addChild(role, stmt); + } + + static ASTNode* convertIfExpression(const SExpr& expr) { + if (expr.list.size() < 3) return nullptr; + auto* node = new IfStatement(); + node->id = IdGenerator::next("if"); + node->setChild("condition", convertExpression(expr.list[1])); + + appendExpressionStatement(node, "thenBranch", convertExpression(expr.list[2])); + if (expr.list.size() > 3) { + appendExpressionStatement(node, "elseBranch", convertExpression(expr.list[3])); + } + return node; + } + + static ASTNode* convertCondExpression(const SExpr& expr) { + if (expr.list.size() < 2) return nullptr; + IfStatement* root = nullptr; + IfStatement* current = nullptr; + + for (size_t i = 1; i < expr.list.size(); ++i) { + const SExpr& clause = expr.list[i]; + if (!clause.isList || clause.list.empty()) continue; + + auto* node = new IfStatement(); + node->id = IdGenerator::next("if"); + node->setChild("condition", convertExpression(clause.list[0])); + if (clause.list.size() >= 2) { + appendExpressionStatement(node, "thenBranch", convertExpression(clause.list[1])); + } + + if (!root) root = node; + if (current) current->addChild("elseBranch", node); + current = node; + } + + return root; + } + + static ASTNode* convertLambdaExpression(const SExpr& expr) { + auto* lambda = new LambdaExpression(); + lambda->id = IdGenerator::next("lambda"); + + if (expr.list.size() > 1 && expr.list[1].isList) { + for (const auto& p : expr.list[1].list) { + if (p.isList || p.atom.empty()) continue; + std::string name = p.atom; + if (!name.empty() && name[0] == '&') continue; + auto* param = new Parameter(IdGenerator::next("param"), name); + lambda->addChild("parameters", param); + } + } + + for (size_t i = 2; i < expr.list.size(); ++i) { + appendExpressionStatement(lambda, "body", convertExpression(expr.list[i])); + } + return lambda; + } + + static ASTNode* convertLetExpression(const SExpr& expr) { + auto* block = new Block(); + block->id = IdGenerator::next("blk"); + + if (expr.list.size() > 1 && expr.list[1].isList) { + for (const auto& binding : expr.list[1].list) { + if (!binding.isList || binding.list.empty()) continue; + std::string name = atomText(binding.list[0]); + if (name.empty()) continue; + + auto* var = new Variable(IdGenerator::next("var"), name); + if (binding.list.size() > 1) { + var->setChild("value", convertExpression(binding.list[1])); + } + block->addChild("statements", var); + } + } + + for (size_t i = 2; i < expr.list.size(); ++i) { + appendExpressionStatement(block, "statements", convertExpression(expr.list[i])); + } + return block; + } + + static ASTNode* convertLoopForm(const SExpr& expr) { + auto* loop = new ForLoop(); + loop->id = IdGenerator::next("for"); + std::string head = headSymbol(expr); + + if (head == "dotimes" && expr.list.size() > 1 && expr.list[1].isList && !expr.list[1].list.empty()) { + loop->iteratorName = atomText(expr.list[1].list[0]); + if (expr.list[1].list.size() > 1) { + loop->setChild("iterable", convertExpression(expr.list[1].list[1])); + } + } else { + loop->iteratorName = head; + } + + size_t start = (head == "dotimes") ? 2 : 1; + for (size_t i = start; i < expr.list.size(); ++i) { + appendExpressionStatement(loop, "body", convertExpression(expr.list[i])); + } + return loop; + } + + static void attachQuotedAnnotation(ASTNode* node) { + if (!node) return; + auto* meta = new MetaAnnotation(); + meta->id = IdGenerator::next("anno"); + meta->state = "quoted"; + meta->phase = "compile"; + node->addChild("annotations", meta); + } + + static ASTNode* convertCallLike(const SExpr& expr) { + if (!expr.isList || expr.list.empty()) return nullptr; + auto* call = new FunctionCall(); + call->id = IdGenerator::next("call"); + + std::string head = headSymbol(expr); + if (head == "funcall" || head == "apply") { + if (expr.list.size() > 1) call->functionName = atomText(expr.list[1]); + for (size_t i = 2; i < expr.list.size(); ++i) { + ASTNode* arg = convertExpression(expr.list[i]); + if (arg) call->addChild("arguments", arg); + } + return call; + } + + call->functionName = atomText(expr.list[0]); + for (size_t i = 1; i < expr.list.size(); ++i) { + ASTNode* arg = convertExpression(expr.list[i]); + if (arg) call->addChild("arguments", arg); + } + return call; + } + + static ASTNode* convertValuesExpression(const SExpr& expr) { + auto* call = new FunctionCall(); + call->id = IdGenerator::next("call"); + call->functionName = "values"; + for (size_t i = 1; i < expr.list.size(); ++i) { + ASTNode* arg = convertExpression(expr.list[i]); + if (arg) call->addChild("arguments", arg); + } + + auto* contract = new ContractAnnotation(); + contract->id = IdGenerator::next("anno"); + contract->returnShape = "multiple-values"; + call->addChild("annotations", contract); + return call; + } + + static ASTNode* convertExpression(const SExpr& expr) { + if (!expr.isList) { + std::string atom = expr.atom; + if (atom.empty()) return nullptr; + if (isInteger(atom)) return new IntegerLiteral(IdGenerator::next("int"), std::stoi(atom)); + if (isFloat(atom)) return new FloatLiteral(IdGenerator::next("flt"), atom); + if (isStringToken(atom)) return new StringLiteral(IdGenerator::next("str"), atom.substr(1, atom.size() - 2)); + if (atom == "t") return new BooleanLiteral(IdGenerator::next("bool"), true); + if (atom == "nil") return new BooleanLiteral(IdGenerator::next("bool"), false); + return new VariableReference(IdGenerator::next("ref"), atom); + } + + std::string head = headSymbol(expr); + if (head == "if") return convertIfExpression(expr); + if (head == "cond") return convertCondExpression(expr); + if (head == "lambda") return convertLambdaExpression(expr); + if (head == "let" || head == "let*") return convertLetExpression(expr); + if (head == "loop" || head == "do" || head == "dotimes") return convertLoopForm(expr); + if (head == "quote") { + ASTNode* quotedExpr = (expr.list.size() > 1) ? convertExpression(expr.list[1]) : nullptr; + if (!quotedExpr) quotedExpr = new VariableReference(IdGenerator::next("ref"), "nil"); + attachQuotedAnnotation(quotedExpr); + return quotedExpr; + } + if (head == "values") return convertValuesExpression(expr); + return convertCallLike(expr); + } + + static void parseDeclareTypeForms( + const std::vector& body, + std::unordered_map& declaredTypes) { + for (const auto& form : body) { + if (!form.isList || headSymbol(form) != "declare") continue; + for (size_t i = 1; i < form.list.size(); ++i) { + const SExpr& decl = form.list[i]; + if (!decl.isList || decl.list.size() < 3) continue; + if (headSymbol(decl) != "type") continue; + std::string tname = atomText(decl.list[1]); + for (size_t j = 2; j < decl.list.size(); ++j) { + std::string var = atomText(decl.list[j]); + if (!var.empty()) declaredTypes[var] = tname; + } + } + } + } + + static void addDynamicBindingAnnotationIfNeeded(ASTNode* node, const std::string& symbol) { + if (!isEarmuff(symbol)) return; + auto* binding = new BindingAnnotation(); + binding->id = IdGenerator::next("anno"); + binding->time = "dynamic"; + node->addChild("annotations", binding); + } + + static void convertDefun(const SExpr& form, Module* module) { + if (form.list.size() < 3) return; + std::string name = atomText(form.list[1]); + auto* fn = new Function(IdGenerator::next("fn"), name); + + if (form.list[2].isList) { + for (const auto& p : form.list[2].list) { + if (p.isList || p.atom.empty() || p.atom[0] == '&') continue; + auto* param = new Parameter(IdGenerator::next("param"), p.atom); + fn->addChild("parameters", param); + } + } + + std::vector body; + for (size_t i = 3; i < form.list.size(); ++i) body.push_back(form.list[i]); + + std::unordered_map declaredTypes; + parseDeclareTypeForms(body, declaredTypes); + for (ASTNode* p : fn->getChildren("parameters")) { + auto* param = static_cast(p); + auto it = declaredTypes.find(param->name); + if (it != declaredTypes.end()) { + param->setChild("type", mapTypeName(it->second)); + } + } + + for (const auto& expr : body) { + if (expr.isList && headSymbol(expr) == "declare") continue; + appendExpressionStatement(fn, "body", convertExpression(expr)); + } + module->addChild("functions", fn); + } + + static void convertDefmethod(const SExpr& form, Module* module) { + if (form.list.size() < 3) return; + std::string name = atomText(form.list[1]); + auto* method = new MethodDeclaration(IdGenerator::next("method"), name); + + if (form.list[2].isList) { + for (const auto& paramForm : form.list[2].list) { + if (!paramForm.isList) { + std::string pname = atomText(paramForm); + if (pname.empty()) continue; + auto* p = new Parameter(IdGenerator::next("param"), pname); + method->addChild("parameters", p); + continue; + } + if (paramForm.list.empty()) continue; + std::string pname = atomText(paramForm.list[0]); + if (pname.empty()) continue; + + auto* p = new Parameter(IdGenerator::next("param"), pname); + if (paramForm.list.size() > 1) { + std::string cls = atomText(paramForm.list[1]); + p->setChild("type", new CustomType(IdGenerator::next("type"), cls)); + if (method->className.empty()) method->className = cls; + } + method->addChild("parameters", p); + } + } + + for (size_t i = 3; i < form.list.size(); ++i) { + appendExpressionStatement(method, "body", convertExpression(form.list[i])); + } + module->addChild("functions", method); + } + + static void convertDefclass(const SExpr& form, Module* module) { + if (form.list.size() < 2) return; + auto* cls = new ClassDeclaration(IdGenerator::next("cls"), atomText(form.list[1])); + + if (form.list.size() > 2 && form.list[2].isList) { + for (const auto& base : form.list[2].list) { + std::string b = atomText(base); + if (!b.empty()) cls->addBase(b); + } + } + + if (form.list.size() > 3 && form.list[3].isList) { + for (const auto& slot : form.list[3].list) { + if (!slot.isList || slot.list.empty()) continue; + std::string fieldName = atomText(slot.list[0]); + if (fieldName.empty()) continue; + auto* field = new Variable(IdGenerator::next("var"), fieldName); + cls->addChild("fields", field); + } + } + + module->addChild("classes", cls); + } + + static void convertDefvar(const SExpr& form, Module* module) { + if (form.list.size() < 2) return; + std::string name = atomText(form.list[1]); + auto* var = new Variable(IdGenerator::next("var"), name); + + addDynamicBindingAnnotationIfNeeded(var, name); + if (form.list.size() > 2) { + var->setChild("value", convertExpression(form.list[2])); + } + module->addChild("variables", var); + } + + static void convertDefmacro(const SExpr& form, Module* module) { + if (form.list.size() < 2) return; + auto* macro = new MacroDefinition(IdGenerator::next("mac"), atomText(form.list[1])); + macro->isFunctionLike = true; + + if (form.list.size() > 2 && form.list[2].isList) { + for (const auto& p : form.list[2].list) { + if (!p.isList && !p.atom.empty()) macro->parameters.push_back(p.atom); + } + } + + std::string body = "("; + for (size_t i = 3; i < form.list.size(); ++i) { + if (i > 3) body += ' '; + body += atomText(form.list[i]).empty() ? "form" : atomText(form.list[i]); + } + body += ")"; + macro->body = body; + module->addChild("statements", macro); + } + + static void convertTopLevelForm(const SExpr& form, Module* module) { + if (!form.isList || form.list.empty()) return; + std::string head = headSymbol(form); + + if (head == "defun") { + convertDefun(form, module); + return; + } + if (head == "defmethod") { + convertDefmethod(form, module); + return; + } + if (head == "defclass") { + convertDefclass(form, module); + return; + } + if (head == "defvar" || head == "defparameter") { + convertDefvar(form, module); + return; + } + if (head == "defmacro") { + convertDefmacro(form, module); + return; + } + + appendExpressionStatement(module, "statements", convertExpression(form)); + } +}; diff --git a/editor/src/ast/Parser.h b/editor/src/ast/Parser.h index ef05462..171bd40 100644 --- a/editor/src/ast/Parser.h +++ b/editor/src/ast/Parser.h @@ -166,3 +166,4 @@ private: #include "ast/CSharpParser.h" #include "ast/CParser.h" #include "ast/WatParser.h" +#include "ast/CommonLispParser.h" diff --git a/editor/tests/step370_test.cpp b/editor/tests/step370_test.cpp new file mode 100644 index 0000000..7d84600 --- /dev/null +++ b/editor/tests/step370_test.cpp @@ -0,0 +1,219 @@ +// Step 370: Common Lisp Parser (12 tests) + +#include +#include +#include +#include +#include "Pipeline.h" +#include "ast/CommonLispParser.h" +#include "ast/ClassDeclaration.h" + +static bool hasBodyNode(const Function* fn, const std::string& conceptType) { + for (auto* node : fn->getChildren("body")) { + if (node->conceptType == conceptType) return true; + if (node->conceptType == "ExpressionStatement") { + ASTNode* expr = node->getChild("expression"); + if (expr && expr->conceptType == conceptType) return true; + } + } + return false; +} + +int main() { + int passed = 0; + + // Test 1: defun parsing + { + std::string src = "(defun add (x y) (+ x y))"; + auto mod = CommonLispParser::parseCommonLisp(src); + assert(mod->targetLanguage == "common-lisp"); + assert(mod->getChildren("functions").size() == 1); + auto* fn = static_cast(mod->getChildren("functions")[0]); + assert(fn->name == "add"); + assert(fn->getChildren("parameters").size() == 2); + std::cout << "Test 1 PASSED: defun parsing\n"; + passed++; + } + + // Test 2: defclass parsing + { + std::string src = "(defclass point (shape) ((x) (y)))"; + auto mod = CommonLispParser::parseCommonLisp(src); + assert(mod->getChildren("classes").size() == 1); + auto* cls = static_cast(mod->getChildren("classes")[0]); + assert(cls->name == "point"); + assert(cls->getChildren("fields").size() == 2); + assert(!cls->getBases().empty() && cls->getBases()[0].name == "shape"); + std::cout << "Test 2 PASSED: defclass parsing\n"; + passed++; + } + + // Test 3: lambda parsing + { + std::string src = "(defun mk () (lambda (x) (+ x 1)))"; + auto mod = CommonLispParser::parseCommonLisp(src); + auto* fn = static_cast(mod->getChildren("functions")[0]); + assert(hasBodyNode(fn, "LambdaExpression")); + std::cout << "Test 3 PASSED: lambda parsing\n"; + passed++; + } + + // Test 4: let bindings + { + std::string src = "(defun f () (let ((x 1) (y 2)) (+ x y)))"; + auto mod = CommonLispParser::parseCommonLisp(src); + auto* fn = static_cast(mod->getChildren("functions")[0]); + bool foundBlock = false; + bool foundVar = false; + for (auto* stmt : fn->getChildren("body")) { + ASTNode* expr = stmt->getChild("expression"); + if (!expr || expr->conceptType != "Block") continue; + foundBlock = true; + for (auto* child : expr->getChildren("statements")) { + if (child->conceptType == "Variable") foundVar = true; + } + } + assert(foundBlock && foundVar); + std::cout << "Test 4 PASSED: let bindings\n"; + passed++; + } + + // Test 5: if and cond parsing + { + std::string src = + "(defun choose (x) (if x 1 0))\n" + "(defun choose2 (x) (cond ((> x 0) 1) ((< x 0) -1) (t 0)))"; + auto mod = CommonLispParser::parseCommonLisp(src); + auto* fn1 = static_cast(mod->getChildren("functions")[0]); + auto* fn2 = static_cast(mod->getChildren("functions")[1]); + assert(hasBodyNode(fn1, "IfStatement")); + assert(hasBodyNode(fn2, "IfStatement")); + std::cout << "Test 5 PASSED: if/cond parsing\n"; + passed++; + } + + // Test 6: loop variants + { + std::string src = + "(defun a () (loop for i from 0 to 3 do (print i)))\n" + "(defun b () (do ((i 0 (+ i 1))) ((> i 3) i)))\n" + "(defun c () (dotimes (i 3) (print i)))"; + auto mod = CommonLispParser::parseCommonLisp(src); + auto* fn1 = static_cast(mod->getChildren("functions")[0]); + auto* fn2 = static_cast(mod->getChildren("functions")[1]); + auto* fn3 = static_cast(mod->getChildren("functions")[2]); + assert(hasBodyNode(fn1, "ForLoop")); + assert(hasBodyNode(fn2, "ForLoop")); + assert(hasBodyNode(fn3, "ForLoop")); + std::cout << "Test 6 PASSED: loop/do/dotimes parsing\n"; + passed++; + } + + // Test 7: defmacro parsing + { + std::string src = "(defmacro when1 (test body) `(if ,test ,body nil))"; + auto mod = CommonLispParser::parseCommonLisp(src); + bool foundMacro = false; + for (auto* stmt : mod->getChildren("statements")) { + if (stmt->conceptType == "MacroDefinition") foundMacro = true; + } + assert(foundMacro); + std::cout << "Test 7 PASSED: defmacro parsing\n"; + passed++; + } + + // Test 8: quote and shorthand quote annotation + { + std::string src = "(defun q () (quote x) 'y)"; + auto mod = CommonLispParser::parseCommonLisp(src); + auto* fn = static_cast(mod->getChildren("functions")[0]); + bool hasQuoted = false; + for (auto* stmt : fn->getChildren("body")) { + ASTNode* expr = stmt->getChild("expression"); + if (!expr) continue; + for (auto* anno : expr->getChildren("annotations")) { + if (anno->conceptType == "MetaAnnotation") hasQuoted = true; + } + } + assert(hasQuoted); + std::cout << "Test 8 PASSED: quote annotation\n"; + passed++; + } + + // Test 9: earmuff dynamic variable detection + { + std::string src = "(defvar *runtime-mode* 1)"; + auto mod = CommonLispParser::parseCommonLisp(src); + assert(mod->getChildren("variables").size() == 1); + auto* var = static_cast(mod->getChildren("variables")[0]); + bool dynamic = false; + for (auto* anno : var->getChildren("annotations")) { + if (anno->conceptType == "BindingAnnotation") { + auto* b = static_cast(anno); + if (b->time == "dynamic") dynamic = true; + } + } + assert(dynamic); + std::cout << "Test 9 PASSED: earmuff dynamic variable detection\n"; + passed++; + } + + // Test 10: deep nested s-expressions + { + std::string src = "(defun deep (x) (a (b (c (d (e (f (g x))))))))"; + auto mod = CommonLispParser::parseCommonLisp(src); + auto* fn = static_cast(mod->getChildren("functions")[0]); + assert(hasBodyNode(fn, "FunctionCall")); + std::cout << "Test 10 PASSED: deep nesting parse\n"; + passed++; + } + + // Test 11: mixed definitions in one file + { + std::string src = + "(defclass user () ((name) (age)))\n" + "(defvar +limit+ 32)\n" + "(defmacro with-u (u body) `(let ((x ,u)) ,body))\n" + "(cl:defun my-package:run (n) n)"; + auto mod = CommonLispParser::parseCommonLisp(src); + assert(mod->getChildren("classes").size() == 1); + assert(mod->getChildren("variables").size() == 1); + assert(mod->getChildren("functions").size() == 1); + assert(static_cast(mod->getChildren("functions")[0])->name == "my-package:run"); + bool macroFound = false; + for (auto* stmt : mod->getChildren("statements")) { + if (stmt->conceptType == "MacroDefinition") macroFound = true; + } + assert(macroFound); + std::cout << "Test 11 PASSED: mixed definitions and package prefixes\n"; + passed++; + } + + // Test 12: CLOS method + declare type + pipeline route aliases + { + std::string src = + "(defmethod area ((obj rectangle))\n" + " (declare (type fixnum obj))\n" + " (values 1 2))"; + auto mod = CommonLispParser::parseCommonLisp(src); + assert(mod->getChildren("functions").size() == 1); + auto* method = static_cast(mod->getChildren("functions")[0]); + assert(method->conceptType == "MethodDeclaration"); + assert(method->className == "rectangle"); + auto* param = static_cast(method->getChildren("parameters")[0]); + assert(param->getChild("type") != nullptr); + + Pipeline p; + std::vector diags; + auto parsed1 = p.parse("(defun f (x) x)", "common-lisp", diags); + auto parsed2 = p.parse("(defun f (x) x)", "lisp", diags); + assert(parsed1 != nullptr && parsed2 != nullptr); + assert(parsed1->targetLanguage == "common-lisp"); + std::cout << "Test 12 PASSED: defmethod + declare + pipeline aliases\n"; + passed++; + } + + std::cout << "\nResults: " << passed << "/12\n"; + assert(passed == 12); + return 0; +} diff --git a/progress.md b/progress.md index 8710a9d..ad48526 100644 --- a/progress.md +++ b/progress.md @@ -2396,6 +2396,52 @@ persistence, and MCP contract checks. - `editor/src/ast/WatParser.h` remains within header-size limit after diagnostic additions (`248` lines <= `600`) +### Step 370: Common Lisp Parser +**Status:** PASS (12/12 tests) + +Added a dedicated Common Lisp parser with S-expression parsing and direct +mapping for core CL forms into the Whetstone AST, including CLOS and macro +surface forms. + +**Files created:** +- `editor/src/ast/CommonLispParser.h` — Common Lisp parse support: + - recursive S-expression reader with comment/string handling + - top-level form conversion for `defun`, `defmethod`, `defclass`, + `defvar`/`defparameter`, and `defmacro` + - expression conversion for `if`, `cond`, `let`, `lambda`, loop forms, + `funcall`/`apply`, `quote`, and `values` + - convention handling for package-qualified symbols and `*earmuff*` + dynamic-variable detection + - `(declare (type ...))` mapping to parameter type nodes +- `editor/tests/step370_test.cpp` — 12 tests covering: + 1. `defun` parsing + 2. `defclass` parsing + 3. `lambda` parsing + 4. `let` binding/scope mapping + 5. `if`/`cond` mapping + 6. `loop`/`do`/`dotimes` mapping + 7. `defmacro` mapping + 8. quote shorthand and `@Meta(quoted)` annotation mapping + 9. dynamic `*earmuff*` variable annotation + 10. deep nested S-expression parsing + 11. mixed top-level definitions in one source + 12. CLOS method parsing + pipeline language alias routing + +**Files modified:** +- `editor/src/ast/Parser.h` — include `ast/CommonLispParser.h` +- `editor/src/Pipeline.h` — add parse routing for `\"common-lisp\"`, + `\"commonlisp\"`, `\"lisp\"`, and `\"cl\"` +- `editor/CMakeLists.txt` — `step370_test` target + +**Verification run:** +- `step370_test` — PASS (12/12) new step coverage +- `step369_test` — PASS (8/8) regression coverage +- `step365_test` — PASS (12/12) regression coverage + +**Architecture gate check:** +- `editor/src/ast/CommonLispParser.h` is within header size limit + (`598` lines <= `600`) + # Roadmap Planning — Sprints 12-25+ ## Status: Planning Complete (Sprints 12-19 detailed, 20-25 in roadmap.md)