Step 305: Python + Java parser deepening — class, async, lambda, decorator, generics (12/12 tests)
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -1821,4 +1821,13 @@ add_executable(step304_test tests/step304_test.cpp)
|
||||
target_include_directories(step304_test PRIVATE src)
|
||||
target_link_libraries(step304_test PRIVATE nlohmann_json::nlohmann_json)
|
||||
|
||||
# Step 305: Python + Java Parser Deepening
|
||||
add_executable(step305_test tests/step305_test.cpp)
|
||||
target_include_directories(step305_test PRIVATE src)
|
||||
target_link_libraries(step305_test PRIVATE
|
||||
unofficial::tree-sitter::tree-sitter
|
||||
tree_sitter_python
|
||||
tree_sitter_java)
|
||||
|
||||
|
||||
# Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies)
|
||||
|
||||
@@ -83,16 +83,65 @@ private:
|
||||
static void convertJavaTypeDeclaration(TSNode node,
|
||||
const std::string& source,
|
||||
Module* module) {
|
||||
std::string jtype = nodeType(node);
|
||||
TSNode nameNode = childByFieldName(node, "name");
|
||||
std::string className = nodeText(nameNode, source);
|
||||
TSNode bodyNode = childByFieldName(node, "body");
|
||||
if (ts_node_is_null(bodyNode)) return;
|
||||
|
||||
// Create ClassDeclaration or InterfaceDeclaration for supported types
|
||||
ClassDeclaration* cls = nullptr;
|
||||
InterfaceDeclaration* iface = nullptr;
|
||||
|
||||
if (jtype == "class_declaration") {
|
||||
cls = new ClassDeclaration(IdGenerator::next("cls"), className);
|
||||
applySpan(cls, node);
|
||||
|
||||
// Superclass
|
||||
TSNode superNode = childByFieldName(node, "superclass");
|
||||
if (!ts_node_is_null(superNode)) {
|
||||
uint32_t sc = ts_node_named_child_count(superNode);
|
||||
if (sc > 0) cls->superClass = nodeText(ts_node_named_child(superNode, 0), source);
|
||||
}
|
||||
|
||||
// Abstract check
|
||||
uint32_t nodeChildren = ts_node_named_child_count(node);
|
||||
for (uint32_t i = 0; i < nodeChildren; ++i) {
|
||||
TSNode ch = ts_node_named_child(node, i);
|
||||
if (nodeType(ch) == "modifiers") {
|
||||
uint32_t mc = ts_node_child_count(ch);
|
||||
for (uint32_t j = 0; j < mc; ++j) {
|
||||
if (nodeText(ts_node_child(ch, j), source) == "abstract")
|
||||
cls->isAbstract = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Type parameters
|
||||
TSNode typeParamsNode = childByFieldName(node, "type_parameters");
|
||||
if (!ts_node_is_null(typeParamsNode)) {
|
||||
convertJavaTypeParameters(typeParamsNode, source, cls);
|
||||
}
|
||||
} else if (jtype == "interface_declaration") {
|
||||
iface = new InterfaceDeclaration(IdGenerator::next("iface"), className);
|
||||
applySpan(iface, node);
|
||||
}
|
||||
|
||||
// Process body members
|
||||
uint32_t count = ts_node_named_child_count(bodyNode);
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
TSNode child = ts_node_named_child(bodyNode, i);
|
||||
std::string type = nodeType(child);
|
||||
if (type == "method_declaration") {
|
||||
// MethodDeclaration for class/interface hierarchy
|
||||
if (cls) {
|
||||
auto* meth = convertJavaMethodDeclaration(child, source, className);
|
||||
if (meth) cls->addChild("methods", meth);
|
||||
} else if (iface) {
|
||||
auto* meth = convertJavaMethodDeclaration(child, source, className);
|
||||
if (meth) iface->addChild("methods", meth);
|
||||
}
|
||||
// Backward compat: also add Function to module
|
||||
auto* fn = convertJavaMethod(child, source, className);
|
||||
if (fn) module->addChild("functions", fn);
|
||||
} else if (type == "constructor_declaration" || type == "compact_constructor_declaration") {
|
||||
@@ -106,6 +155,86 @@ private:
|
||||
convertJavaTypeDeclaration(child, source, module);
|
||||
}
|
||||
}
|
||||
|
||||
// Add class/interface to module
|
||||
if (cls) module->addChild("classes", cls);
|
||||
if (iface) module->addChild("classes", iface);
|
||||
}
|
||||
|
||||
static void convertJavaTypeParameters(TSNode typeParamsNode, const std::string& source,
|
||||
ClassDeclaration* cls) {
|
||||
uint32_t tpc = ts_node_named_child_count(typeParamsNode);
|
||||
for (uint32_t i = 0; i < tpc; ++i) {
|
||||
TSNode tp = ts_node_named_child(typeParamsNode, i);
|
||||
if (nodeType(tp) == "type_parameter") {
|
||||
std::string tpName;
|
||||
std::string constraint;
|
||||
uint32_t tpcc = ts_node_named_child_count(tp);
|
||||
for (uint32_t j = 0; j < tpcc; ++j) {
|
||||
TSNode tpChild = ts_node_named_child(tp, j);
|
||||
std::string tpType = nodeType(tpChild);
|
||||
if ((tpType == "identifier" || tpType == "type_identifier") && tpName.empty()) {
|
||||
tpName = nodeText(tpChild, source);
|
||||
} else if (tpType == "type_bound") {
|
||||
if (ts_node_named_child_count(tpChild) > 0) {
|
||||
constraint = nodeText(ts_node_named_child(tpChild, 0), source);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!tpName.empty()) {
|
||||
auto* typeParam = new TypeParameter(IdGenerator::next("tp"), tpName);
|
||||
typeParam->constraint = constraint;
|
||||
applySpan(typeParam, tp);
|
||||
cls->addChild("typeParameters", typeParam);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static MethodDeclaration* convertJavaMethodDeclaration(TSNode node,
|
||||
const std::string& source,
|
||||
const std::string& className) {
|
||||
TSNode nameNode = childByFieldName(node, "name");
|
||||
if (ts_node_is_null(nameNode)) return nullptr;
|
||||
|
||||
auto* meth = new MethodDeclaration(IdGenerator::next("meth"), nodeText(nameNode, source));
|
||||
applySpan(meth, node);
|
||||
meth->className = className;
|
||||
|
||||
TSNode typeNode = childByFieldName(node, "type");
|
||||
if (!ts_node_is_null(typeNode)) {
|
||||
if (auto* t = convertJavaType(typeNode, source)) meth->setChild("returnType", t);
|
||||
}
|
||||
|
||||
TSNode paramsNode = childByFieldName(node, "parameters");
|
||||
if (!ts_node_is_null(paramsNode)) {
|
||||
convertJavaParameters(paramsNode, source, meth);
|
||||
}
|
||||
|
||||
TSNode bodyNode = childByFieldName(node, "body");
|
||||
if (!ts_node_is_null(bodyNode)) {
|
||||
convertJavaBlockStatements(bodyNode, source, meth);
|
||||
}
|
||||
|
||||
// Check modifiers for visibility, static, abstract
|
||||
uint32_t allCount = ts_node_named_child_count(node);
|
||||
for (uint32_t i = 0; i < allCount; ++i) {
|
||||
TSNode child = ts_node_named_child(node, i);
|
||||
if (nodeType(child) == "modifiers") {
|
||||
uint32_t mc = ts_node_child_count(child);
|
||||
for (uint32_t j = 0; j < mc; ++j) {
|
||||
TSNode mod = ts_node_child(child, j);
|
||||
std::string modText = nodeText(mod, source);
|
||||
if (modText == "static") meth->isStatic = true;
|
||||
else if (modText == "public") meth->visibility = "public";
|
||||
else if (modText == "private") meth->visibility = "private";
|
||||
else if (modText == "protected") meth->visibility = "protected";
|
||||
else if (modText == "abstract") meth->isVirtual = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return meth;
|
||||
}
|
||||
|
||||
static void convertJavaFieldDeclaration(TSNode node,
|
||||
@@ -512,9 +641,49 @@ private:
|
||||
return convertJavaExpression(ts_node_named_child(node, 0), source);
|
||||
}
|
||||
} else if (type == "lambda_expression") {
|
||||
auto* ref = new VariableReference(IdGenerator::next("var"), nodeText(node, source));
|
||||
applySpan(ref, node);
|
||||
return ref;
|
||||
auto* lam = new LambdaExpression(IdGenerator::next("lam"));
|
||||
applySpan(lam, node);
|
||||
TSNode paramsNode = childByFieldName(node, "parameters");
|
||||
if (!ts_node_is_null(paramsNode)) {
|
||||
std::string pType = nodeType(paramsNode);
|
||||
if (pType == "identifier") {
|
||||
auto* param = new Parameter(IdGenerator::next("param"), nodeText(paramsNode, source));
|
||||
applySpan(param, paramsNode);
|
||||
lam->addChild("parameters", param);
|
||||
} else {
|
||||
uint32_t pc = ts_node_named_child_count(paramsNode);
|
||||
for (uint32_t pi = 0; pi < pc; ++pi) {
|
||||
TSNode pChild = ts_node_named_child(paramsNode, pi);
|
||||
TSNode pName = childByFieldName(pChild, "name");
|
||||
if (ts_node_is_null(pName)) pName = pChild;
|
||||
std::string paramName = nodeText(pName, source);
|
||||
if (!paramName.empty()) {
|
||||
auto* param = new Parameter(IdGenerator::next("param"), paramName);
|
||||
applySpan(param, pChild);
|
||||
lam->addChild("parameters", param);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
TSNode bodyNode = childByFieldName(node, "body");
|
||||
if (!ts_node_is_null(bodyNode)) {
|
||||
if (nodeType(bodyNode) == "block") {
|
||||
uint32_t bc = ts_node_named_child_count(bodyNode);
|
||||
for (uint32_t bi = 0; bi < bc; ++bi) {
|
||||
ASTNode* stmt = convertJavaStatement(ts_node_named_child(bodyNode, bi), source);
|
||||
if (stmt) lam->addChild("body", stmt);
|
||||
}
|
||||
} else {
|
||||
ASTNode* expr = convertJavaExpression(bodyNode, source);
|
||||
if (expr) {
|
||||
auto* exprStmt = new ExpressionStatement();
|
||||
exprStmt->id = IdGenerator::next("exprstmt");
|
||||
exprStmt->setChild("expression", expr);
|
||||
lam->addChild("body", exprStmt);
|
||||
}
|
||||
}
|
||||
}
|
||||
return lam;
|
||||
}
|
||||
|
||||
std::string text = nodeText(node, source);
|
||||
|
||||
@@ -9,6 +9,9 @@
|
||||
#include "Expression.h"
|
||||
#include "Type.h"
|
||||
#include "Annotation.h"
|
||||
#include "ClassDeclaration.h"
|
||||
#include "GenericType.h"
|
||||
#include "AsyncNodes.h"
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
@@ -153,7 +156,9 @@ public:
|
||||
#include "ast/JavaParser.h"
|
||||
#include "ast/RustParser.h"
|
||||
#include "ast/GoParser.h"
|
||||
#include "ast/KotlinParser.h"
|
||||
#include "ast/CSharpParser.h"
|
||||
private:
|
||||
};
|
||||
|
||||
// Standalone parsers (not tree-sitter fragment includes)
|
||||
#include "ast/KotlinParser.h"
|
||||
#include "ast/CSharpParser.h"
|
||||
|
||||
@@ -45,7 +45,7 @@ public:
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
private:
|
||||
// Python CST → AST
|
||||
// Python CST -> AST
|
||||
// ---------------------------------------------------------------
|
||||
static void convertPythonModule(TSNode root, const std::string& source, Module* module) {
|
||||
uint32_t count = ts_node_named_child_count(root);
|
||||
@@ -55,20 +55,38 @@ private:
|
||||
if (type == "function_definition") {
|
||||
auto* fn = convertPythonFunction(child, source);
|
||||
if (fn) module->addChild("functions", fn);
|
||||
} else if (type == "class_definition") {
|
||||
auto* cls = convertPythonClass(child, source);
|
||||
if (cls) module->addChild("classes", cls);
|
||||
} else if (type == "decorated_definition") {
|
||||
convertPythonDecorated(child, source, module);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static Function* convertPythonFunction(TSNode node, const std::string& source) {
|
||||
// Get function name
|
||||
TSNode nameNode = childByFieldName(node, "name");
|
||||
if (ts_node_is_null(nameNode)) return nullptr;
|
||||
|
||||
auto* fn = new Function();
|
||||
fn->id = IdGenerator::next("fn");
|
||||
applySpan(fn, node);
|
||||
applySpan(fn, node);
|
||||
fn->name = nodeText(nameNode, source);
|
||||
// Detect async keyword (non-named child with text "async")
|
||||
bool isAsync = false;
|
||||
uint32_t totalCount = ts_node_child_count(node);
|
||||
for (uint32_t i = 0; i < totalCount; ++i) {
|
||||
TSNode ch = ts_node_child(node, i);
|
||||
if (!ts_node_is_named(ch) && nodeText(ch, source) == "async") {
|
||||
isAsync = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Function* fn;
|
||||
if (isAsync) {
|
||||
fn = new AsyncFunction(IdGenerator::next("fn"), nodeText(nameNode, source));
|
||||
} else {
|
||||
fn = new Function();
|
||||
fn->id = IdGenerator::next("fn");
|
||||
fn->name = nodeText(nameNode, source);
|
||||
}
|
||||
applySpan(fn, node);
|
||||
|
||||
// Parameters
|
||||
@@ -90,6 +108,136 @@ private:
|
||||
return fn;
|
||||
}
|
||||
|
||||
static ClassDeclaration* convertPythonClass(TSNode node, const std::string& source) {
|
||||
TSNode nameNode = childByFieldName(node, "name");
|
||||
if (ts_node_is_null(nameNode)) return nullptr;
|
||||
|
||||
auto* cls = new ClassDeclaration(IdGenerator::next("cls"), nodeText(nameNode, source));
|
||||
applySpan(cls, node);
|
||||
|
||||
// Superclasses — first argument is the primary superclass
|
||||
TSNode superNode = childByFieldName(node, "superclasses");
|
||||
if (!ts_node_is_null(superNode)) {
|
||||
uint32_t sc = ts_node_named_child_count(superNode);
|
||||
if (sc > 0) {
|
||||
cls->superClass = nodeText(ts_node_named_child(superNode, 0), source);
|
||||
}
|
||||
}
|
||||
|
||||
// Body — extract methods
|
||||
TSNode bodyNode = childByFieldName(node, "body");
|
||||
if (!ts_node_is_null(bodyNode)) {
|
||||
uint32_t count = ts_node_named_child_count(bodyNode);
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
TSNode child = ts_node_named_child(bodyNode, i);
|
||||
std::string type = nodeType(child);
|
||||
if (type == "function_definition") {
|
||||
auto* meth = convertPythonMethodDecl(child, source, cls->name);
|
||||
if (meth) cls->addChild("methods", meth);
|
||||
} else if (type == "decorated_definition") {
|
||||
TSNode defNode = childByFieldName(child, "definition");
|
||||
if (!ts_node_is_null(defNode) && nodeType(defNode) == "function_definition") {
|
||||
auto* meth = convertPythonMethodDecl(defNode, source, cls->name);
|
||||
if (meth) {
|
||||
// Attach decorators
|
||||
uint32_t dc = ts_node_named_child_count(child);
|
||||
for (uint32_t j = 0; j < dc; ++j) {
|
||||
TSNode dChild = ts_node_named_child(child, j);
|
||||
if (nodeType(dChild) == "decorator") {
|
||||
auto* dec = convertPythonDecoratorNode(dChild, source);
|
||||
if (dec) meth->addChild("annotations", dec);
|
||||
}
|
||||
}
|
||||
cls->addChild("methods", meth);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return cls;
|
||||
}
|
||||
|
||||
static MethodDeclaration* convertPythonMethodDecl(TSNode node, const std::string& source,
|
||||
const std::string& className) {
|
||||
TSNode nameNode = childByFieldName(node, "name");
|
||||
if (ts_node_is_null(nameNode)) return nullptr;
|
||||
|
||||
auto* meth = new MethodDeclaration(IdGenerator::next("meth"), nodeText(nameNode, source));
|
||||
applySpan(meth, node);
|
||||
meth->className = className;
|
||||
|
||||
// Parameters
|
||||
TSNode paramsNode = childByFieldName(node, "parameters");
|
||||
if (!ts_node_is_null(paramsNode)) {
|
||||
convertPythonParameters(paramsNode, source, meth);
|
||||
}
|
||||
|
||||
// Body
|
||||
TSNode bodyNode = childByFieldName(node, "body");
|
||||
if (!ts_node_is_null(bodyNode)) {
|
||||
convertPythonBody(bodyNode, source, meth);
|
||||
}
|
||||
|
||||
return meth;
|
||||
}
|
||||
|
||||
static void convertPythonDecorated(TSNode node, const std::string& source, Module* module) {
|
||||
// Collect decorators
|
||||
std::vector<DecoratorAnnotation*> decorators;
|
||||
uint32_t count = ts_node_named_child_count(node);
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
TSNode child = ts_node_named_child(node, i);
|
||||
if (nodeType(child) == "decorator") {
|
||||
auto* dec = convertPythonDecoratorNode(child, source);
|
||||
if (dec) decorators.push_back(dec);
|
||||
}
|
||||
}
|
||||
|
||||
// Get the wrapped definition
|
||||
TSNode defNode = childByFieldName(node, "definition");
|
||||
if (ts_node_is_null(defNode)) {
|
||||
for (auto* d : decorators) delete d;
|
||||
return;
|
||||
}
|
||||
|
||||
std::string defType = nodeType(defNode);
|
||||
if (defType == "function_definition") {
|
||||
auto* fn = convertPythonFunction(defNode, source);
|
||||
if (fn) {
|
||||
for (auto* dec : decorators) fn->addChild("annotations", dec);
|
||||
module->addChild("functions", fn);
|
||||
} else {
|
||||
for (auto* d : decorators) delete d;
|
||||
}
|
||||
} else if (defType == "class_definition") {
|
||||
auto* cls = convertPythonClass(defNode, source);
|
||||
if (cls) {
|
||||
for (auto* dec : decorators) cls->addChild("annotations", dec);
|
||||
module->addChild("classes", cls);
|
||||
} else {
|
||||
for (auto* d : decorators) delete d;
|
||||
}
|
||||
} else {
|
||||
for (auto* d : decorators) delete d;
|
||||
}
|
||||
}
|
||||
|
||||
static DecoratorAnnotation* convertPythonDecoratorNode(TSNode node, const std::string& source) {
|
||||
uint32_t dc = ts_node_named_child_count(node);
|
||||
if (dc == 0) return nullptr;
|
||||
TSNode exprNode = ts_node_named_child(node, 0);
|
||||
std::string name = nodeText(exprNode, source);
|
||||
// For call decorators like @app.route("/"), extract the function name
|
||||
if (nodeType(exprNode) == "call") {
|
||||
TSNode funcNode = childByFieldName(exprNode, "function");
|
||||
if (!ts_node_is_null(funcNode)) name = nodeText(funcNode, source);
|
||||
}
|
||||
auto* dec = new DecoratorAnnotation(IdGenerator::next("dec"), name);
|
||||
applySpan(dec, node);
|
||||
return dec;
|
||||
}
|
||||
|
||||
static void convertPythonParameters(TSNode paramsNode, const std::string& source, Function* fn) {
|
||||
uint32_t count = ts_node_named_child_count(paramsNode);
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
@@ -100,7 +248,6 @@ private:
|
||||
applySpan(param, child);
|
||||
fn->addChild("parameters", param);
|
||||
} else if (type == "default_parameter") {
|
||||
// def f(x=10) → Parameter with defaultValue
|
||||
TSNode nameN = childByFieldName(child, "name");
|
||||
TSNode valueN = childByFieldName(child, "value");
|
||||
if (!ts_node_is_null(nameN)) {
|
||||
@@ -117,7 +264,6 @@ private:
|
||||
}
|
||||
|
||||
static void convertPythonBody(TSNode bodyNode, const std::string& source, Function* fn) {
|
||||
// bodyNode is typically a "block" node
|
||||
uint32_t count = ts_node_named_child_count(bodyNode);
|
||||
for (uint32_t i = 0; i < count; ++i) {
|
||||
TSNode child = ts_node_named_child(bodyNode, i);
|
||||
@@ -132,7 +278,6 @@ private:
|
||||
auto* ret = new Return();
|
||||
ret->id = IdGenerator::next("ret");
|
||||
applySpan(ret, node);
|
||||
// The return value is the first named child (if any)
|
||||
uint32_t count = ts_node_named_child_count(node);
|
||||
if (count > 0) {
|
||||
TSNode valNode = ts_node_named_child(node, 0);
|
||||
@@ -240,7 +385,6 @@ private:
|
||||
applySpan(lit, node);
|
||||
return lit;
|
||||
} else if (type == "comparison_operator" || type == "boolean_operator") {
|
||||
// Treat like binary op
|
||||
auto* binOp = new BinaryOperation();
|
||||
binOp->id = IdGenerator::next("binop");
|
||||
applySpan(binOp, node);
|
||||
@@ -251,7 +395,6 @@ private:
|
||||
ASTNode* right = convertPythonExpression(ts_node_named_child(node, count - 1), source);
|
||||
if (right) binOp->setChild("right", right);
|
||||
}
|
||||
// Operator is a non-named child between the named ones
|
||||
uint32_t totalCount = ts_node_child_count(node);
|
||||
for (uint32_t i = 0; i < totalCount; ++i) {
|
||||
TSNode c = ts_node_child(node, i);
|
||||
@@ -281,6 +424,21 @@ private:
|
||||
}
|
||||
}
|
||||
return call;
|
||||
} else if (type == "assignment") {
|
||||
auto* assign = new Assignment();
|
||||
assign->id = IdGenerator::next("assign");
|
||||
applySpan(assign, node);
|
||||
TSNode leftNode = childByFieldName(node, "left");
|
||||
TSNode rightNode = childByFieldName(node, "right");
|
||||
if (!ts_node_is_null(leftNode)) {
|
||||
ASTNode* target = convertPythonExpression(leftNode, source);
|
||||
if (target) assign->setChild("target", target);
|
||||
}
|
||||
if (!ts_node_is_null(rightNode)) {
|
||||
ASTNode* value = convertPythonExpression(rightNode, source);
|
||||
if (value) assign->setChild("value", value);
|
||||
}
|
||||
return assign;
|
||||
} else if (type == "parenthesized_expression") {
|
||||
uint32_t count = ts_node_named_child_count(node);
|
||||
if (count > 0) return convertPythonExpression(ts_node_named_child(node, 0), source);
|
||||
@@ -298,6 +456,41 @@ private:
|
||||
if (operand) unOp->setChild("operand", operand);
|
||||
}
|
||||
return unOp;
|
||||
} else if (type == "await") {
|
||||
auto* awExpr = new AwaitExpression(IdGenerator::next("await"));
|
||||
applySpan(awExpr, node);
|
||||
uint32_t count = ts_node_named_child_count(node);
|
||||
if (count > 0) {
|
||||
ASTNode* expr = convertPythonExpression(ts_node_named_child(node, 0), source);
|
||||
if (expr) awExpr->setChild("expression", expr);
|
||||
}
|
||||
return awExpr;
|
||||
} else if (type == "lambda") {
|
||||
auto* lam = new LambdaExpression(IdGenerator::next("lam"));
|
||||
applySpan(lam, node);
|
||||
TSNode paramsNode = childByFieldName(node, "parameters");
|
||||
if (!ts_node_is_null(paramsNode)) {
|
||||
uint32_t pc = ts_node_named_child_count(paramsNode);
|
||||
for (uint32_t i = 0; i < pc; ++i) {
|
||||
TSNode pChild = ts_node_named_child(paramsNode, i);
|
||||
if (nodeType(pChild) == "identifier") {
|
||||
auto* param = new Parameter(IdGenerator::next("param"), nodeText(pChild, source));
|
||||
applySpan(param, pChild);
|
||||
lam->addChild("parameters", param);
|
||||
}
|
||||
}
|
||||
}
|
||||
TSNode bodyNode = childByFieldName(node, "body");
|
||||
if (!ts_node_is_null(bodyNode)) {
|
||||
ASTNode* bodyExpr = convertPythonExpression(bodyNode, source);
|
||||
if (bodyExpr) {
|
||||
auto* exprStmt = new ExpressionStatement();
|
||||
exprStmt->id = IdGenerator::next("exprstmt");
|
||||
exprStmt->setChild("expression", bodyExpr);
|
||||
lam->addChild("body", exprStmt);
|
||||
}
|
||||
}
|
||||
return lam;
|
||||
}
|
||||
// Fallback: treat as variable reference with raw text
|
||||
std::string text = nodeText(node, source);
|
||||
|
||||
355
editor/tests/step305_test.cpp
Normal file
355
editor/tests/step305_test.cpp
Normal file
@@ -0,0 +1,355 @@
|
||||
// Step 305: Python + Java Parser Deepening (12 tests)
|
||||
// Tests that PythonParser and JavaParser correctly produce the new AST node
|
||||
// types: ClassDeclaration, InterfaceDeclaration, MethodDeclaration,
|
||||
// AsyncFunction, AwaitExpression, LambdaExpression, DecoratorAnnotation,
|
||||
// and TypeParameter via tree-sitter parsing of real source strings.
|
||||
|
||||
#include "ast/Parser.h"
|
||||
#include "ast/ClassDeclaration.h"
|
||||
#include "ast/GenericType.h"
|
||||
#include "ast/AsyncNodes.h"
|
||||
#include "ast/Module.h"
|
||||
#include "ast/Function.h"
|
||||
#include "ast/Variable.h"
|
||||
#include "ast/Parameter.h"
|
||||
#include "ast/Statement.h"
|
||||
#include "ast/Expression.h"
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
#include <functional>
|
||||
|
||||
static int passed = 0, failed = 0;
|
||||
#define TEST(name) { std::cout << " " << #name << "... "; }
|
||||
#define PASS() { std::cout << "PASS\n"; ++passed; }
|
||||
#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; }
|
||||
#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Python tests (1-6)
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
// 1. Python backward compat — simple function still yields Function
|
||||
void test_python_backward_compat() {
|
||||
TEST(python_backward_compat);
|
||||
std::string src = "def greet(name):\n return name\n";
|
||||
auto mod = TreeSitterParser::parsePython(src);
|
||||
CHECK(mod != nullptr, "module null");
|
||||
|
||||
auto& fns = mod->getChildren("functions");
|
||||
CHECK(!fns.empty(), "expected at least one function");
|
||||
CHECK(fns[0]->conceptType == "Function", "expected Function, got " + fns[0]->conceptType);
|
||||
auto* fn0 = dynamic_cast<Function*>(fns[0]);
|
||||
CHECK(fn0 != nullptr, "dynamic_cast to Function failed");
|
||||
CHECK(fn0->name == "greet", "expected name 'greet', got " + fn0->name);
|
||||
PASS();
|
||||
}
|
||||
|
||||
// 2. Python class — class_definition yields ClassDeclaration
|
||||
void test_python_class() {
|
||||
TEST(python_class);
|
||||
std::string src =
|
||||
"class Animal(LivingThing):\n"
|
||||
" def speak(self):\n"
|
||||
" return \"...\"\n";
|
||||
auto mod = TreeSitterParser::parsePython(src);
|
||||
CHECK(mod != nullptr, "module null");
|
||||
|
||||
auto& classes = mod->getChildren("classes");
|
||||
CHECK(!classes.empty(), "expected at least one class");
|
||||
CHECK(classes[0]->conceptType == "ClassDeclaration",
|
||||
"expected ClassDeclaration, got " + classes[0]->conceptType);
|
||||
|
||||
auto* cls = dynamic_cast<ClassDeclaration*>(classes[0]);
|
||||
CHECK(cls != nullptr, "dynamic_cast to ClassDeclaration failed");
|
||||
CHECK(cls->name == "Animal", "expected name 'Animal', got " + cls->name);
|
||||
CHECK(cls->superClass == "LivingThing",
|
||||
"expected superClass 'LivingThing', got " + cls->superClass);
|
||||
|
||||
auto& methods = cls->getChildren("methods");
|
||||
CHECK(!methods.empty(), "expected at least one method");
|
||||
auto* meth = dynamic_cast<MethodDeclaration*>(methods[0]);
|
||||
CHECK(meth != nullptr, "dynamic_cast to MethodDeclaration failed");
|
||||
CHECK(meth->name == "speak", "expected method 'speak', got " + meth->name);
|
||||
CHECK(meth->className == "Animal", "expected className 'Animal', got " + meth->className);
|
||||
PASS();
|
||||
}
|
||||
|
||||
// 3. Python async function — async def yields AsyncFunction
|
||||
void test_python_async_function() {
|
||||
TEST(python_async_function);
|
||||
std::string src = "async def fetch_data():\n pass\n";
|
||||
auto mod = TreeSitterParser::parsePython(src);
|
||||
CHECK(mod != nullptr, "module null");
|
||||
|
||||
auto& fns = mod->getChildren("functions");
|
||||
CHECK(!fns.empty(), "expected at least one function");
|
||||
|
||||
auto* af = dynamic_cast<AsyncFunction*>(fns[0]);
|
||||
CHECK(af != nullptr, "expected AsyncFunction, dynamic_cast failed (got " + fns[0]->conceptType + ")");
|
||||
CHECK(af->name == "fetch_data", "expected name 'fetch_data', got " + af->name);
|
||||
CHECK(af->isAsync, "isAsync should be true");
|
||||
PASS();
|
||||
}
|
||||
|
||||
// 4. Python await — await expression in body yields AwaitExpression
|
||||
void test_python_await() {
|
||||
TEST(python_await);
|
||||
std::string src =
|
||||
"async def load():\n"
|
||||
" result = await get_data()\n";
|
||||
auto mod = TreeSitterParser::parsePython(src);
|
||||
CHECK(mod != nullptr, "module null");
|
||||
|
||||
auto& fns = mod->getChildren("functions");
|
||||
CHECK(!fns.empty(), "expected function");
|
||||
|
||||
// The body should contain a statement with an await expression somewhere
|
||||
auto& body = fns[0]->getChildren("body");
|
||||
CHECK(!body.empty(), "expected body statements");
|
||||
|
||||
// Walk body looking for an AwaitExpression
|
||||
bool foundAwait = false;
|
||||
std::function<void(ASTNode*)> findAwait = [&](ASTNode* node) {
|
||||
if (!node) return;
|
||||
if (node->conceptType == "AwaitExpression") { foundAwait = true; return; }
|
||||
for (auto* child : node->allChildren()) findAwait(child);
|
||||
};
|
||||
for (auto* stmt : body) findAwait(stmt);
|
||||
|
||||
CHECK(foundAwait, "expected AwaitExpression in function body");
|
||||
PASS();
|
||||
}
|
||||
|
||||
// 5. Python lambda — lambda expression yields LambdaExpression
|
||||
void test_python_lambda() {
|
||||
TEST(python_lambda);
|
||||
std::string src =
|
||||
"def make():\n"
|
||||
" return lambda x: x + 1\n";
|
||||
auto mod = TreeSitterParser::parsePython(src);
|
||||
CHECK(mod != nullptr, "module null");
|
||||
|
||||
auto& fns = mod->getChildren("functions");
|
||||
CHECK(!fns.empty(), "expected function");
|
||||
|
||||
// Walk body looking for LambdaExpression
|
||||
bool foundLambda = false;
|
||||
std::string lambdaType;
|
||||
std::function<void(ASTNode*)> findLambda = [&](ASTNode* node) {
|
||||
if (!node) return;
|
||||
if (node->conceptType == "LambdaExpression") {
|
||||
foundLambda = true;
|
||||
lambdaType = node->conceptType;
|
||||
return;
|
||||
}
|
||||
for (auto* child : node->allChildren()) findLambda(child);
|
||||
};
|
||||
auto& body = fns[0]->getChildren("body");
|
||||
for (auto* stmt : body) findLambda(stmt);
|
||||
|
||||
CHECK(foundLambda, "expected LambdaExpression in function body");
|
||||
PASS();
|
||||
}
|
||||
|
||||
// 6. Python decorator — @decorator yields DecoratorAnnotation
|
||||
void test_python_decorator() {
|
||||
TEST(python_decorator);
|
||||
std::string src =
|
||||
"@cache\n"
|
||||
"def expensive():\n"
|
||||
" return 42\n";
|
||||
auto mod = TreeSitterParser::parsePython(src);
|
||||
CHECK(mod != nullptr, "module null");
|
||||
|
||||
auto& fns = mod->getChildren("functions");
|
||||
CHECK(!fns.empty(), "expected function");
|
||||
|
||||
// The function should have a DecoratorAnnotation in annotations
|
||||
auto& annos = fns[0]->getChildren("annotations");
|
||||
bool foundDecorator = false;
|
||||
for (auto* anno : annos) {
|
||||
if (anno->conceptType == "DecoratorAnnotation") {
|
||||
auto* dec = dynamic_cast<DecoratorAnnotation*>(anno);
|
||||
CHECK(dec != nullptr, "dynamic_cast to DecoratorAnnotation failed");
|
||||
CHECK(dec->name == "cache", "expected decorator 'cache', got " + dec->name);
|
||||
foundDecorator = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
CHECK(foundDecorator, "expected DecoratorAnnotation on function");
|
||||
PASS();
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Java tests (7-12)
|
||||
// ---------------------------------------------------------------
|
||||
|
||||
// 7. Java backward compat — method still appears as Function in module.functions
|
||||
void test_java_backward_compat() {
|
||||
TEST(java_backward_compat);
|
||||
std::string src =
|
||||
"class Dog {\n"
|
||||
" void bark() { }\n"
|
||||
"}\n";
|
||||
auto mod = TreeSitterParser::parseJava(src);
|
||||
CHECK(mod != nullptr, "module null");
|
||||
|
||||
auto& fns = mod->getChildren("functions");
|
||||
CHECK(!fns.empty(), "expected at least one function (backward compat)");
|
||||
|
||||
bool found = false;
|
||||
for (auto* fn : fns) {
|
||||
auto* func = dynamic_cast<Function*>(fn);
|
||||
if (func && func->name.find("bark") != std::string::npos) { found = true; break; }
|
||||
}
|
||||
CHECK(found, "expected a Function with 'bark' in name");
|
||||
PASS();
|
||||
}
|
||||
|
||||
// 8. Java ClassDeclaration — class parsed into classes child
|
||||
void test_java_class_declaration() {
|
||||
TEST(java_class_declaration);
|
||||
std::string src =
|
||||
"class Dog extends Animal {\n"
|
||||
" void bark() { }\n"
|
||||
"}\n";
|
||||
auto mod = TreeSitterParser::parseJava(src);
|
||||
CHECK(mod != nullptr, "module null");
|
||||
|
||||
auto& classes = mod->getChildren("classes");
|
||||
CHECK(!classes.empty(), "expected at least one class");
|
||||
|
||||
auto* cls = dynamic_cast<ClassDeclaration*>(classes[0]);
|
||||
CHECK(cls != nullptr, "dynamic_cast to ClassDeclaration failed (got " + classes[0]->conceptType + ")");
|
||||
CHECK(cls->name == "Dog", "expected name 'Dog', got " + cls->name);
|
||||
CHECK(cls->superClass == "Animal", "expected superClass 'Animal', got " + cls->superClass);
|
||||
PASS();
|
||||
}
|
||||
|
||||
// 9. Java InterfaceDeclaration — interface parsed into classes child
|
||||
void test_java_interface_declaration() {
|
||||
TEST(java_interface_declaration);
|
||||
std::string src =
|
||||
"interface Drawable {\n"
|
||||
" void draw();\n"
|
||||
"}\n";
|
||||
auto mod = TreeSitterParser::parseJava(src);
|
||||
CHECK(mod != nullptr, "module null");
|
||||
|
||||
auto& classes = mod->getChildren("classes");
|
||||
CHECK(!classes.empty(), "expected at least one class/interface entry");
|
||||
|
||||
bool foundIface = false;
|
||||
for (auto* entry : classes) {
|
||||
if (entry->conceptType == "InterfaceDeclaration") {
|
||||
auto* iface = dynamic_cast<InterfaceDeclaration*>(entry);
|
||||
CHECK(iface != nullptr, "dynamic_cast to InterfaceDeclaration failed");
|
||||
CHECK(iface->name == "Drawable", "expected name 'Drawable', got " + iface->name);
|
||||
foundIface = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
CHECK(foundIface, "expected InterfaceDeclaration in classes");
|
||||
PASS();
|
||||
}
|
||||
|
||||
// 10. Java MethodDeclaration — method as child of ClassDeclaration
|
||||
void test_java_method_declaration() {
|
||||
TEST(java_method_declaration);
|
||||
std::string src =
|
||||
"class Service {\n"
|
||||
" public static void process() { }\n"
|
||||
"}\n";
|
||||
auto mod = TreeSitterParser::parseJava(src);
|
||||
CHECK(mod != nullptr, "module null");
|
||||
|
||||
auto& classes = mod->getChildren("classes");
|
||||
CHECK(!classes.empty(), "expected class");
|
||||
|
||||
auto* cls = dynamic_cast<ClassDeclaration*>(classes[0]);
|
||||
CHECK(cls != nullptr, "expected ClassDeclaration");
|
||||
|
||||
auto& methods = cls->getChildren("methods");
|
||||
CHECK(!methods.empty(), "expected at least one method in class");
|
||||
|
||||
auto* meth = dynamic_cast<MethodDeclaration*>(methods[0]);
|
||||
CHECK(meth != nullptr, "dynamic_cast to MethodDeclaration failed");
|
||||
CHECK(meth->name == "process", "expected method 'process', got " + meth->name);
|
||||
CHECK(meth->className == "Service", "expected className 'Service', got " + meth->className);
|
||||
CHECK(meth->isStatic, "expected isStatic=true for 'public static'");
|
||||
CHECK(meth->visibility == "public", "expected visibility 'public', got " + meth->visibility);
|
||||
PASS();
|
||||
}
|
||||
|
||||
// 11. Java lambda — lambda expression yields LambdaExpression
|
||||
void test_java_lambda() {
|
||||
TEST(java_lambda);
|
||||
std::string src =
|
||||
"class App {\n"
|
||||
" void run() {\n"
|
||||
" Runnable r = () -> { };\n"
|
||||
" }\n"
|
||||
"}\n";
|
||||
auto mod = TreeSitterParser::parseJava(src);
|
||||
CHECK(mod != nullptr, "module null");
|
||||
|
||||
// Walk entire AST looking for LambdaExpression
|
||||
bool foundLambda = false;
|
||||
std::function<void(ASTNode*)> findLambda = [&](ASTNode* node) {
|
||||
if (!node) return;
|
||||
if (node->conceptType == "LambdaExpression") { foundLambda = true; return; }
|
||||
for (auto* child : node->allChildren()) findLambda(child);
|
||||
};
|
||||
findLambda(mod.get());
|
||||
|
||||
CHECK(foundLambda, "expected LambdaExpression somewhere in AST");
|
||||
PASS();
|
||||
}
|
||||
|
||||
// 12. Java generic class — type parameters yield TypeParameter children
|
||||
void test_java_generic_class() {
|
||||
TEST(java_generic_class);
|
||||
std::string src =
|
||||
"class Box<T extends Comparable> {\n"
|
||||
" T value;\n"
|
||||
"}\n";
|
||||
auto mod = TreeSitterParser::parseJava(src);
|
||||
CHECK(mod != nullptr, "module null");
|
||||
|
||||
auto& classes = mod->getChildren("classes");
|
||||
CHECK(!classes.empty(), "expected class");
|
||||
|
||||
auto* cls = dynamic_cast<ClassDeclaration*>(classes[0]);
|
||||
CHECK(cls != nullptr, "expected ClassDeclaration");
|
||||
CHECK(cls->name == "Box", "expected name 'Box', got " + cls->name);
|
||||
|
||||
auto& typeParams = cls->getChildren("typeParameters");
|
||||
CHECK(!typeParams.empty(), "expected at least one TypeParameter");
|
||||
|
||||
auto* tp = dynamic_cast<TypeParameter*>(typeParams[0]);
|
||||
CHECK(tp != nullptr, "dynamic_cast to TypeParameter failed");
|
||||
CHECK(tp->name == "T", "expected type param 'T', got " + tp->name);
|
||||
CHECK(tp->constraint == "Comparable",
|
||||
"expected constraint 'Comparable', got " + tp->constraint);
|
||||
PASS();
|
||||
}
|
||||
|
||||
int main() {
|
||||
std::cout << "=== Step 305: Python + Java Parser Deepening ===\n";
|
||||
test_python_backward_compat();
|
||||
test_python_class();
|
||||
test_python_async_function();
|
||||
test_python_await();
|
||||
test_python_lambda();
|
||||
test_python_decorator();
|
||||
test_java_backward_compat();
|
||||
test_java_class_declaration();
|
||||
test_java_interface_declaration();
|
||||
test_java_method_declaration();
|
||||
test_java_lambda();
|
||||
test_java_generic_class();
|
||||
std::cout << "\nResults: " << passed << "/" << (passed + failed) << " passed\n";
|
||||
return failed > 0 ? 1 : 0;
|
||||
}
|
||||
Reference in New Issue
Block a user