bill/whetstone_DSL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Step 149: add Java parser and generator

Browse Source
This commit is contained in:
Bill
2026-02-09 18:54:52 -07:00
parent fa030c0186
commit 52b54575ec
12 changed files with 1324 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
PROGRESS.md
View File

@@ -496,3 +496,4 @@ Sprint 5 in progress. Step 141 (Emacs daemon with user config) done. Next: Step
| 2026-02-10 | Codex | Step 146: Emacs-Whetstone bridge with buffer pull/push, Emacs frame opening, and sync commands. 5/5 tests pass. |
| 2026-02-10 | Codex | Step 147: JavaScript/TypeScript CST-to-AST parsing with functions, params, classes, and arrow-function discovery. 4/4 tests pass. |
| 2026-02-10 | Codex | Step 148: JavaScript/TypeScript generator with imports, class method grouping, type annotations, and WeakRef/FinalizationRegistry mapping. 6/6 tests pass. |
| 2026-02-10 | Codex | Step 149: Java CST-to-AST parsing + generator with class/method/constructor handling, import support, and GC-aware annotations. 8/8 tests pass. |

7
editor/CMakeLists.txt
View File

@@ -835,6 +835,13 @@ add_executable(step148_test tests/step148_test.cpp)
target_include_directories(step148_test PRIVATE src)
target_link_libraries(step148_test PRIVATE nlohmann_json::nlohmann_json)
add_executable(step149_test tests/step149_test.cpp)
target_include_directories(step149_test PRIVATE src)
target_link_libraries(step149_test PRIVATE
nlohmann_json::nlohmann_json
unofficial::tree-sitter::tree-sitter
tree_sitter_java)
find_package(SDL2 CONFIG REQUIRED)
find_package(OpenGL REQUIRED)
find_package(glad CONFIG REQUIRED)

9
editor/src/APIDocGenerator.h
View File

@@ -16,12 +16,13 @@ public:
return {
// Parsers
{"TreeSitterParser", "parser",
"Parses source code (Python, C++, Elisp, JavaScript, TypeScript) into Whetstone AST using tree-sitter grammars",
"Parses source code (Python, C++, Elisp, JavaScript, TypeScript, Java) into Whetstone AST using tree-sitter grammars",
{"parsePython(source)", "parsePythonWithDiagnostics(source)",
"parseCpp(source)", "parseCppWithDiagnostics(source)",
"parseElisp(source)", "parseElispWithDiagnostics(source)",
"parseJavaScript(source)", "parseJavaScriptWithDiagnostics(source)",
"parseTypeScript(source)", "parseTypeScriptWithDiagnostics(source)"}},
"parseTypeScript(source)", "parseTypeScriptWithDiagnostics(source)",
"parseJava(source)", "parseJavaWithDiagnostics(source)"}},
// Generators
{"PythonGenerator", "generator",
@@ -44,6 +45,10 @@ public:
"Generates TypeScript source code from Whetstone AST with type annotations",
{"generate(node)"}},
{"JavaGenerator", "generator",
"Generates Java source code from Whetstone AST",
{"generate(node)"}},
// Validators
{"AnnotationValidator", "validator",
"Validates memory annotations for correctness: missing intent, aliasing, conflicts",

4
editor/src/EditorUtils.h
View File

@@ -50,6 +50,10 @@ static std::string generateForLanguage(const Module* ast, const std::string& lan
TypeScriptGenerator gen;
return gen.generate(ast);
}
if (language == "java") {
JavaGenerator gen;
return gen.generate(ast);
}
PythonGenerator gen;
return gen.generate(ast);
}

9
editor/src/Orchestrator.h
View File

@@ -402,6 +402,8 @@ public:
targetLanguage = "javascript";
} else if (path.substr(path.find_last_of(".") + 1) == "ts") {
targetLanguage = "typescript";
} else if (path.substr(path.find_last_of(".") + 1) == "java") {
targetLanguage = "java";
}
// Parse the content using the appropriate tree-sitter parser
@@ -416,6 +418,8 @@ public:
module = TreeSitterParser::parseJavaScript(content);
} else if (targetLanguage == "typescript") {
module = TreeSitterParser::parseTypeScript(content);
} else if (targetLanguage == "java") {
module = TreeSitterParser::parseJava(content);
} else {
// For unknown languages, create a basic module with the content
module = std::make_unique<Module>();
@@ -451,6 +455,8 @@ public:
targetLanguage = "javascript";
} else if (path.substr(path.find_last_of(".") + 1) == "ts") {
targetLanguage = "typescript";
} else if (path.substr(path.find_last_of(".") + 1) == "java") {
targetLanguage = "java";
}
}
@@ -471,6 +477,9 @@ public:
} else if (targetLanguage == "typescript") {
TypeScriptGenerator gen;
content = gen.generate(ast);
} else if (targetLanguage == "java") {
JavaGenerator gen;
content = gen.generate(ast);
} else {
// Default to Python generator for unknown languages
PythonGenerator gen;

7
editor/src/Pipeline.h
View File

@@ -98,6 +98,10 @@ public:
auto pr = TreeSitterParser::parseTypeScriptWithDiagnostics(source);
diags = std::move(pr.diagnostics);
return std::move(pr.module);
} else if (language == "java") {
auto pr = TreeSitterParser::parseJavaWithDiagnostics(source);
diags = std::move(pr.diagnostics);
return std::move(pr.module);
}
return nullptr;
}
@@ -120,6 +124,9 @@ public:
} else if (language == "typescript") {
TypeScriptGenerator gen;
return gen.generate(ast);
} else if (language == "java") {
JavaGenerator gen;
return gen.generate(ast);
}
return "";
}

5
editor/src/TextASTSync.h
View File

@@ -47,6 +47,9 @@ public:
} else if (language_ == "typescript") {
TypeScriptGenerator gen;
return gen.generate(module_.get());
} else if (language_ == "java") {
JavaGenerator gen;
return gen.generate(module_.get());
}
return text_;
}
@@ -63,6 +66,8 @@ public:
module_ = TreeSitterParser::parseJavaScript(text_);
} else if (language_ == "typescript") {
module_ = TreeSitterParser::parseTypeScript(text_);
} else if (language_ == "java") {
module_ = TreeSitterParser::parseJava(text_);
}
parsePending_ = false;
}

1
editor/src/ast/Generator.h
View File

@@ -6,3 +6,4 @@
#include "ElispGenerator.h"
#include "CppGenerator.h"
#include "JavaScriptGenerator.h"
#include "JavaGenerator.h"

631
editor/src/ast/JavaGenerator.h Normal file
View File

@@ -0,0 +1,631 @@
#pragma once
#include "ProjectionGenerator.h"
#include "Import.h"
#include <map>
#include <unordered_map>
class JavaGenerator : public ProjectionGenerator {
public:
std::string generate(const ASTNode* node) override {
return dispatchGenerate(this, node, "// Unknown concept: ");
}
std::string visitModule(const Module* module) override {
std::ostringstream oss;
auto imports = module->getChildren("imports");
for (const auto* impNode : imports) {
if (impNode->conceptType != "Import") continue;
oss << emitImport(static_cast<const Import*>(impNode)) << "\n";
}
if (!imports.empty()) oss << "\n";
// Group functions and variables by class prefix.
std::map<std::string, std::vector<const Function*>> classMethods;
std::map<std::string, std::vector<const Variable*>> classFields;
std::vector<const Function*> topLevelMethods;
std::vector<const Variable*> topLevelFields;
auto functions = module->getChildren("functions");
for (const auto* fnNode : functions) {
if (fnNode->conceptType != "Function") continue;
const Function* fn = static_cast<const Function*>(fnNode);
auto pos = fn->name.find('.');
if (pos != std::string::npos && pos > 0 && pos + 1 < fn->name.size()) {
std::string className = fn->name.substr(0, pos);
classMethods[className].push_back(fn);
} else {
topLevelMethods.push_back(fn);
}
}
auto variables = module->getChildren("variables");
for (const auto* varNode : variables) {
if (varNode->conceptType != "Variable") continue;
const Variable* var = static_cast<const Variable*>(varNode);
auto pos = var->name.find('.');
if (pos != std::string::npos && pos > 0 && pos + 1 < var->name.size()) {
std::string className = var->name.substr(0, pos);
classFields[className].push_back(var);
} else {
topLevelFields.push_back(var);
}
}
std::string moduleClass = module->name.empty() ? "Main" : module->name;
if (!topLevelMethods.empty() || !topLevelFields.empty()) {
emitClass(oss, moduleClass, topLevelFields, topLevelMethods);
if (!classMethods.empty() || !classFields.empty()) oss << "\n";
}
for (const auto& [className, methods] : classMethods) {
std::vector<const Variable*> fields;
auto it = classFields.find(className);
if (it != classFields.end()) fields = it->second;
emitClass(oss, className, fields, methods);
oss << "\n";
}
if (!classFields.empty() && classMethods.empty()) {
for (const auto& [className, fields] : classFields) {
emitClass(oss, className, fields, {});
oss << "\n";
}
}
return oss.str();
}
std::string visitFunction(const Function* function) override {
std::ostringstream oss;
emitAnnotations(oss, function->getChildren("annotations"), "");
std::string methodName = function->name;
std::string returnType = "void";
auto retType = function->getChild("returnType");
if (retType) returnType = generate(retType);
oss << returnType << " " << methodName << "(";
emitParameters(oss, function->getChildren("parameters"));
oss << ") {\n";
emitBody(oss, function->getChildren("body"), " ");
oss << "}\n";
return oss.str();
}
std::string visitVariable(const Variable* variable) override {
std::ostringstream oss;
emitAnnotations(oss, variable->getChildren("annotations"), "");
std::string typeStr = "Object";
auto type = variable->getChild("type");
if (type) typeStr = generate(type);
std::string name = stripClassPrefix(variable->name);
oss << typeStr << " " << name;
auto initializer = variable->getChild("initializer");
if (initializer) {
oss << " = " << generate(initializer);
}
oss << ";";
return oss.str();
}
std::string visitParameter(const Parameter* parameter) override {
std::ostringstream oss;
std::string typeStr = "Object";
auto type = parameter->getChild("type");
if (type) typeStr = generate(type);
oss << typeStr << " " << parameter->name;
return oss.str();
}
std::string visitAssignment(const Assignment* assignment) override {
std::ostringstream oss;
auto target = assignment->getChild("target");
auto value = assignment->getChild("value");
if (target) {
oss << generate(target);
} else {
oss << "/* missing target */";
}
oss << " = ";
if (value) {
oss << generate(value);
} else {
oss << "null";
}
oss << ";";
return oss.str();
}
std::string visitReturn(const Return* ret) override {
std::ostringstream oss;
oss << "return";
auto value = ret->getChild("value");
if (value) {
oss << " " << generate(value);
}
oss << ";";
return oss.str();
}
std::string visitBinaryOperation(const BinaryOperation* binOp) override {
std::ostringstream oss;
auto left = binOp->getChild("left");
auto right = binOp->getChild("right");
if (left) {
oss << generate(left);
} else {
oss << "/* missing left */";
}
oss << " " << binOp->op << " ";
if (right) {
oss << generate(right);
} else {
oss << "/* missing right */";
}
return oss.str();
}
std::string visitVariableReference(const VariableReference* varRef) override {
return varRef->variableName;
}
std::string visitIntegerLiteral(const IntegerLiteral* lit) override {
return std::to_string(lit->value);
}
std::string visitFloatLiteral(const FloatLiteral* lit) override {
return lit->value;
}
std::string visitStringLiteral(const StringLiteral* lit) override {
if (isQuotedLiteral(lit->value)) return lit->value;
return "\"" + lit->value + "\"";
}
std::string visitBooleanLiteral(const BooleanLiteral* lit) override {
return lit->value ? "true" : "false";
}
std::string visitNullLiteral(const NullLiteral*) override {
return "null";
}
std::string visitIfStatement(const IfStatement* stmt) override {
std::ostringstream oss;
auto condition = stmt->getChild("condition");
auto thenBranch = stmt->getChildren("thenBranch");
auto elseBranch = stmt->getChildren("elseBranch");
oss << "if (";
if (condition) {
oss << generate(condition);
} else {
oss << "true";
}
oss << ") {\n";
emitBody(oss, thenBranch, " ");
oss << "}";
if (!elseBranch.empty()) {
oss << " else {\n";
emitBody(oss, elseBranch, " ");
oss << "}";
}
return oss.str();
}
std::string visitWhileLoop(const WhileLoop* loop) override {
std::ostringstream oss;
auto condition = loop->getChild("condition");
auto body = loop->getChildren("body");
oss << "while (";
if (condition) {
oss << generate(condition);
} else {
oss << "true";
}
oss << ") {\n";
emitBody(oss, body, " ");
oss << "}";
return oss.str();
}
std::string visitForLoop(const ForLoop* loop) override {
std::ostringstream oss;
auto iterable = loop->getChild("iterable");
auto body = loop->getChildren("body");
std::string iterName = loop->iteratorName.empty() ? "item" : loop->iteratorName;
oss << "for (var " << iterName << " : ";
if (iterable) {
oss << generate(iterable);
} else {
oss << "java.util.List.of()";
}
oss << ") {\n";
emitBody(oss, body, " ");
oss << "}";
return oss.str();
}
std::string visitExpressionStatement(const ExpressionStatement* stmt) override {
std::ostringstream oss;
auto expr = stmt->getChild("expression");
if (expr) {
oss << generate(expr);
}
oss << ";";
return oss.str();
}
std::string visitUnaryOperation(const UnaryOperation* unOp) override {
std::ostringstream oss;
std::string op = unOp->op;
bool wordOp = !op.empty() && std::isalpha(static_cast<unsigned char>(op[0]));
oss << op;
if (wordOp) oss << " ";
auto operand = unOp->getChild("operand");
if (operand) {
oss << generate(operand);
} else {
oss << "null";
}
return oss.str();
}
std::string visitFunctionCall(const FunctionCall* call) override {
std::ostringstream oss;
oss << call->functionName << "(";
auto arguments = call->getChildren("arguments");
for (size_t i = 0; i < arguments.size(); ++i) {
if (i > 0) oss << ", ";
oss << generate(arguments[i]);
}
oss << ")";
return oss.str();
}
std::string visitBlock(const Block* block) override {
std::ostringstream oss;
auto statements = block->getChildren("statements");
oss << "{\n";
emitBody(oss, statements, " ");
oss << "}";
return oss.str();
}
std::string visitListLiteral(const ListLiteral* lit) override {
std::ostringstream oss;
oss << "java.util.List.of(";
auto elements = lit->getChildren("elements");
for (size_t i = 0; i < elements.size(); ++i) {
if (i > 0) oss << ", ";
oss << generate(elements[i]);
}
oss << ")";
return oss.str();
}
std::string visitIndexAccess(const IndexAccess* access) override {
std::ostringstream oss;
auto target = access->getChild("target");
auto index = access->getChild("index");
if (target) {
oss << generate(target);
} else {
oss << "/* missing target */";
}
oss << "[";
if (index) {
oss << generate(index);
} else {
oss << "0";
}
oss << "]";
return oss.str();
}
std::string visitMemberAccess(const MemberAccess* access) override {
std::ostringstream oss;
auto target = access->getChild("target");
if (target) {
oss << generate(target);
} else {
oss << "/* missing target */";
}
oss << "." << access->memberName;
return oss.str();
}
std::string visitPrimitiveType(const PrimitiveType* type) override {
std::string kind = type->kind;
if (kind == "string") return "String";
if (kind == "bool") return "boolean";
if (kind == "float") return "float";
if (kind == "double") return "double";
if (kind == "int" || kind == "long" || kind == "short" || kind == "byte") return kind;
if (kind == "char") return "char";
if (kind == "void") return "void";
return kind;
}
std::string visitListType(const ListType* type) override {
std::ostringstream oss;
oss << "java.util.List<";
auto elementType = type->getChild("elementType");
if (elementType) {
oss << generate(elementType);
} else {
oss << "Object";
}
oss << ">";
return oss.str();
}
std::string visitSetType(const SetType* type) override {
std::ostringstream oss;
oss << "java.util.Set<";
auto elementType = type->getChild("elementType");
if (elementType) {
oss << generate(elementType);
} else {
oss << "Object";
}
oss << ">";
return oss.str();
}
std::string visitMapType(const MapType* type) override {
std::ostringstream oss;
oss << "java.util.Map<";
auto keyType = type->getChild("keyType");
auto valueType = type->getChild("valueType");
if (keyType) {
oss << generate(keyType);
} else {
oss << "Object";
}
oss << ", ";
if (valueType) {
oss << generate(valueType);
} else {
oss << "Object";
}
oss << ">";
return oss.str();
}
std::string visitTupleType(const TupleType* type) override {
std::ostringstream oss;
oss << "java.util.List<";
auto elementTypes = type->getChildren("elementTypes");
if (!elementTypes.empty()) {
oss << generate(elementTypes[0]);
} else {
oss << "Object";
}
oss << ">";
return oss.str();
}
std::string visitArrayType(const ArrayType* type) override {
std::ostringstream oss;
auto elementType = type->getChild("elementType");
if (elementType) {
oss << generate(elementType);
} else {
oss << "Object";
}
oss << "[]";
return oss.str();
}
std::string visitOptionalType(const OptionalType* type) override {
std::ostringstream oss;
oss << "java.util.Optional<";
auto inner = type->getChild("innerType");
if (inner) {
oss << generate(inner);
} else {
oss << "Object";
}
oss << ">";
return oss.str();
}
std::string visitCustomType(const CustomType* type) override {
return type->typeName;
}
std::string visitDerefStrategy(const DerefStrategy* annotation) override {
return "// @deref(" + annotation->strategy + ")";
}
std::string visitOptimizationLock(const OptimizationLock* annotation) override {
return "// @lock(" + annotation->lockedBy + ") - " + annotation->lockReason;
}
std::string visitLangSpecific(const LangSpecific* annotation) override {
if (annotation->language == "java" && !annotation->rawSyntax.empty()) {
return "// " + annotation->rawSyntax;
}
return "// @lang_specific(" + annotation->language + ", " + annotation->idiomType + ")";
}
std::string visitDeallocateAnnotation(const DeallocateAnnotation* annotation) override {
return "// @dealloc(" + annotation->strategy + ")";
}
std::string visitLifetimeAnnotation(const LifetimeAnnotation* annotation) override {
return "// @lifetime(" + annotation->strategy + ")";
}
std::string visitReclaimAnnotation(const ReclaimAnnotation* annotation) override {
return "// @reclaim(" + annotation->strategy + ") - GC-managed";
}
std::string visitOwnerAnnotation(const OwnerAnnotation* annotation) override {
if (annotation->strategy == "Single") {
return "// @owner(Single) - AutoCloseable/try-with-resources";
}
return "// @owner(" + annotation->strategy + ")";
}
std::string visitAllocateAnnotation(const AllocateAnnotation* annotation) override {
return "// @allocate(" + annotation->strategy + ")";
}
std::string visitHotColdAnnotation(const HotColdAnnotation* annotation) override {
return "// @" + annotation->hint;
}
std::string visitInlineAnnotation(const InlineAnnotation* annotation) override {
return "// @inline(" + annotation->mode + ")";
}
std::string visitPureAnnotation(const PureAnnotation*) override {
return "// @pure";
}
std::string visitConstExprAnnotation(const ConstExprAnnotation*) override {
return "// @constexpr";
}
private:
static std::string emitImport(const Import* imp) {
std::string moduleName = imp->moduleName.empty() ? "module" : imp->moduleName;
if (imp->importKind == "require") {
return "// require " + moduleName;
}
return "import " + moduleName + ";";
}
static void emitAnnotations(std::ostringstream& oss,
const std::vector<ASTNode*>& annotations,
const std::string& indent) {
for (const auto* annotation : annotations) {
if (!annotation) continue;
std::string code;
if (annotation->conceptType == "LangSpecific") {
auto* lang = static_cast<const LangSpecific*>(annotation);
if (lang->language == "java" && !lang->rawSyntax.empty()) {
code = "// " + lang->rawSyntax;
}
}
if (code.empty()) {
JavaGenerator gen;
code = gen.generate(annotation);
}
if (!code.empty()) {
oss << indent << code << "\n";
}
}
}
void emitParameters(std::ostringstream& oss,
const std::vector<ASTNode*>& params) {
for (size_t i = 0; i < params.size(); ++i) {
if (i > 0) oss << ", ";
oss << visitParameter(static_cast<const Parameter*>(params[i]));
}
}
void emitBody(std::ostringstream& oss,
const std::vector<ASTNode*>& body,
const std::string& indent) {
if (body.empty()) {
oss << indent << "// TODO: implement\n";
return;
}
for (const auto* stmt : body) {
std::string stmtCode = generate(stmt);
appendIndented(oss, stmtCode, indent);
oss << "\n";
}
}
void emitClass(std::ostringstream& oss,
const std::string& className,
const std::vector<const Variable*>& fields,
const std::vector<const Function*>& methods) {
oss << "class " << className << " {\n";
for (const auto* field : fields) {
oss << " " << visitVariable(field) << "\n";
}
if (!fields.empty() && !methods.empty()) oss << "\n";
for (const auto* method : methods) {
emitMethod(oss, className, method);
oss << "\n";
}
oss << "}";
}
void emitMethod(std::ostringstream& oss,
const std::string& className,
const Function* method) {
std::string methodName = methodNameFromQualified(method->name, className);
bool isConstructor = (methodName == "constructor" || methodName == className);
emitAnnotations(oss, method->getChildren("annotations"), " ");
oss << " ";
if (!isConstructor) {
std::string returnType = "void";
auto retType = method->getChild("returnType");
if (retType) returnType = generate(retType);
oss << returnType << " ";
oss << methodName;
} else {
oss << className;
}
oss << "(";
emitParameters(oss, method->getChildren("parameters"));
oss << ") {\n";
emitBody(oss, method->getChildren("body"), " ");
oss << " }";
}
static void appendIndented(std::ostringstream& oss,
const std::string& code,
const std::string& indent) {
size_t pos = 0;
while (pos < code.length()) {
size_t newlinePos = code.find('\n', pos);
if (newlinePos == std::string::npos) {
oss << indent << code.substr(pos);
break;
}
oss << indent << code.substr(pos, newlinePos - pos) << "\n";
pos = newlinePos + 1;
if (pos >= code.length()) break;
}
}
static std::string methodNameFromQualified(const std::string& name,
const std::string& className) {
if (name.rfind(className + ".", 0) == 0) {
return name.substr(className.size() + 1);
}
auto pos = name.find('.');
if (pos != std::string::npos && pos + 1 < name.size()) {
return name.substr(pos + 1);
}
return name;
}
static std::string stripClassPrefix(const std::string& name) {
auto pos = name.find('.');
if (pos != std::string::npos && pos + 1 < name.size()) {
return name.substr(pos + 1);
}
return name;
}
static bool isQuotedLiteral(const std::string& text) {
if (text.size() < 2) return false;
char first = text.front();
char last = text.back();
if ((first == '\'' && last == '\'') || (first == '"' && last == '"')) {
return true;
}
return false;
}
};

586
editor/src/ast/Parser.h
View File

@@ -4,6 +4,7 @@
#include "Function.h"
#include "Variable.h"
#include "Parameter.h"
#include "Import.h"
#include "Statement.h"
#include "Expression.h"
#include "Type.h"
@@ -23,6 +24,7 @@ extern "C" {
const TSLanguage* tree_sitter_elisp();
const TSLanguage* tree_sitter_javascript();
const TSLanguage* tree_sitter_typescript();
const TSLanguage* tree_sitter_java();
}
// Unique ID generator for AST nodes
@@ -271,6 +273,49 @@ public:
return result;
}
// ---------------------------------------------------------------
// Java
// ---------------------------------------------------------------
static std::unique_ptr<Module> parseJava(const std::string& source) {
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, tree_sitter_java());
TSTree* tree = ts_parser_parse_string(parser, nullptr, source.c_str(), (uint32_t)source.size());
TSNode root = ts_tree_root_node(tree);
auto module = std::make_unique<Module>();
module->id = IdGenerator::next("mod");
module->name = "parsed_java_module";
module->targetLanguage = "java";
applySpan(module.get(), root);
convertJavaCompilationUnit(root, source, module.get());
ts_tree_delete(tree);
ts_parser_delete(parser);
return module;
}
static ParseResult parseJavaWithDiagnostics(const std::string& source) {
ParseResult result;
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, tree_sitter_java());
TSTree* tree = ts_parser_parse_string(parser, nullptr, source.c_str(), (uint32_t)source.size());
TSNode root = ts_tree_root_node(tree);
result.module = std::make_unique<Module>();
result.module->id = IdGenerator::next("mod");
result.module->name = "parsed_java_module";
result.module->targetLanguage = "java";
applySpan(result.module.get(), root);
convertJavaCompilationUnit(root, source, result.module.get());
collectDiagnostics(root, source, result.diagnostics);
ts_tree_delete(tree);
ts_parser_delete(parser);
return result;
}
private:
// ---------------------------------------------------------------
// Helpers
@@ -1607,4 +1652,545 @@ private:
}
return nullptr;
}
// ---------------------------------------------------------------
// Java CST -> AST
// ---------------------------------------------------------------
static void convertJavaCompilationUnit(TSNode root,
const std::string& source,
Module* module) {
uint32_t count = ts_node_named_child_count(root);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(root, i);
std::string type = nodeType(child);
if (type == "import_declaration") {
std::string importName;
uint32_t ic = ts_node_named_child_count(child);
for (uint32_t j = 0; j < ic; ++j) {
TSNode part = ts_node_named_child(child, j);
std::string pType = nodeType(part);
if (pType == "scoped_identifier" || pType == "identifier") {
importName = nodeText(part, source);
break;
}
}
if (importName.empty()) {
importName = nodeText(child, source);
}
if (!importName.empty()) {
auto* imp = new Import(IdGenerator::next("imp"), importName, "module");
module->addChild("imports", imp);
}
} else if (type == "class_declaration" || type == "interface_declaration" ||
type == "enum_declaration" || type == "record_declaration" ||
type == "annotation_type_declaration") {
convertJavaTypeDeclaration(child, source, module);
}
}
}
static void convertJavaTypeDeclaration(TSNode node,
const std::string& source,
Module* module) {
TSNode nameNode = childByFieldName(node, "name");
std::string className = nodeText(nameNode, source);
TSNode bodyNode = childByFieldName(node, "body");
if (ts_node_is_null(bodyNode)) return;
uint32_t count = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(bodyNode, i);
std::string type = nodeType(child);
if (type == "method_declaration") {
auto* fn = convertJavaMethod(child, source, className);
if (fn) module->addChild("functions", fn);
} else if (type == "constructor_declaration" || type == "compact_constructor_declaration") {
auto* fn = convertJavaConstructor(child, source, className);
if (fn) module->addChild("functions", fn);
} else if (type == "field_declaration") {
convertJavaFieldDeclaration(child, source, module, className);
} else if (type == "class_declaration" || type == "interface_declaration" ||
type == "enum_declaration" || type == "record_declaration" ||
type == "annotation_type_declaration") {
convertJavaTypeDeclaration(child, source, module);
}
}
}
static void convertJavaFieldDeclaration(TSNode node,
const std::string& source,
Module* module,
const std::string& className) {
TSNode typeNode = childByFieldName(node, "type");
uint32_t count = ts_node_named_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(node, i);
if (nodeType(child) != "variable_declarator") continue;
TSNode nameNode = childByFieldName(child, "name");
if (ts_node_is_null(nameNode)) continue;
std::string varName = nodeText(nameNode, source);
if (!className.empty()) varName = className + "." + varName;
auto* var = new Variable(IdGenerator::next("var"), varName);
applySpan(var, child);
if (!ts_node_is_null(typeNode)) {
if (auto* t = convertJavaType(typeNode, source)) var->setChild("type", t);
}
TSNode valueNode = childByFieldName(child, "value");
if (!ts_node_is_null(valueNode)) {
ASTNode* init = convertJavaExpression(valueNode, source);
if (init) var->setChild("initializer", init);
}
module->addChild("variables", var);
}
}
static Function* convertJavaMethod(TSNode node,
const std::string& source,
const std::string& className) {
TSNode nameNode = childByFieldName(node, "name");
if (ts_node_is_null(nameNode)) return nullptr;
auto* fn = new Function();
fn->id = IdGenerator::next("fn");
applySpan(fn, node);
std::string methodName = nodeText(nameNode, source);
fn->name = className.empty() ? methodName : (className + "." + methodName);
TSNode typeNode = childByFieldName(node, "type");
if (!ts_node_is_null(typeNode)) {
if (auto* t = convertJavaType(typeNode, source)) fn->setChild("returnType", t);
}
TSNode paramsNode = childByFieldName(node, "parameters");
if (!ts_node_is_null(paramsNode)) {
convertJavaParameters(paramsNode, source, fn);
}
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
convertJavaBlockStatements(bodyNode, source, fn);
}
attachJavaAnnotations(node, source, fn);
auto* reclaim = new ReclaimAnnotation(IdGenerator::next("anno"), "Tracing");
fn->addChild("annotations", reclaim);
return fn;
}
static Function* convertJavaConstructor(TSNode node,
const std::string& source,
const std::string& className) {
if (className.empty()) return nullptr;
auto* fn = new Function();
fn->id = IdGenerator::next("fn");
applySpan(fn, node);
fn->name = className + ".constructor";
TSNode paramsNode = childByFieldName(node, "parameters");
if (!ts_node_is_null(paramsNode)) {
convertJavaParameters(paramsNode, source, fn);
}
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
convertJavaBlockStatements(bodyNode, source, fn);
}
attachJavaAnnotations(node, source, fn);
auto* reclaim = new ReclaimAnnotation(IdGenerator::next("anno"), "Tracing");
fn->addChild("annotations", reclaim);
return fn;
}
static void convertJavaParameters(TSNode paramsNode,
const std::string& source,
Function* fn) {
uint32_t count = ts_node_named_child_count(paramsNode);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(paramsNode, i);
std::string type = nodeType(child);
if (type == "formal_parameter" || type == "spread_parameter") {
TSNode nameNode = childByFieldName(child, "name");
if (ts_node_is_null(nameNode)) {
nameNode = findDescendantByType(child, "_variable_declarator_id");
if (!ts_node_is_null(nameNode)) {
TSNode innerName = childByFieldName(nameNode, "name");
if (!ts_node_is_null(innerName)) nameNode = innerName;
}
}
if (ts_node_is_null(nameNode)) continue;
std::string name = nodeText(nameNode, source);
if (type == "spread_parameter") name = "..." + name;
auto* param = new Parameter(IdGenerator::next("param"), name);
applySpan(param, child);
TSNode typeNode = childByFieldName(child, "type");
if (!ts_node_is_null(typeNode)) {
if (auto* t = convertJavaType(typeNode, source)) param->setChild("type", t);
}
fn->addChild("parameters", param);
}
}
}
static void convertJavaBlockStatements(TSNode bodyNode,
const std::string& source,
Function* fn) {
uint32_t count = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < count; ++i) {
ASTNode* stmt = convertJavaStatement(ts_node_named_child(bodyNode, i), source);
if (stmt) fn->addChild("body", stmt);
}
}
static ASTNode* convertJavaStatement(TSNode node,
const std::string& source) {
std::string type = nodeType(node);
if (type == "return_statement") {
auto* ret = new Return();
ret->id = IdGenerator::next("ret");
applySpan(ret, node);
if (ts_node_named_child_count(node) > 0) {
ASTNode* val = convertJavaExpression(ts_node_named_child(node, 0), source);
if (val) ret->setChild("value", val);
}
return ret;
} else if (type == "expression_statement") {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, node);
if (ts_node_named_child_count(node) > 0) {
ASTNode* expr = convertJavaExpression(ts_node_named_child(node, 0), source);
if (expr) exprStmt->setChild("expression", expr);
}
return exprStmt;
} else if (type == "local_variable_declaration") {
uint32_t count = ts_node_named_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(node, i);
if (nodeType(child) != "variable_declarator") continue;
TSNode nameNode = childByFieldName(child, "name");
if (ts_node_is_null(nameNode)) continue;
auto* assign = new Assignment();
assign->id = IdGenerator::next("assign");
applySpan(assign, child);
auto* target = new VariableReference(IdGenerator::next("var"), nodeText(nameNode, source));
applySpan(target, nameNode);
assign->setChild("target", target);
TSNode valueNode = childByFieldName(child, "value");
if (!ts_node_is_null(valueNode)) {
ASTNode* val = convertJavaExpression(valueNode, source);
if (val) assign->setChild("value", val);
}
return assign;
}
} else if (type == "if_statement") {
auto* ifStmt = new IfStatement();
ifStmt->id = IdGenerator::next("if");
applySpan(ifStmt, node);
TSNode condNode = childByFieldName(node, "condition");
if (!ts_node_is_null(condNode)) {
ASTNode* cond = convertJavaExpression(condNode, source);
if (cond) ifStmt->setChild("condition", cond);
}
TSNode consNode = childByFieldName(node, "consequence");
if (!ts_node_is_null(consNode)) {
uint32_t cc = ts_node_named_child_count(consNode);
for (uint32_t i = 0; i < cc; ++i) {
ASTNode* s = convertJavaStatement(ts_node_named_child(consNode, i), source);
if (s) ifStmt->addChild("thenBranch", s);
}
}
TSNode altNode = childByFieldName(node, "alternative");
if (!ts_node_is_null(altNode)) {
uint32_t ac = ts_node_named_child_count(altNode);
for (uint32_t i = 0; i < ac; ++i) {
ASTNode* s = convertJavaStatement(ts_node_named_child(altNode, i), source);
if (s) ifStmt->addChild("elseBranch", s);
}
}
return ifStmt;
} else if (type == "while_statement") {
auto* loop = new WhileLoop();
loop->id = IdGenerator::next("while");
applySpan(loop, node);
TSNode condNode = childByFieldName(node, "condition");
if (!ts_node_is_null(condNode)) {
ASTNode* cond = convertJavaExpression(condNode, source);
if (cond) loop->setChild("condition", cond);
}
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
uint32_t bc = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < bc; ++i) {
ASTNode* s = convertJavaStatement(ts_node_named_child(bodyNode, i), source);
if (s) loop->addChild("body", s);
}
}
return loop;
} else if (type == "for_statement") {
auto* loop = new WhileLoop();
loop->id = IdGenerator::next("for");
applySpan(loop, node);
TSNode condNode = childByFieldName(node, "condition");
if (!ts_node_is_null(condNode)) {
ASTNode* cond = convertJavaExpression(condNode, source);
if (cond) loop->setChild("condition", cond);
}
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
uint32_t bc = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < bc; ++i) {
ASTNode* s = convertJavaStatement(ts_node_named_child(bodyNode, i), source);
if (s) loop->addChild("body", s);
}
}
return loop;
} else if (type == "enhanced_for_statement") {
auto* loop = new ForLoop();
loop->id = IdGenerator::next("for");
applySpan(loop, node);
TSNode nameNode = childByFieldName(node, "name");
if (!ts_node_is_null(nameNode)) {
loop->iteratorName = nodeText(nameNode, source);
}
TSNode valueNode = childByFieldName(node, "value");
if (!ts_node_is_null(valueNode)) {
ASTNode* iter = convertJavaExpression(valueNode, source);
if (iter) loop->setChild("iterable", iter);
}
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
uint32_t bc = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < bc; ++i) {
ASTNode* s = convertJavaStatement(ts_node_named_child(bodyNode, i), source);
if (s) loop->addChild("body", s);
}
}
return loop;
} else if (type == "block") {
auto* block = new Block();
block->id = IdGenerator::next("block");
applySpan(block, node);
uint32_t bc = ts_node_named_child_count(node);
for (uint32_t i = 0; i < bc; ++i) {
ASTNode* s = convertJavaStatement(ts_node_named_child(node, i), source);
if (s) block->addChild("statements", s);
}
return block;
} else if (type == "try_statement") {
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
auto* block = new Block();
block->id = IdGenerator::next("block");
applySpan(block, bodyNode);
uint32_t bc = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < bc; ++i) {
ASTNode* s = convertJavaStatement(ts_node_named_child(bodyNode, i), source);
if (s) block->addChild("statements", s);
}
return block;
}
}
ASTNode* expr = convertJavaExpression(node, source);
if (expr) {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, node);
exprStmt->setChild("expression", expr);
return exprStmt;
}
return nullptr;
}
static ASTNode* convertJavaExpression(TSNode node,
const std::string& source) {
std::string type = nodeType(node);
if (type == "binary_expression") {
auto* binOp = new BinaryOperation();
binOp->id = IdGenerator::next("binop");
applySpan(binOp, node);
TSNode leftNode = childByFieldName(node, "left");
TSNode rightNode = childByFieldName(node, "right");
TSNode opNode = childByFieldName(node, "operator");
if (!ts_node_is_null(opNode)) binOp->op = nodeText(opNode, source);
if (!ts_node_is_null(leftNode)) {
ASTNode* left = convertJavaExpression(leftNode, source);
if (left) binOp->setChild("left", left);
}
if (!ts_node_is_null(rightNode)) {
ASTNode* right = convertJavaExpression(rightNode, source);
if (right) binOp->setChild("right", right);
}
return binOp;
} else if (type == "assignment_expression") {
auto* assign = new Assignment();
assign->id = IdGenerator::next("assign");
applySpan(assign, node);
TSNode leftNode = childByFieldName(node, "left");
TSNode rightNode = childByFieldName(node, "right");
if (!ts_node_is_null(leftNode)) {
ASTNode* target = convertJavaExpression(leftNode, source);
if (target) assign->setChild("target", target);
}
if (!ts_node_is_null(rightNode)) {
ASTNode* value = convertJavaExpression(rightNode, source);
if (value) assign->setChild("value", value);
}
return assign;
} else if (type == "method_invocation") {
auto* call = new FunctionCall();
call->id = IdGenerator::next("call");
applySpan(call, node);
TSNode nameNode = childByFieldName(node, "name");
TSNode objectNode = childByFieldName(node, "object");
if (!ts_node_is_null(objectNode) && !ts_node_is_null(nameNode)) {
call->functionName = nodeText(objectNode, source) + "." + nodeText(nameNode, source);
} else if (!ts_node_is_null(nameNode)) {
call->functionName = nodeText(nameNode, source);
} else {
call->functionName = nodeText(node, source);
}
TSNode argsNode = childByFieldName(node, "arguments");
if (!ts_node_is_null(argsNode)) {
uint32_t count = ts_node_named_child_count(argsNode);
for (uint32_t i = 0; i < count; ++i) {
ASTNode* arg = convertJavaExpression(ts_node_named_child(argsNode, i), source);
if (arg) call->addChild("arguments", arg);
}
}
return call;
} else if (type == "field_access") {
auto* mem = new MemberAccess();
mem->id = IdGenerator::next("member");
applySpan(mem, node);
TSNode objNode = childByFieldName(node, "object");
TSNode fieldNode = childByFieldName(node, "field");
if (!ts_node_is_null(fieldNode)) {
mem->memberName = nodeText(fieldNode, source);
}
if (!ts_node_is_null(objNode)) {
ASTNode* target = convertJavaExpression(objNode, source);
if (target) mem->setChild("target", target);
}
return mem;
} else if (type == "array_access") {
auto* access = new IndexAccess();
access->id = IdGenerator::next("index");
applySpan(access, node);
TSNode arrayNode = childByFieldName(node, "array");
TSNode indexNode = childByFieldName(node, "index");
if (!ts_node_is_null(arrayNode)) {
ASTNode* target = convertJavaExpression(arrayNode, source);
if (target) access->setChild("target", target);
}
if (!ts_node_is_null(indexNode)) {
ASTNode* idx = convertJavaExpression(indexNode, source);
if (idx) access->setChild("index", idx);
}
return access;
} else if (type == "identifier") {
auto* ref = new VariableReference(IdGenerator::next("var"), nodeText(node, source));
applySpan(ref, node);
return ref;
} else if (type == "integer_literal") {
std::string text = nodeText(node, source);
int val = 0;
try { val = std::stoi(text); } catch (...) {}
auto* lit = new IntegerLiteral(IdGenerator::next("int"), val);
applySpan(lit, node);
return lit;
} else if (type == "floating_point_literal") {
auto* lit = new FloatLiteral(IdGenerator::next("float"), nodeText(node, source));
applySpan(lit, node);
return lit;
} else if (type == "string_literal" || type == "character_literal") {
auto* lit = new StringLiteral(IdGenerator::next("str"), nodeText(node, source));
applySpan(lit, node);
return lit;
} else if (type == "true" || type == "false") {
auto* lit = new BooleanLiteral(IdGenerator::next("bool"), type == "true");
applySpan(lit, node);
return lit;
} else if (type == "null_literal") {
auto* lit = new NullLiteral();
lit->id = IdGenerator::next("null");
applySpan(lit, node);
return lit;
} else if (type == "parenthesized_expression") {
if (ts_node_named_child_count(node) > 0) {
return convertJavaExpression(ts_node_named_child(node, 0), source);
}
} else if (type == "lambda_expression") {
auto* ref = new VariableReference(IdGenerator::next("var"), nodeText(node, source));
applySpan(ref, node);
return ref;
}
std::string text = nodeText(node, source);
if (!text.empty()) {
auto* ref = new VariableReference(IdGenerator::next("var"), text);
applySpan(ref, node);
return ref;
}
return nullptr;
}
static Type* convertJavaType(TSNode node, const std::string& source) {
std::string type = nodeType(node);
if (type == "integral_type" || type == "floating_point_type" ||
type == "boolean_type" || type == "void_type") {
auto* prim = new PrimitiveType();
prim->id = IdGenerator::next("type");
prim->kind = nodeText(node, source);
return prim;
} else if (type == "array_type") {
auto* arr = new ArrayType();
arr->id = IdGenerator::next("type");
TSNode element = childByFieldName(node, "element");
if (!ts_node_is_null(element)) {
if (auto* et = convertJavaType(element, source)) arr->setChild("elementType", et);
}
return arr;
}
auto* custom = new CustomType();
custom->id = IdGenerator::next("type");
custom->typeName = nodeText(node, source);
return custom;
}
static void attachJavaAnnotations(TSNode node,
const std::string& source,
ASTNode* target) {
if (!target) return;
uint32_t count = ts_node_named_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(node, i);
if (nodeType(child) != "modifiers") continue;
uint32_t mc = ts_node_named_child_count(child);
for (uint32_t j = 0; j < mc; ++j) {
TSNode modChild = ts_node_named_child(child, j);
std::string mType = nodeType(modChild);
if (mType == "marker_annotation" || mType == "annotation") {
auto* anno = new LangSpecific();
anno->id = IdGenerator::next("anno");
anno->language = "java";
TSNode nameNode = childByFieldName(modChild, "name");
anno->idiomType = ts_node_is_null(nameNode) ? "" : nodeText(nameNode, source);
anno->rawSyntax = nodeText(modChild, source);
target->addChild("annotations", anno);
}
}
}
}
static TSNode findDescendantByType(TSNode node, const std::string& typeName) {
if (nodeType(node) == typeName) return node;
uint32_t count = ts_node_named_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(node, i);
TSNode found = findDescendantByType(child, typeName);
if (!ts_node_is_null(found)) return found;
}
return TSNode{};
}
};

65
editor/tests/step149_test.cpp Normal file
View File

@@ -0,0 +1,65 @@
// Step 149 TDD Test: Java CST-to-AST + generator
#include "ast/Parser.h"
#include "ast/Generator.h"
#include <iostream>
static void expect(bool cond, const std::string& name, int& passed, int& failed) {
if (cond) {
std::cout << "Test " << (passed + failed + 1) << " PASS: " << name << "\n";
++passed;
} else {
std::cout << "Test " << (passed + failed + 1) << " FAIL: " << name << "\n";
++failed;
}
}
int main() {
int passed = 0;
int failed = 0;
std::string java = R"(
import java.util.List;
public class Box {
@Deprecated
public int size(int n) { return n + 1; }
public Box() { }
}
)";
auto mod = TreeSitterParser::parseJava(java);
expect(mod != nullptr, "java module created", passed, failed);
auto funcs = mod->getChildren("functions");
expect(funcs.size() >= 1, "java functions discovered", passed, failed);
bool foundSize = false;
bool foundIntReturn = false;
for (const auto* fnNode : funcs) {
if (fnNode->conceptType != "Function") continue;
auto* fn = static_cast<const Function*>(fnNode);
if (fn->name == "Box.size") {
foundSize = true;
auto* retType = fn->getChild("returnType");
if (retType && retType->conceptType == "PrimitiveType") {
auto* prim = static_cast<const PrimitiveType*>(retType);
foundIntReturn = (prim->kind == "int");
}
}
}
expect(foundSize, "java method named Box.size", passed, failed);
expect(foundIntReturn, "java return type parsed", passed, failed);
JavaGenerator gen;
std::string out = gen.generate(mod.get());
expect(out.find("import java.util.List;") != std::string::npos,
"java imports generated", passed, failed);
expect(out.find("class Box") != std::string::npos,
"java class generated", passed, failed);
expect(out.find("int size") != std::string::npos,
"java method signature generated", passed, failed);
expect(out.find("Box(") != std::string::npos,
"java constructor generated", passed, failed);
std::cout << "\n=== Step 149 Results: " << passed << " passed, "
<< failed << " failed ===\n";
return failed == 0 ? 0 : 1;
}

2
sprint5_plan.md
View File

@@ -243,7 +243,7 @@ adds full semantic support.
for documentation.
*Modifies:* `Generator.h`
- [ ] **Step 149: Java CST-to-AST and generator**
- [x] **Step 149: Java CST-to-AST and generator**
Full tree-sitter Java parsing: classes, interfaces, methods, generics,
annotations (Java @annotations map to Whetstone annotations), try/catch,
lambdas. Generator produces compilable Java with proper class wrapping,