Step 168: split oversized headers

This commit is contained in:
Bill
2026-02-09 21:09:24 -07:00
parent de13e514e1
commit 08b7d71a38
31 changed files with 5084 additions and 4864 deletions

View File

@@ -515,3 +515,4 @@ Sprint 5 in progress. Step 141 (Emacs daemon with user config) done. Next: Step
| 2026-02-10 | Codex | Step 165: Sprint 5 integration tests across primitives, projection, pipeline, composition, and Emacs indexing. 8/8 tests pass. |
| 2026-02-10 | Codex | Added `file_limits_test` to enforce architecture file size limits (with temporary allowlist for known oversize headers). 4/4 tests pass. |
| 2026-02-10 | Codex | Step 167: Split EditorState into focused sub-states (Search/Agent/Build/Library/Emacs/UI), updated panels/handlers, and added step167_test. 1/1 tests pass. |
| 2026-02-10 | Codex | Step 168: Split oversized editor/AST component headers (CodeEditorWidget, SyntaxHighlighter, Parser, CppGenerator) with new step168 unit + integration tests. 79/79 tests pass (step168_test 75/75, step168_integration_test 4/4). file_limits_test 4/4 passes. |

View File

@@ -951,6 +951,22 @@ target_include_directories(step167_test PRIVATE src)
target_link_libraries(step167_test PRIVATE
nlohmann_json::nlohmann_json)
add_executable(step168_test tests/step168_test.cpp)
target_include_directories(step168_test PRIVATE src)
add_executable(step168_integration_test tests/step168_integration_test.cpp)
target_include_directories(step168_integration_test PRIVATE src)
target_link_libraries(step168_integration_test PRIVATE
unofficial::tree-sitter::tree-sitter
tree_sitter_python
tree_sitter_cpp
tree_sitter_elisp
tree_sitter_javascript
tree_sitter_typescript
tree_sitter_java
tree_sitter_rust
tree_sitter_go)
find_package(SDL2 CONFIG REQUIRED)
find_package(OpenGL REQUIRED)
find_package(glad CONFIG REQUIRED)

View File

@@ -0,0 +1,284 @@
#pragma once
// Included inside CodeEditorWidget (render helpers).
static ImU32 colorFor(TokenCategory cat) {
ImU32 fallback = IM_COL32(220, 220, 220, 255);
switch (cat) {
case TokenCategory::Keyword: fallback = IM_COL32(196, 126, 220, 255); break;
case TokenCategory::String: fallback = IM_COL32(207, 138, 94, 255); break;
case TokenCategory::Comment: fallback = IM_COL32(107, 133, 89, 255); break;
case TokenCategory::Number: fallback = IM_COL32(181, 204, 140, 255); break;
case TokenCategory::Function: fallback = IM_COL32(220, 220, 140, 255); break;
case TokenCategory::Parameter: fallback = IM_COL32(153, 199, 229, 255); break;
case TokenCategory::Type: fallback = IM_COL32(77, 179, 174, 255); break;
case TokenCategory::Operator: fallback = IM_COL32(220, 220, 220, 255); break;
case TokenCategory::Punctuation: fallback = IM_COL32(160, 160, 160, 255); break;
case TokenCategory::Builtin: fallback = IM_COL32(77, 179, 229, 255); break;
case TokenCategory::Identifier: fallback = IM_COL32(160, 200, 230, 255); break;
default: break;
}
return ThemeEngine::instance().syntaxColor(cat, fallback);
}
static int lineFromPos(int pos, const std::vector<int>& lineStarts) {
if (lineStarts.empty()) return 0;
int line = 0;
for (int i = 0; i < (int)lineStarts.size(); ++i) {
if (lineStarts[i] <= pos) line = i;
else break;
}
return line;
}
static int lineFromMouseY(float mouseY, float baseY, float lineHeight, int lineCount) {
int line = (int)((mouseY - baseY) / lineHeight);
line = std::max(0, std::min(line, lineCount - 1));
return line;
}
static int positionFromMouse(const ImVec2& mouse,
const ImVec2& base,
const std::vector<int>& lineStarts,
const std::string& text,
float charAdvance,
float lineHeight) {
int line = (int)((mouse.y - base.y) / lineHeight);
line = std::max(0, std::min(line, (int)lineStarts.size() - 1));
int start = lineStarts[line];
int end = (line + 1 < (int)lineStarts.size()) ? lineStarts[line + 1] - 1 : (int)text.size();
int len = std::max(0, end - start);
int col = (int)((mouse.x - base.x) / charAdvance);
col = std::max(0, std::min(col, len));
return start + col;
}
static float calcGutterWidth(int lineCount, ImFont* font, float charAdvance) {
int digits = 1;
int n = std::max(1, lineCount);
while (n >= 10) { n /= 10; ++digits; }
const float pad = 6.0f;
float digitsWidth = font->CalcTextSizeA(font->FontSize, FLT_MAX, -1.0f,
std::string(digits, '0').c_str()).x;
float width = digitsWidth + pad * 2.0f;
return std::max(width, charAdvance * 2.0f + pad * 2.0f);
}
const FoldRegion* findFoldAtLine(int line) const {
for (const auto& f : folds_) {
if (f.startLine == line) return &f;
}
return nullptr;
}
static void drawTriangle(ImDrawList* drawList, const ImVec2& center, ImU32 color, bool down) {
float s = 4.0f;
if (down) {
drawList->AddTriangleFilled(
ImVec2(center.x - s, center.y - s * 0.6f),
ImVec2(center.x + s, center.y - s * 0.6f),
ImVec2(center.x, center.y + s),
color);
} else {
drawList->AddTriangleFilled(
ImVec2(center.x - s, center.y - s),
ImVec2(center.x - s, center.y + s),
ImVec2(center.x + s, center.y),
color);
}
}
static bool lineIn(const std::vector<int>* lines, int line) {
if (!lines) return false;
return std::find(lines->begin(), lines->end(), line) != lines->end();
}
static bool annotationAtLine(const std::vector<AnnotationMarker>& markers,
int line,
AnnotationMarker& out) {
for (const auto& m : markers) {
if (m.line == line) {
out = m;
return true;
}
}
return false;
}
static bool suggestionAtLine(const std::vector<SuggestionMarker>& markers,
int line,
SuggestionMarker& out) {
for (const auto& m : markers) {
if (m.line == line) {
out = m;
return true;
}
}
return false;
}
static bool conflictAtLine(const std::vector<AnnotationConflictMarker>& markers,
int line,
AnnotationConflictMarker& out) {
for (const auto& m : markers) {
if (m.childLine == line || m.parentLine == line) {
out = m;
return true;
}
}
return false;
}
static std::string diagnosticMessageAtLine(const std::vector<DiagnosticRange>& diags, int line) {
for (const auto& d : diags) {
if (line >= d.startLine && line <= d.endLine) return d.message;
}
return "";
}
static void renderSquiggles(const std::vector<DiagnosticRange>& diags,
int line,
float y,
float lineHeight,
float charAdvance,
float textBaseX,
int lineLen,
ImDrawList* drawList) {
float baseY = y + lineHeight - 2.0f;
for (const auto& d : diags) {
if (line < d.startLine || line > d.endLine) continue;
int startCol = (line == d.startLine) ? d.startCol : 0;
int endCol = (line == d.endLine) ? d.endCol : lineLen;
startCol = std::max(0, startCol);
endCol = std::max(startCol, endCol);
float xStart = textBaseX + startCol * charAdvance;
float xEnd = textBaseX + endCol * charAdvance;
ImU32 color = (d.severity == 1)
? ThemeEngine::instance().editorColor("diag_error", IM_COL32(220, 80, 80, 255))
: ThemeEngine::instance().editorColor("diag_warning", IM_COL32(220, 160, 60, 255));
float x = xStart;
float amp = 2.0f;
bool up = true;
while (x < xEnd) {
float x2 = std::min(x + 4.0f, xEnd);
float y1 = baseY + (up ? -amp : amp);
float y2 = baseY + (up ? amp : -amp);
drawList->AddLine(ImVec2(x, y1), ImVec2(x2, y2), color, 1.0f);
up = !up;
x = x2;
}
}
}
static std::string diagnosticMessageAtPoint(const std::vector<DiagnosticRange>& diags,
int line,
const ImVec2& mouse,
float y,
float lineHeight,
float charAdvance,
float textBaseX) {
if (mouse.y < y || mouse.y > y + lineHeight) return "";
for (const auto& d : diags) {
if (line < d.startLine || line > d.endLine) continue;
int startCol = (line == d.startLine) ? d.startCol : 0;
int endCol = (line == d.endLine) ? d.endCol : startCol + 1;
float xStart = textBaseX + startCol * charAdvance;
float xEnd = textBaseX + endCol * charAdvance;
if (mouse.x >= xStart && mouse.x <= xEnd) return d.message;
}
return "";
}
void updateFolds(const std::string& text, const std::string& language) {
if (text == lastFoldText_ && language == lastFoldLang_) return;
lastFoldText_ = text;
lastFoldLang_ = language;
std::vector<FoldRegion> newFolds;
if (text.empty()) { folds_.clear(); return; }
const TSLanguage* lang = nullptr;
if (language == "python") lang = tree_sitter_python();
else if (language == "cpp") lang = tree_sitter_cpp();
else if (language == "elisp") lang = tree_sitter_elisp();
if (!lang) { folds_.clear(); return; }
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, lang);
TSTree* tree = ts_parser_parse_string(parser, nullptr, text.c_str(), (uint32_t)text.size());
TSNode root = ts_tree_root_node(tree);
collectFoldNodes(root, language, newFolds);
// Preserve folded state by startLine
for (auto& f : newFolds) {
for (const auto& old : folds_) {
if (old.startLine == f.startLine && old.endLine == f.endLine) {
f.folded = old.folded;
break;
}
}
}
folds_ = std::move(newFolds);
if (applyDesiredFoldState_) {
for (auto& f : folds_) {
f.folded = std::find(desiredFoldedLines_.begin(),
desiredFoldedLines_.end(),
f.startLine) != desiredFoldedLines_.end();
}
applyDesiredFoldState_ = false;
}
ts_tree_delete(tree);
ts_parser_delete(parser);
}
static bool isFoldNodeType(const std::string& language, const char* type) {
if (language == "python") {
return strcmp(type, "function_definition") == 0 ||
strcmp(type, "class_definition") == 0 ||
strcmp(type, "if_statement") == 0 ||
strcmp(type, "for_statement") == 0 ||
strcmp(type, "while_statement") == 0 ||
strcmp(type, "with_statement") == 0 ||
strcmp(type, "try_statement") == 0;
} else if (language == "cpp") {
return strcmp(type, "function_definition") == 0 ||
strcmp(type, "class_specifier") == 0 ||
strcmp(type, "struct_specifier") == 0 ||
strcmp(type, "namespace_definition") == 0 ||
strcmp(type, "if_statement") == 0 ||
strcmp(type, "for_statement") == 0 ||
strcmp(type, "while_statement") == 0 ||
strcmp(type, "compound_statement") == 0;
} else if (language == "elisp") {
return strcmp(type, "function_definition") == 0 ||
strcmp(type, "lambda_expression") == 0 ||
strcmp(type, "if_expression") == 0 ||
strcmp(type, "while_expression") == 0 ||
strcmp(type, "cond_expression") == 0;
}
return false;
}
static void collectFoldNodes(TSNode node, const std::string& language, std::vector<FoldRegion>& out) {
if (ts_node_is_null(node)) return;
const char* type = ts_node_type(node);
if (isFoldNodeType(language, type)) {
TSPoint start = ts_node_start_point(node);
TSPoint end = ts_node_end_point(node);
if (end.row > start.row) {
FoldRegion f;
f.startLine = (int)start.row;
f.endLine = (int)end.row;
f.folded = false;
out.push_back(f);
}
}
uint32_t count = ts_node_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
collectFoldNodes(ts_node_child(node, i), language, out);
}
}

View File

@@ -0,0 +1,561 @@
#pragma once
// Included inside CodeEditorWidget (rendering + folds).
public:
CodeEditorResult render(const char* id,
std::string& text,
const std::vector<HighlightSpan>& spans,
const CodeEditorOptions& options,
const ImVec2& size,
ImFont* font) {
CodeEditorResult result;
if (!font) font = ImGui::GetFont();
ImGuiIO& io = ImGui::GetIO();
// Build line start offsets
std::vector<int> lineStarts;
lineStarts.reserve(128);
lineStarts.push_back(0);
for (int i = 0; i < (int)text.size(); ++i) {
if (text[i] == '\n') lineStarts.push_back(i + 1);
}
const int lineCount = (int)lineStarts.size();
// Clamp cursor
if (cursor_ < 0) cursor_ = 0;
if (cursor_ > (int)text.size()) cursor_ = (int)text.size();
// Build per-byte category map
std::vector<TokenCategory> charCats(text.size(), TokenCategory::Plain);
for (const auto& span : spans) {
uint32_t end = std::min<uint32_t>(span.end, (uint32_t)charCats.size());
for (uint32_t i = span.start; i < end; ++i) {
charCats[i] = span.category;
}
}
if (options.enableFolding && options.mode) {
updateFolds(text, options.mode->getLanguage());
} else {
folds_.clear();
}
// Measure
const float lineHeight = ImGui::GetTextLineHeightWithSpacing();
const float charAdvance = font->CalcTextSizeA(font->FontSize, FLT_MAX, -1.0f, "M").x;
const float gutterWidth = options.showLineNumbers ?
calcGutterWidth(lineCount, font, charAdvance) : 12.0f;
const float minimapWidth = options.showMinimap ? 80.0f : 0.0f;
// Estimate content size for scrollbars
int maxLineLen = 0;
for (int ln = 0; ln < lineCount; ++ln) {
int start = lineStarts[ln];
int end = (ln + 1 < lineCount) ? lineStarts[ln + 1] - 1 : (int)text.size();
int len = std::max(0, end - start);
maxLineLen = std::max(maxLineLen, len);
}
ImVec2 contentSize(gutterWidth + maxLineLen * charAdvance + minimapWidth + 4.0f,
lineCount * lineHeight + 4.0f);
ImGui::BeginChild(id, size, false, ImGuiWindowFlags_HorizontalScrollbar);
if (options.syncScrollX && !options.scrollMaster) {
ImGui::SetScrollX(*options.syncScrollX);
}
if (options.syncScrollY && !options.scrollMaster) {
ImGui::SetScrollY(*options.syncScrollY);
}
std::string annoPopupId = std::string(id) + "_AnnoPopup";
ImVec2 origin = ImGui::GetCursorScreenPos();
// Dummy to set scroll extents
ImGui::Dummy(contentSize);
ImDrawList* drawList = ImGui::GetWindowDrawList();
const float scrollX = ImGui::GetScrollX();
const float scrollY = ImGui::GetScrollY();
if (options.syncScrollX && options.scrollMaster) {
*options.syncScrollX = scrollX;
}
if (options.syncScrollY && options.scrollMaster) {
*options.syncScrollY = scrollY;
}
ImVec2 gutterBase(origin.x, origin.y - scrollY);
ImVec2 textBase(origin.x + gutterWidth - scrollX, origin.y - scrollY);
const float windowWidth = ImGui::GetWindowContentRegionMax().x - ImGui::GetWindowContentRegionMin().x;
const float minimapBaseX = origin.x + std::max(0.0f, windowWidth - minimapWidth);
const float minimapBaseY = origin.y - scrollY;
// Focus and input
const bool hovered = ImGui::IsWindowHovered();
const bool focused = ImGui::IsWindowFocused(ImGuiFocusedFlags_RootAndChildWindows);
// Mouse handling
if (hovered && ImGui::IsMouseClicked(ImGuiMouseButton_Left)) {
ImGui::SetKeyboardFocusHere(-1);
const ImVec2 mouse = ImGui::GetMousePos();
if (options.showMinimap && mouse.x >= minimapBaseX) {
float miniHeight = lineCount * lineHeight;
if (miniHeight > 0.0f) {
float localY = mouse.y - minimapBaseY;
float t = std::max(0.0f, std::min(1.0f, localY / miniHeight));
float targetScroll = t * lineCount * lineHeight;
ImGui::SetScrollY(targetScroll);
}
selecting_ = false;
goto after_mouse;
}
bool ctrlClick = io.KeyCtrl || io.KeySuper;
int clickCount = ImGui::GetIO().MouseClickedCount[0];
if (mouse.x < origin.x + gutterWidth || clickCount >= 3) {
int line = lineFromMouseY(mouse.y, gutterBase.y, lineHeight, lineCount);
result.lineClicked = true;
result.clickedLine = line;
const FoldRegion* fold = findFoldAtLine(line);
if (fold && mouse.x < origin.x + 12.0f) {
toggleFoldAtLine(line);
}
int lineStart = lineStarts[line];
int lineEnd = (line + 1 < lineCount) ? lineStarts[line + 1] : (int)text.size();
cursor_ = lineStart;
selStart_ = lineStart;
selEnd_ = lineEnd;
} else {
int pos = positionFromMouse(mouse, textBase, lineStarts, text, charAdvance, lineHeight);
result.lineClicked = true;
result.clickedLine = lineFromPos(pos, lineStarts);
if (ctrlClick) {
result.ctrlClick = true;
result.ctrlClickPos = pos;
result.ctrlClickLine = result.clickedLine;
result.ctrlClickCol = pos - lineStarts[result.ctrlClickLine];
}
if (ImGui::IsMouseDoubleClicked(ImGuiMouseButton_Left)) {
selectWordAt(text, pos);
} else if (ImGui::GetIO().KeyShift) {
if (!hasSelection()) selStart_ = cursor_;
cursor_ = pos;
selEnd_ = cursor_;
} else {
cursor_ = pos;
selStart_ = cursor_;
selEnd_ = cursor_;
}
}
selecting_ = true;
}
after_mouse:
if (hovered && selecting_ && ImGui::IsMouseDown(ImGuiMouseButton_Left)) {
const ImVec2 mouse = ImGui::GetMousePos();
if (mouse.x >= origin.x + gutterWidth) {
int pos = positionFromMouse(mouse, textBase, lineStarts, text, charAdvance, lineHeight);
selEnd_ = pos;
}
}
if (selecting_ && ImGui::IsMouseReleased(ImGuiMouseButton_Left)) {
selecting_ = false;
}
if (hovered) {
const ImVec2 mouse = ImGui::GetMousePos();
if (mouse.x >= origin.x + gutterWidth &&
(!options.showMinimap || mouse.x < minimapBaseX)) {
int pos = positionFromMouse(mouse, textBase, lineStarts, text, charAdvance, lineHeight);
int line = lineFromPos(pos, lineStarts);
result.hoverValid = true;
result.hoverPos = pos;
result.hoverLine = line;
result.hoverCol = pos - lineStarts[line];
}
}
// Keyboard input
if (focused && !options.readOnly) {
handleKeyboard(text, result.changed, lineStarts, options.mode);
}
// Render visible lines
const int firstLine = std::max(0, (int)(scrollY / lineHeight));
const int visibleLines = (int)(ImGui::GetContentRegionAvail().y / lineHeight) + 2;
const int lastLine = std::min(lineCount - 1, firstLine + visibleLines);
const float textAreaWidth = std::max(0.0f, windowWidth - gutterWidth - minimapWidth);
const int currentLine = lineFromPos(cursor_, lineStarts);
std::vector<bool> hiddenLines(lineCount, false);
for (const auto& f : folds_) {
if (!f.folded) continue;
for (int l = f.startLine + 1; l <= f.endLine && l < lineCount; ++l) {
hiddenLines[l] = true;
}
}
for (int ln = firstLine; ln <= lastLine; ++ln) {
if (ln >= 0 && ln < lineCount && hiddenLines[ln]) continue;
int start = lineStarts[ln];
int end = (ln + 1 < lineCount) ? lineStarts[ln + 1] - 1 : (int)text.size();
int len = std::max(0, end - start);
float y = textBase.y + ln * lineHeight;
// Gutter background
ImVec2 gutterA(origin.x, y);
ImVec2 gutterB(origin.x + gutterWidth, y + lineHeight);
drawList->AddRectFilled(
gutterA, gutterB,
ThemeEngine::instance().editorColor("gutter_bg",
IM_COL32(20, 20, 20, 255)));
// Line number (right-aligned)
if (options.showLineNumbers) {
char numBuf[16];
snprintf(numBuf, sizeof(numBuf), "%d", ln + 1);
ImVec2 numSize = font->CalcTextSizeA(font->FontSize, FLT_MAX, -1.0f, numBuf);
ImVec2 numPos(origin.x + gutterWidth - 4.0f - numSize.x, y);
drawList->AddText(font, font->FontSize, numPos,
ThemeEngine::instance().editorColor("gutter_text",
IM_COL32(120, 120, 120, 255)),
numBuf);
}
// Diagnostics marker
bool hasError = lineIn(options.errorLines, ln);
bool hasWarn = lineIn(options.warningLines, ln);
if (hasError || hasWarn) {
ImU32 color = hasError
? ThemeEngine::instance().editorColor("diag_error", IM_COL32(220, 80, 80, 255))
: ThemeEngine::instance().editorColor("diag_warning", IM_COL32(220, 160, 60, 255));
ImVec2 center(origin.x + 3.0f, y + lineHeight * 0.5f);
drawList->AddCircleFilled(center, 3.0f, color);
}
if (options.diagnostics) {
std::string msg = diagnosticMessageAtLine(*options.diagnostics, ln);
ImVec2 gutterA(origin.x, y);
ImVec2 gutterB(origin.x + gutterWidth, y + lineHeight);
if (!msg.empty() && ImGui::IsMouseHoveringRect(gutterA, gutterB)) {
ImGui::BeginTooltip();
ImGui::TextUnformatted(msg.c_str());
ImGui::EndTooltip();
}
}
// Annotation marker
if (options.annotations) {
AnnotationMarker marker;
if (annotationAtLine(*options.annotations, ln, marker)) {
ImVec2 center(origin.x + 12.0f, y + lineHeight * 0.5f);
drawList->AddCircleFilled(center, 3.0f, marker.color);
ImVec2 a(center.x - 4.0f, center.y - 4.0f);
ImVec2 b(center.x + 4.0f, center.y + 4.0f);
if (ImGui::IsMouseHoveringRect(a, b)) {
if (ImGui::IsMouseClicked(ImGuiMouseButton_Left)) {
annotationPopupMessage_ = marker.message;
ImGui::OpenPopup(annoPopupId.c_str());
}
ImGui::BeginTooltip();
ImGui::TextUnformatted(marker.message.c_str());
ImGui::EndTooltip();
}
}
}
// Suggestion marker (lightbulb)
if (options.suggestions) {
SuggestionMarker suggestion;
if (suggestionAtLine(*options.suggestions, ln, suggestion)) {
ImVec2 center(origin.x + 20.0f, y + lineHeight * 0.5f);
ImU32 color = ThemeEngine::instance().editorColor("suggestion",
IM_COL32(240, 200, 40, 255));
drawList->AddCircleFilled(center, 3.0f, color);
ImVec2 a(center.x - 4.0f, center.y - 4.0f);
ImVec2 b(center.x + 4.0f, center.y + 4.0f);
if (ImGui::IsMouseHoveringRect(a, b)) {
if (ImGui::IsMouseClicked(ImGuiMouseButton_Left)) {
result.suggestionClicked = true;
result.clickedSuggestion = suggestion;
}
ImGui::BeginTooltip();
ImGui::Text("%s (%.2f)", suggestion.label.c_str(), suggestion.confidence);
ImGui::Separator();
ImGui::TextUnformatted(suggestion.reason.c_str());
ImGui::EndTooltip();
}
}
}
// Conflict markers: highlight and connecting line
if (options.conflicts) {
AnnotationConflictMarker conflict;
if (conflictAtLine(*options.conflicts, ln, conflict)) {
ImVec2 center(origin.x + 12.0f, y + lineHeight * 0.5f);
drawList->AddCircle(center, 5.0f,
ThemeEngine::instance().editorColor("conflict",
IM_COL32(220, 80, 80, 255)),
12, 1.5f);
if (ln == std::min(conflict.childLine, conflict.parentLine) &&
conflict.childLine >= 0 && conflict.parentLine >= 0) {
float y1 = textBase.y + conflict.childLine * lineHeight + lineHeight * 0.5f;
float y2 = textBase.y + conflict.parentLine * lineHeight + lineHeight * 0.5f;
drawList->AddLine(ImVec2(origin.x + 12.0f, y1),
ImVec2(origin.x + 12.0f, y2),
ThemeEngine::instance().editorColor("conflict_line",
IM_COL32(220, 80, 80, 180)),
1.0f);
}
ImVec2 a(center.x - 5.0f, center.y - 5.0f);
ImVec2 b(center.x + 5.0f, center.y + 5.0f);
if (ImGui::IsMouseHoveringRect(a, b)) {
ImGui::BeginTooltip();
ImGui::TextUnformatted(conflict.message.c_str());
ImGui::EndTooltip();
}
}
}
// Fold indicator
const FoldRegion* fold = findFoldAtLine(ln);
if (fold) {
ImVec2 triCenter(origin.x + 6.0f, y + lineHeight * 0.5f);
if (fold->folded) {
drawTriangle(drawList, triCenter,
ThemeEngine::instance().editorColor("fold_indicator",
IM_COL32(160, 160, 160, 255)),
true);
} else {
drawTriangle(drawList, triCenter,
ThemeEngine::instance().editorColor("fold_indicator",
IM_COL32(160, 160, 160, 255)),
false);
}
}
// Current line highlight (text area only)
if (options.showCurrentLine && ln == currentLine) {
ImVec2 hlA(textBase.x, y);
ImVec2 hlB(textBase.x + textAreaWidth, y + lineHeight);
drawList->AddRectFilled(hlA, hlB,
ThemeEngine::instance().editorColor("line_highlight",
IM_COL32(40, 40, 40, 120)));
}
if (options.highlightLine >= 0 && ln == options.highlightLine) {
ImVec2 hlA(textBase.x, y);
ImVec2 hlB(textBase.x + textAreaWidth, y + lineHeight);
drawList->AddRectFilled(hlA, hlB, options.highlightLineColor);
}
if (options.highlightLines) {
if (std::find(options.highlightLines->begin(),
options.highlightLines->end(), ln) != options.highlightLines->end()) {
ImVec2 hlA(textBase.x, y);
ImVec2 hlB(textBase.x + textAreaWidth, y + lineHeight);
drawList->AddRectFilled(hlA, hlB, options.highlightLineColor);
}
}
// Selection background
if (hasSelection()) {
int selA = std::min(selStart_, selEnd_);
int selB = std::max(selStart_, selEnd_);
int lineSelStart = std::max(selA, start);
int lineSelEnd = std::min(selB, end);
if (lineSelStart < lineSelEnd) {
int colA = lineSelStart - start;
int colB = lineSelEnd - start;
ImVec2 a(textBase.x + colA * charAdvance, y);
ImVec2 b(textBase.x + colB * charAdvance, y + lineHeight);
drawList->AddRectFilled(a, b,
ThemeEngine::instance().editorColor("selection",
IM_COL32(60, 100, 160, 120)));
}
}
// Render text with colors
int pos = start;
while (pos < end) {
TokenCategory cat = TokenCategory::Plain;
if (pos < (int)charCats.size()) cat = charCats[pos];
int spanEnd = pos + 1;
while (spanEnd < end) {
TokenCategory nextCat = TokenCategory::Plain;
if (spanEnd < (int)charCats.size()) nextCat = charCats[spanEnd];
if (nextCat != cat) break;
++spanEnd;
}
std::string chunk = text.substr(pos, spanEnd - pos);
if (options.showWhitespace) {
for (auto& c : chunk) {
if (c == ' ') c = '.';
else if (c == '\t') c = '>';
}
}
if (!chunk.empty() && chunk[chunk.size() - 1] == '\n') {
chunk.pop_back();
}
ImVec2 p(textBase.x + (pos - start) * charAdvance, y);
drawList->AddText(font, font->FontSize, p, colorFor(cat), chunk.c_str());
pos = spanEnd;
}
// Inline annotation tag
if (options.showAnnotations && options.annotations) {
AnnotationMarker marker;
if (annotationAtLine(*options.annotations, ln, marker)) {
ImVec2 tagPos;
if (options.annotationLayout == 1) {
tagPos = ImVec2(origin.x + 18.0f, y);
} else if (options.annotationLayout == 2) {
tagPos = ImVec2(textBase.x + len * charAdvance + 8.0f, y);
} else {
tagPos = ImVec2(textBase.x, y - lineHeight * 0.7f);
}
ImVec2 textSize = font->CalcTextSizeA(font->FontSize, FLT_MAX, -1.0f, marker.message.c_str());
ImVec2 bgA(tagPos.x - 2.0f, tagPos.y);
ImVec2 bgB(tagPos.x + textSize.x + 6.0f, tagPos.y + lineHeight * 0.8f);
drawList->AddRectFilled(bgA, bgB,
ThemeEngine::instance().editorColor("annotation_tag_bg",
IM_COL32(30, 30, 30, 220)),
2.0f);
drawList->AddText(font, font->FontSize, ImVec2(tagPos.x + 2.0f, tagPos.y),
marker.color, marker.message.c_str());
}
}
// Folded placeholder
if (fold && fold->folded) {
ImVec2 p(textBase.x + len * charAdvance + 6.0f, y);
drawList->AddText(font, font->FontSize, p,
ThemeEngine::instance().editorColor("fold_placeholder",
IM_COL32(140, 140, 140, 255)),
"{...}");
}
if (options.diagnostics) {
renderSquiggles(*options.diagnostics, ln, y, lineHeight, charAdvance,
textBase.x, len, drawList);
if (hovered) {
ImVec2 mouse = ImGui::GetMousePos();
std::string msg = diagnosticMessageAtPoint(*options.diagnostics, ln, mouse,
y, lineHeight, charAdvance, textBase.x);
if (!msg.empty()) {
ImGui::BeginTooltip();
ImGui::TextUnformatted(msg.c_str());
ImGui::EndTooltip();
}
}
}
}
if (ImGui::BeginPopup(annoPopupId.c_str())) {
ImGui::TextUnformatted("Annotation Editor (TODO)");
ImGui::Separator();
ImGui::TextUnformatted(annotationPopupMessage_.c_str());
ImGui::EndPopup();
}
// Cursor
if (focused) {
const double t = ImGui::GetTime();
const bool blinkOn = ((int)(t * 2.0)) % 2 == 0;
if (blinkOn) {
int curLine = currentLine;
int lineStart = lineStarts[curLine];
int col = cursor_ - lineStart;
float x = textBase.x + col * charAdvance;
float y = textBase.y + curLine * lineHeight;
drawList->AddLine(ImVec2(x, y), ImVec2(x, y + lineHeight),
ThemeEngine::instance().editorColor("caret",
IM_COL32(240, 240, 240, 255)),
1.0f);
}
}
// Minimap
if (options.showMinimap && minimapWidth > 0.0f) {
float miniHeight = lineCount * lineHeight;
ImVec2 miniA(minimapBaseX, minimapBaseY);
ImVec2 miniB(minimapBaseX + minimapWidth, minimapBaseY + miniHeight);
drawList->AddRectFilled(miniA, miniB,
ThemeEngine::instance().editorColor("minimap_bg",
IM_COL32(18, 18, 18, 255)));
for (int ln = 0; ln < lineCount; ++ln) {
float y = minimapBaseY + ln * lineHeight;
ImU32 color = ThemeEngine::instance().editorColor("minimap_line",
IM_COL32(80, 80, 80, 255));
if (ln == currentLine) {
color = ThemeEngine::instance().editorColor("minimap_line_active",
IM_COL32(120, 120, 160, 255));
}
drawList->AddRectFilled(ImVec2(minimapBaseX + 2.0f, y),
ImVec2(minimapBaseX + minimapWidth - 2.0f, y + 2.0f),
color);
}
float viewStart = (lineCount > 0) ? (scrollY / (lineCount * lineHeight)) : 0.0f;
float viewEnd = viewStart + (ImGui::GetContentRegionAvail().y / (lineCount * lineHeight));
viewStart = std::max(0.0f, std::min(1.0f, viewStart));
viewEnd = std::max(0.0f, std::min(1.0f, viewEnd));
ImVec2 vA(minimapBaseX, minimapBaseY + viewStart * miniHeight);
ImVec2 vB(minimapBaseX + minimapWidth, minimapBaseY + viewEnd * miniHeight);
drawList->AddRect(vA, vB,
ThemeEngine::instance().editorColor("minimap_view",
IM_COL32(120, 160, 220, 180)));
}
ImGui::EndChild();
result.cursorByte = cursor_;
result.lineCount = lineCount;
result.gutterWidth = gutterWidth;
result.currentLine = currentLine;
result.foldCount = (int)folds_.size();
result.anyFolded = std::any_of(folds_.begin(), folds_.end(),
[](const FoldRegion& f) { return f.folded; });
result.minimapEnabled = options.showMinimap;
result.minimapWidth = minimapWidth;
if (options.showMinimap && lineCount > 0) {
float miniHeight = lineCount * lineHeight;
result.minimapViewportStart = (scrollY / (lineCount * lineHeight)) * miniHeight;
result.minimapViewportEnd = result.minimapViewportStart +
(ImGui::GetContentRegionAvail().y / (lineCount * lineHeight)) * miniHeight;
}
return result;
}
void setCursor(int pos) { cursor_ = pos; }
int getCursor() const { return cursor_; }
const std::vector<FoldRegion>& getFoldRegions() const { return folds_; }
std::vector<int> getFoldedLines() const {
std::vector<int> lines;
for (const auto& f : folds_) {
if (f.folded) lines.push_back(f.startLine);
}
return lines;
}
void setDesiredFoldedLines(const std::vector<int>& lines) {
desiredFoldedLines_ = lines;
applyDesiredFoldState_ = true;
}
void toggleFoldAtLine(int line) {
for (auto& f : folds_) {
if (f.startLine == line) {
f.folded = !f.folded;
return;
}
}
}
private:
int cursor_ = 0;
int selStart_ = -1;
int selEnd_ = -1;
bool selecting_ = false;
std::vector<FoldRegion> folds_;
std::string lastFoldText_;
std::string lastFoldLang_;
std::string annotationPopupMessage_;
std::vector<int> desiredFoldedLines_;
bool applyDesiredFoldState_ = false;
#include "CodeEditorRenderHelpers.h"

View File

@@ -0,0 +1,210 @@
#pragma once
// Included inside CodeEditorWidget (selection + editing).
bool hasSelection() const {
return selStart_ >= 0 && selEnd_ >= 0 && selStart_ != selEnd_;
}
void deleteSelection(std::string& text, bool& changed) {
if (!hasSelection()) return;
int a = std::min(selStart_, selEnd_);
int b = std::max(selStart_, selEnd_);
if (a >= 0 && b <= (int)text.size() && a < b) {
text.erase(a, b - a);
cursor_ = a;
changed = true;
}
selStart_ = selEnd_ = -1;
}
void insertText(std::string& text, const std::string& insert, bool& changed) {
if (insert.empty()) return;
deleteSelection(text, changed);
text.insert(cursor_, insert);
cursor_ += (int)insert.size();
changed = true;
}
void insertPair(std::string& text, char open, char close, bool& changed) {
deleteSelection(text, changed);
text.insert(cursor_, 1, open);
text.insert(cursor_ + 1, 1, close);
cursor_ += 1;
changed = true;
}
static std::string lineTextAt(const std::string& text, const std::vector<int>& lineStarts, int line) {
if (line < 0 || line >= (int)lineStarts.size()) return "";
int start = lineStarts[line];
int end = (line + 1 < (int)lineStarts.size()) ? lineStarts[line + 1] - 1 : (int)text.size();
if (end < start) end = start;
return text.substr(start, end - start);
}
static std::string leadingIndent(const std::string& line) {
size_t i = 0;
while (i < line.size() && (line[i] == ' ' || line[i] == '\t')) ++i;
return line.substr(0, i);
}
void insertNewlineWithIndent(std::string& text,
const std::vector<int>& lineStarts,
const EditorMode* mode,
bool& changed) {
std::string indent;
if (mode) {
int line = lineFromPos(cursor_, lineStarts);
std::string lineText = lineTextAt(text, lineStarts, line);
indent = leadingIndent(lineText);
if (mode->shouldIndentAfter(lineText)) {
indent += mode->getIndentString();
}
}
insertText(text, "\n" + indent, changed);
}
void handleKeyboard(std::string& text,
bool& changed,
const std::vector<int>& lineStarts,
const EditorMode* mode) {
ImGuiIO& io = ImGui::GetIO();
// Text input
for (int n = 0; n < io.InputQueueCharacters.Size; n++) {
ImWchar c = io.InputQueueCharacters[n];
if (c == 0) continue;
if (c == '\r') c = '\n';
if (c == '\n') {
insertNewlineWithIndent(text, lineStarts, mode, changed);
} else if (c == '\t') {
if (mode) insertText(text, mode->getIndentString(), changed);
else insertText(text, "\t", changed);
} else if (c >= 32) {
if (mode) {
char close = mode->getClosingBracket((char)c);
if (close != 0) {
insertPair(text, (char)c, close, changed);
} else {
char buf[5] = {0};
buf[0] = (char)c;
insertText(text, std::string(buf), changed);
}
} else {
char buf[5] = {0};
buf[0] = (char)c;
insertText(text, std::string(buf), changed);
}
}
}
io.InputQueueCharacters.resize(0);
// Navigation
auto moveCursor = [&](int newPos) {
newPos = std::max(0, std::min(newPos, (int)text.size()));
if (io.KeyShift) {
if (!hasSelection()) selStart_ = cursor_;
cursor_ = newPos;
selEnd_ = cursor_;
} else {
cursor_ = newPos;
selStart_ = selEnd_ = -1;
}
};
if (ImGui::IsKeyPressed(ImGuiKey_LeftArrow)) {
if (!io.KeyShift && hasSelection()) {
moveCursor(std::min(selStart_, selEnd_));
} else if (cursor_ > 0) {
moveCursor(cursor_ - 1);
}
}
if (ImGui::IsKeyPressed(ImGuiKey_RightArrow)) {
if (!io.KeyShift && hasSelection()) {
moveCursor(std::max(selStart_, selEnd_));
} else if (cursor_ < (int)text.size()) {
moveCursor(cursor_ + 1);
}
}
if (ImGui::IsKeyPressed(ImGuiKey_UpArrow)) {
int curLine = lineFromPos(cursor_, lineStarts);
if (curLine > 0) {
int curCol = cursor_ - lineStarts[curLine];
int prevStart = lineStarts[curLine - 1];
int prevEnd = (curLine < (int)lineStarts.size()) ? lineStarts[curLine] - 1 : (int)text.size();
int prevLen = std::max(0, prevEnd - prevStart);
moveCursor(prevStart + std::min(curCol, prevLen));
}
}
if (ImGui::IsKeyPressed(ImGuiKey_DownArrow)) {
int curLine = lineFromPos(cursor_, lineStarts);
if (curLine + 1 < (int)lineStarts.size()) {
int curCol = cursor_ - lineStarts[curLine];
int nextStart = lineStarts[curLine + 1];
int nextEnd = (curLine + 2 < (int)lineStarts.size()) ? lineStarts[curLine + 2] - 1 : (int)text.size();
int nextLen = std::max(0, nextEnd - nextStart);
moveCursor(nextStart + std::min(curCol, nextLen));
}
}
// Editing keys
if (ImGui::IsKeyPressed(ImGuiKey_Backspace) && !text.empty()) {
if (hasSelection()) {
deleteSelection(text, changed);
} else if (cursor_ > 0) {
text.erase(cursor_ - 1, 1);
cursor_--;
changed = true;
}
}
if (ImGui::IsKeyPressed(ImGuiKey_Delete) && !text.empty()) {
if (hasSelection()) {
deleteSelection(text, changed);
} else if (cursor_ < (int)text.size()) {
text.erase(cursor_, 1);
changed = true;
}
}
if ((io.KeyCtrl || io.KeySuper) && ImGui::IsKeyPressed(ImGuiKey_A)) {
selStart_ = 0;
selEnd_ = (int)text.size();
cursor_ = selEnd_;
}
if ((io.KeyCtrl || io.KeySuper) && ImGui::IsKeyPressed(ImGuiKey_C)) {
if (hasSelection()) {
ImGui::SetClipboardText(getSelectionText(text).c_str());
}
}
if ((io.KeyCtrl || io.KeySuper) && ImGui::IsKeyPressed(ImGuiKey_X)) {
if (hasSelection()) {
ImGui::SetClipboardText(getSelectionText(text).c_str());
deleteSelection(text, changed);
}
}
if ((io.KeyCtrl || io.KeySuper) && ImGui::IsKeyPressed(ImGuiKey_V)) {
const char* clip = ImGui::GetClipboardText();
if (clip && *clip) {
insertText(text, std::string(clip), changed);
}
}
}
std::string getSelectionText(const std::string& text) const {
if (!hasSelection()) return "";
int a = std::min(selStart_, selEnd_);
int b = std::max(selStart_, selEnd_);
if (a < 0 || b > (int)text.size() || a >= b) return "";
return text.substr(a, b - a);
}
void selectWordAt(const std::string& text, int pos) {
if (pos < 0 || pos > (int)text.size()) return;
auto isWord = [](char c) { return std::isalnum((unsigned char)c) || c == '_'; };
int start = pos;
int end = pos;
while (start > 0 && isWord(text[start - 1])) --start;
while (end < (int)text.size() && isWord(text[end])) ++end;
selStart_ = start;
selEnd_ = end;
cursor_ = end;
}

File diff suppressed because it is too large Load Diff

View File

@@ -6,7 +6,7 @@
// token category (keyword, string, comment, number, identifier, operator,
// type, punctuation, function, parameter).
//
// The highlight data is language-agnostic at the output level consumers
// The highlight data is language-agnostic at the output level — consumers
// map categories to colors via a theme.
#include <string>
@@ -149,623 +149,46 @@ private:
}
}
#include "SyntaxLanguages.h"
#include "SyntaxHighlighterPython.h"
#include "SyntaxHighlighterCpp.h"
#include "SyntaxHighlighterJavaScript.h"
#include "SyntaxHighlighterJava.h"
#include "SyntaxHighlighterRust.h"
#include "SyntaxHighlighterGo.h"
#include "SyntaxHighlighterElisp.h"
#include "SyntaxHighlighterOrg.h"
// --- Python --------------------------------------------------------
static bool isPythonKeyword(const std::string& text) {
static const char* keywords[] = {
"def", "class", "if", "elif", "else", "for", "while", "return",
"import", "from", "as", "try", "except", "finally", "raise",
"with", "yield", "lambda", "pass", "break", "continue",
"and", "or", "not", "in", "is", "del", "global", "nonlocal",
"assert", "async", "await", nullptr
};
for (const char** k = keywords; *k; ++k) {
if (text == *k) return true;
}
return false;
}
static bool isPythonBuiltin(const std::string& text) {
static const char* builtins[] = {
"True", "False", "None", "print", "len", "range", "int",
"str", "float", "list", "dict", "set", "tuple", "type",
"isinstance", "super", "self", nullptr
};
for (const char** b = builtins; *b; ++b) {
if (text == *b) return true;
}
return false;
}
static void walkPython(TSNode node, const std::string& source,
std::vector<HighlightSpan>& spans) {
if (ts_node_is_null(node)) return;
std::string type = nodeType(node);
std::string text = nodeText(node, source);
bool descend = true;
// Leaf / terminal categorization
if (type == "comment") {
addSpan(spans, node, TokenCategory::Comment);
descend = false;
} else if (type == "string" || type == "concatenated_string") {
addSpan(spans, node, TokenCategory::String);
descend = false;
} else if (type == "integer" || type == "float") {
addSpan(spans, node, TokenCategory::Number);
descend = false;
} else if (type == "identifier") {
// Context-dependent: function name, parameter, builtin, or plain
TSNode parent = ts_node_parent(node);
std::string parentType = ts_node_is_null(parent) ? "" : nodeType(parent);
if (parentType == "function_definition") {
TSNode nameField = ts_node_child_by_field_name(parent, "name", 4);
if (!ts_node_is_null(nameField) &&
ts_node_start_byte(nameField) == ts_node_start_byte(node)) {
addSpan(spans, node, TokenCategory::Function);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
} else if (parentType == "parameters" || parentType == "default_parameter") {
addSpan(spans, node, TokenCategory::Parameter);
} else if (parentType == "call") {
TSNode funcField = ts_node_child_by_field_name(parent, "function", 8);
if (!ts_node_is_null(funcField) &&
ts_node_start_byte(funcField) == ts_node_start_byte(node)) {
addSpan(spans, node, TokenCategory::Function);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
} else if (parentType == "type") {
addSpan(spans, node, TokenCategory::Type);
} else if (isPythonBuiltin(text)) {
addSpan(spans, node, TokenCategory::Builtin);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
descend = false;
} else if (type == "type") {
addSpan(spans, node, TokenCategory::Type);
descend = false;
} else if (!ts_node_is_named(node)) {
// Anonymous/unnamed nodes are operators, keywords, punctuation
if (isPythonKeyword(text)) {
addSpan(spans, node, TokenCategory::Keyword);
} else if (text == "(" || text == ")" || text == "[" || text == "]" ||
text == "{" || text == "}" || text == ":" || text == "," ||
text == "." || text == ";") {
addSpan(spans, node, TokenCategory::Punctuation);
} else if (text == "+" || text == "-" || text == "*" || text == "/" ||
text == "%" || text == "=" || text == "==" || text == "!=" ||
text == "<" || text == ">" || text == "<=" || text == ">=" ||
text == "+=" || text == "-=" || text == "*=" || text == "/=" ||
text == "**" || text == "//" || text == "->" || text == "@") {
addSpan(spans, node, TokenCategory::Operator);
}
descend = false;
}
if (descend) {
uint32_t count = ts_node_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
walkPython(ts_node_child(node, i), source, spans);
}
}
}
// --- C++ -----------------------------------------------------------
static bool isCppKeyword(const std::string& text) {
static const char* keywords[] = {
"auto", "break", "case", "catch", "class", "const", "constexpr",
"continue", "default", "delete", "do", "else", "enum", "explicit",
"extern", "for", "friend", "goto", "if", "inline", "mutable",
"namespace", "new", "noexcept", "operator", "private", "protected",
"public", "register", "return", "sizeof", "static", "static_assert",
"static_cast", "struct", "switch", "template", "this", "throw",
"try", "typedef", "typeid", "typename", "union", "using",
"virtual", "volatile", "while", "override", "final",
"co_await", "co_return", "co_yield", "concept", "requires",
"consteval", "constinit", nullptr
};
for (const char** k = keywords; *k; ++k) {
if (text == *k) return true;
}
return false;
}
static bool isCppType(const std::string& text) {
static const char* types[] = {
"void", "bool", "char", "int", "float", "double", "long",
"short", "unsigned", "signed", "size_t", "nullptr_t",
"int8_t", "int16_t", "int32_t", "int64_t",
"uint8_t", "uint16_t", "uint32_t", "uint64_t",
"string", "vector", "map", "set", "array", "tuple",
"unique_ptr", "shared_ptr", "weak_ptr", "optional",
nullptr
};
for (const char** t = types; *t; ++t) {
if (text == *t) return true;
}
return false;
}
// --- JavaScript / TypeScript --------------------------------------
static bool isJsKeyword(const std::string& text) {
static const char* keywords[] = {
"function", "return", "if", "else", "for", "while", "class",
"const", "let", "var", "import", "export", "new", "try", "catch",
"finally", "switch", "case", "break", "continue", "throw",
"async", "await", "yield", nullptr
};
for (const char** k = keywords; *k; ++k) {
if (text == *k) return true;
}
return false;
}
static void walkJavaScript(TSNode node, const std::string& source,
std::vector<HighlightSpan>& spans) {
if (ts_node_is_null(node)) return;
std::string type = nodeType(node);
std::string text = nodeText(node, source);
bool descend = true;
if (type.find("comment") != std::string::npos) {
addSpan(spans, node, TokenCategory::Comment);
descend = false;
} else if (type == "string" || type == "string_fragment" || type == "template_string" ||
type == "template_substitution") {
addSpan(spans, node, TokenCategory::String);
descend = false;
} else if (type == "number" || type == "number_literal") {
addSpan(spans, node, TokenCategory::Number);
descend = false;
} else if (type == "identifier" || type == "property_identifier") {
TSNode parent = ts_node_parent(node);
std::string parentType = ts_node_is_null(parent) ? "" : nodeType(parent);
if (parentType == "function_declaration" || parentType == "method_definition") {
addSpan(spans, node, TokenCategory::Function);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
descend = false;
} else if (type == "true" || type == "false" || type == "null" || type == "undefined") {
addSpan(spans, node, TokenCategory::Builtin);
descend = false;
} else if (!ts_node_is_named(node)) {
if (isJsKeyword(text)) {
addSpan(spans, node, TokenCategory::Keyword);
} else if (text == "(" || text == ")" || text == "[" || text == "]" ||
text == "{" || text == "}" || text == ";" || text == "," ||
text == "." || text == ":" ) {
addSpan(spans, node, TokenCategory::Punctuation);
} else {
addSpan(spans, node, TokenCategory::Operator);
}
descend = false;
}
if (descend) {
uint32_t count = ts_node_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
walkJavaScript(ts_node_child(node, i), source, spans);
}
}
}
static void walkTypeScript(TSNode node, const std::string& source,
std::vector<HighlightSpan>& spans) {
walkJavaScript(node, source, spans);
}
// --- Java -----------------------------------------------------------
static bool isJavaKeyword(const std::string& text) {
static const char* keywords[] = {
"class", "interface", "enum", "public", "private", "protected",
"static", "final", "void", "int", "float", "double", "boolean",
"return", "if", "else", "for", "while", "switch", "case",
"break", "continue", "new", "try", "catch", "finally", "throw",
"extends", "implements", "import", "package", "this", "super",
nullptr
};
for (const char** k = keywords; *k; ++k) {
if (text == *k) return true;
}
return false;
}
static void walkJava(TSNode node, const std::string& source,
std::vector<HighlightSpan>& spans) {
if (ts_node_is_null(node)) return;
std::string type = nodeType(node);
std::string text = nodeText(node, source);
bool descend = true;
if (type.find("comment") != std::string::npos) {
addSpan(spans, node, TokenCategory::Comment);
descend = false;
} else if (type.find("string") != std::string::npos || type == "character_literal") {
addSpan(spans, node, TokenCategory::String);
descend = false;
} else if (type.find("integer") != std::string::npos || type.find("floating") != std::string::npos ||
type == "decimal_integer_literal" || type == "decimal_floating_point_literal") {
addSpan(spans, node, TokenCategory::Number);
descend = false;
} else if (type == "identifier") {
TSNode parent = ts_node_parent(node);
std::string parentType = ts_node_is_null(parent) ? "" : nodeType(parent);
if (parentType == "method_declaration" || parentType == "constructor_declaration") {
addSpan(spans, node, TokenCategory::Function);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
descend = false;
} else if (type == "type_identifier" || type == "integral_type" || type == "floating_point_type") {
addSpan(spans, node, TokenCategory::Type);
descend = false;
} else if (type == "true" || type == "false" || type == "null_literal") {
addSpan(spans, node, TokenCategory::Builtin);
descend = false;
} else if (!ts_node_is_named(node)) {
if (isJavaKeyword(text)) {
addSpan(spans, node, TokenCategory::Keyword);
} else if (text == "(" || text == ")" || text == "[" || text == "]" ||
text == "{" || text == "}" || text == ";" || text == "," ||
text == "." || text == ":" ) {
addSpan(spans, node, TokenCategory::Punctuation);
} else {
addSpan(spans, node, TokenCategory::Operator);
}
descend = false;
}
if (descend) {
uint32_t count = ts_node_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
walkJava(ts_node_child(node, i), source, spans);
}
}
}
// --- Rust -----------------------------------------------------------
static bool isRustKeyword(const std::string& text) {
static const char* keywords[] = {
"fn", "let", "mut", "pub", "impl", "trait", "struct", "enum",
"use", "mod", "crate", "self", "super", "return", "if", "else",
"for", "while", "loop", "match", "break", "continue", "const",
"static", "async", "await", "move", nullptr
};
for (const char** k = keywords; *k; ++k) {
if (text == *k) return true;
}
return false;
}
static void walkRust(TSNode node, const std::string& source,
std::vector<HighlightSpan>& spans) {
if (ts_node_is_null(node)) return;
std::string type = nodeType(node);
std::string text = nodeText(node, source);
bool descend = true;
if (type.find("comment") != std::string::npos) {
addSpan(spans, node, TokenCategory::Comment);
descend = false;
} else if (type == "string_literal" || type == "char_literal" ||
type == "raw_string_literal") {
addSpan(spans, node, TokenCategory::String);
descend = false;
} else if (type.find("integer") != std::string::npos || type.find("float") != std::string::npos ||
type == "number") {
addSpan(spans, node, TokenCategory::Number);
descend = false;
} else if (type == "identifier") {
TSNode parent = ts_node_parent(node);
std::string parentType = ts_node_is_null(parent) ? "" : nodeType(parent);
if (parentType == "function_item") {
addSpan(spans, node, TokenCategory::Function);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
descend = false;
} else if (type == "type_identifier" || type == "primitive_type") {
addSpan(spans, node, TokenCategory::Type);
descend = false;
} else if (!ts_node_is_named(node)) {
if (isRustKeyword(text)) {
addSpan(spans, node, TokenCategory::Keyword);
} else if (text == "(" || text == ")" || text == "[" || text == "]" ||
text == "{" || text == "}" || text == ";" || text == "," ||
text == ":" ) {
addSpan(spans, node, TokenCategory::Punctuation);
} else {
addSpan(spans, node, TokenCategory::Operator);
}
descend = false;
}
if (descend) {
uint32_t count = ts_node_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
walkRust(ts_node_child(node, i), source, spans);
}
}
}
// --- Go -------------------------------------------------------------
static bool isGoKeyword(const std::string& text) {
static const char* keywords[] = {
"func", "package", "import", "return", "if", "else", "for",
"switch", "case", "break", "continue", "struct", "interface",
"type", "var", "const", "go", "defer", "range", nullptr
};
for (const char** k = keywords; *k; ++k) {
if (text == *k) return true;
}
return false;
}
static void walkGo(TSNode node, const std::string& source,
std::vector<HighlightSpan>& spans) {
if (ts_node_is_null(node)) return;
std::string type = nodeType(node);
std::string text = nodeText(node, source);
bool descend = true;
if (type.find("comment") != std::string::npos) {
addSpan(spans, node, TokenCategory::Comment);
descend = false;
} else if (type == "interpreted_string_literal" || type == "raw_string_literal" ||
type == "rune_literal") {
addSpan(spans, node, TokenCategory::String);
descend = false;
} else if (type.find("int") != std::string::npos || type.find("float") != std::string::npos ||
type == "number") {
addSpan(spans, node, TokenCategory::Number);
descend = false;
} else if (type == "identifier") {
TSNode parent = ts_node_parent(node);
std::string parentType = ts_node_is_null(parent) ? "" : nodeType(parent);
if (parentType == "function_declaration" || parentType == "method_declaration") {
addSpan(spans, node, TokenCategory::Function);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
descend = false;
} else if (type == "type_identifier" || type == "primitive_type") {
addSpan(spans, node, TokenCategory::Type);
descend = false;
} else if (!ts_node_is_named(node)) {
if (isGoKeyword(text)) {
addSpan(spans, node, TokenCategory::Keyword);
} else if (text == "(" || text == ")" || text == "[" || text == "]" ||
text == "{" || text == "}" || text == ";" || text == "," ||
text == ":" ) {
addSpan(spans, node, TokenCategory::Punctuation);
} else {
addSpan(spans, node, TokenCategory::Operator);
}
descend = false;
}
if (descend) {
uint32_t count = ts_node_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
walkGo(ts_node_child(node, i), source, spans);
}
}
}
static void walkCpp(TSNode node, const std::string& source,
std::vector<HighlightSpan>& spans) {
if (ts_node_is_null(node)) return;
std::string type = nodeType(node);
std::string text = nodeText(node, source);
bool descend = true;
if (type == "comment") {
addSpan(spans, node, TokenCategory::Comment);
descend = false;
} else if (type == "string_literal" || type == "raw_string_literal" ||
type == "char_literal" || type == "string_content") {
addSpan(spans, node, TokenCategory::String);
descend = false;
} else if (type == "number_literal") {
addSpan(spans, node, TokenCategory::Number);
descend = false;
} else if (type == "primitive_type" || type == "sized_type_specifier" ||
type == "type_identifier") {
addSpan(spans, node, TokenCategory::Type);
descend = false;
} else if (type == "identifier") {
TSNode parent = ts_node_parent(node);
std::string parentType = ts_node_is_null(parent) ? "" : nodeType(parent);
if (parentType == "function_declarator") {
TSNode declField = ts_node_child_by_field_name(parent, "declarator", 10);
if (!ts_node_is_null(declField) &&
ts_node_start_byte(declField) == ts_node_start_byte(node)) {
addSpan(spans, node, TokenCategory::Function);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
} else if (parentType == "call_expression") {
TSNode funcField = ts_node_child_by_field_name(parent, "function", 8);
if (!ts_node_is_null(funcField) &&
ts_node_start_byte(funcField) == ts_node_start_byte(node)) {
addSpan(spans, node, TokenCategory::Function);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
} else if (parentType == "parameter_declaration") {
TSNode declField = ts_node_child_by_field_name(parent, "declarator", 10);
if (!ts_node_is_null(declField) &&
ts_node_start_byte(declField) == ts_node_start_byte(node)) {
addSpan(spans, node, TokenCategory::Parameter);
} else if (isCppType(text)) {
addSpan(spans, node, TokenCategory::Type);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
} else if (isCppType(text)) {
addSpan(spans, node, TokenCategory::Type);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
descend = false;
} else if (type == "true" || type == "false" || type == "nullptr") {
addSpan(spans, node, TokenCategory::Builtin);
descend = false;
} else if (type == "preproc_include" || type == "preproc_def" ||
type == "preproc_ifdef" || type == "preproc_else" ||
type == "preproc_endif" || type == "preproc_call") {
addSpan(spans, node, TokenCategory::Keyword);
descend = false;
} else if (type == "system_lib_string") {
addSpan(spans, node, TokenCategory::String);
descend = false;
} else if (!ts_node_is_named(node)) {
if (isCppKeyword(text)) {
addSpan(spans, node, TokenCategory::Keyword);
} else if (text == "(" || text == ")" || text == "[" || text == "]" ||
text == "{" || text == "}" || text == ";" || text == "," ||
text == "." || text == "::" || text == ":" || text == "->") {
addSpan(spans, node, TokenCategory::Punctuation);
} else if (text == "+" || text == "-" || text == "*" || text == "/" ||
text == "%" || text == "=" || text == "==" || text == "!=" ||
text == "<" || text == ">" || text == "<=" || text == ">=" ||
text == "+=" || text == "-=" || text == "*=" || text == "/=" ||
text == "&&" || text == "||" || text == "!" || text == "&" ||
text == "|" || text == "^" || text == "~" || text == "<<" ||
text == ">>" || text == "++" || text == "--") {
addSpan(spans, node, TokenCategory::Operator);
} else if (isCppType(text)) {
addSpan(spans, node, TokenCategory::Type);
}
descend = false;
}
if (descend) {
uint32_t count = ts_node_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
walkCpp(ts_node_child(node, i), source, spans);
}
}
}
// --- Elisp ---------------------------------------------------------
static bool isElispKeyword(const std::string& text) {
static const char* keywords[] = {
"defun", "defvar", "defconst", "defmacro", "defcustom",
"let", "let*", "if", "when", "unless", "cond", "while",
"dolist", "dotimes", "progn", "prog1", "prog2",
"lambda", "setq", "setf", "require", "provide",
"interactive", "save-excursion", "save-restriction",
"condition-case", "unwind-protect", "catch", "throw",
nullptr
};
for (const char** k = keywords; *k; ++k) {
if (text == *k) return true;
}
return false;
}
static void walkElisp(TSNode node, const std::string& source,
std::vector<HighlightSpan>& spans) {
if (ts_node_is_null(node)) return;
std::string type = nodeType(node);
std::string text = nodeText(node, source);
bool descend = true;
if (type == "comment") {
addSpan(spans, node, TokenCategory::Comment);
descend = false;
} else if (type == "string") {
addSpan(spans, node, TokenCategory::String);
descend = false;
} else if (type == "integer" || type == "float") {
addSpan(spans, node, TokenCategory::Number);
descend = false;
} else if (type == "symbol") {
if (isElispKeyword(text)) {
addSpan(spans, node, TokenCategory::Keyword);
} else if (text.size() > 0 && text[0] == ':') {
// keyword symbol like :test
addSpan(spans, node, TokenCategory::Builtin);
} else if (text == "t" || text == "nil") {
addSpan(spans, node, TokenCategory::Builtin);
} else {
TSNode parent = ts_node_parent(node);
std::string parentType = ts_node_is_null(parent) ? "" : nodeType(parent);
if (parentType == "function_definition" || parentType == "special_form") {
// Check if this is the name position (second element)
TSNode firstSibling = ts_node_named_child(parent, 0);
TSNode secondSibling = ts_node_named_child(parent, 1);
if (!ts_node_is_null(firstSibling) &&
isElispKeyword(nodeText(firstSibling, source)) &&
!ts_node_is_null(secondSibling) &&
ts_node_start_byte(secondSibling) == ts_node_start_byte(node)) {
addSpan(spans, node, TokenCategory::Function);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
}
descend = false;
} else if (!ts_node_is_named(node)) {
if (text == "(" || text == ")" || text == "[" || text == "]") {
addSpan(spans, node, TokenCategory::Punctuation);
} else if (text == "'" || text == "`" || text == ",") {
addSpan(spans, node, TokenCategory::Operator);
}
descend = false;
}
if (descend) {
uint32_t count = ts_node_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
walkElisp(ts_node_child(node, i), source, spans);
}
}
}
// --- Org -----------------------------------------------------------
static void walkOrgSimple(const std::string& source,
std::vector<HighlightSpan>& spans) {
size_t start = 0;
while (start < source.size()) {
size_t end = source.find('\n', start);
if (end == std::string::npos) end = source.size();
std::string line = source.substr(start, end - start);
std::string trimmed = line;
while (!trimmed.empty() && (trimmed.back() == '\r' || trimmed.back() == '\n')) {
trimmed.pop_back();
}
if (!trimmed.empty() && trimmed[0] == '*') {
spans.push_back({(uint32_t)start, (uint32_t)end, TokenCategory::Keyword});
} else if (trimmed.rfind("#+begin_src", 0) == 0 ||
trimmed.rfind("#+end_src", 0) == 0 ||
trimmed.rfind("#+", 0) == 0) {
spans.push_back({(uint32_t)start, (uint32_t)end, TokenCategory::Comment});
}
start = end + 1;
}
}
};

View File

@@ -0,0 +1,100 @@
#pragma once
// SyntaxHighlighterCpp.h helpers for SyntaxHighlighter.
static void walkCpp(TSNode node, const std::string& source,
std::vector<HighlightSpan>& spans) {
if (ts_node_is_null(node)) return;
std::string type = nodeType(node);
std::string text = nodeText(node, source);
bool descend = true;
if (type == "comment") {
addSpan(spans, node, TokenCategory::Comment);
descend = false;
} else if (type == "string_literal" || type == "raw_string_literal" ||
type == "char_literal" || type == "string_content") {
addSpan(spans, node, TokenCategory::String);
descend = false;
} else if (type == "number_literal") {
addSpan(spans, node, TokenCategory::Number);
descend = false;
} else if (type == "primitive_type" || type == "sized_type_specifier" ||
type == "type_identifier") {
addSpan(spans, node, TokenCategory::Type);
descend = false;
} else if (type == "identifier") {
TSNode parent = ts_node_parent(node);
std::string parentType = ts_node_is_null(parent) ? "" : nodeType(parent);
if (parentType == "function_declarator") {
TSNode declField = ts_node_child_by_field_name(parent, "declarator", 10);
if (!ts_node_is_null(declField) &&
ts_node_start_byte(declField) == ts_node_start_byte(node)) {
addSpan(spans, node, TokenCategory::Function);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
} else if (parentType == "call_expression") {
TSNode funcField = ts_node_child_by_field_name(parent, "function", 8);
if (!ts_node_is_null(funcField) &&
ts_node_start_byte(funcField) == ts_node_start_byte(node)) {
addSpan(spans, node, TokenCategory::Function);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
} else if (parentType == "parameter_declaration") {
TSNode declField = ts_node_child_by_field_name(parent, "declarator", 10);
if (!ts_node_is_null(declField) &&
ts_node_start_byte(declField) == ts_node_start_byte(node)) {
addSpan(spans, node, TokenCategory::Parameter);
} else if (isCppType(text)) {
addSpan(spans, node, TokenCategory::Type);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
} else if (isCppType(text)) {
addSpan(spans, node, TokenCategory::Type);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
descend = false;
} else if (type == "true" || type == "false" || type == "nullptr") {
addSpan(spans, node, TokenCategory::Builtin);
descend = false;
} else if (type == "preproc_include" || type == "preproc_def" ||
type == "preproc_ifdef" || type == "preproc_else" ||
type == "preproc_endif" || type == "preproc_call") {
addSpan(spans, node, TokenCategory::Keyword);
descend = false;
} else if (type == "system_lib_string") {
addSpan(spans, node, TokenCategory::String);
descend = false;
} else if (!ts_node_is_named(node)) {
if (isCppKeyword(text)) {
addSpan(spans, node, TokenCategory::Keyword);
} else if (text == "(" || text == ")" || text == "[" || text == "]" ||
text == "{" || text == "}" || text == ";" || text == "," ||
text == "." || text == "::" || text == ":" || text == "->") {
addSpan(spans, node, TokenCategory::Punctuation);
} else if (text == "+" || text == "-" || text == "*" || text == "/" ||
text == "%" || text == "=" || text == "==" || text == "!=" ||
text == "<" || text == ">" || text == "<=" || text == ">=" ||
text == "+=" || text == "-=" || text == "*=" || text == "/=" ||
text == "&&" || text == "||" || text == "!" || text == "&" ||
text == "|" || text == "^" || text == "~" || text == "<<" ||
text == ">>" || text == "++" || text == "--") {
addSpan(spans, node, TokenCategory::Operator);
} else if (isCppType(text)) {
addSpan(spans, node, TokenCategory::Type);
}
descend = false;
}
if (descend) {
uint32_t count = ts_node_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
walkCpp(ts_node_child(node, i), source, spans);
}
}
}

View File

@@ -0,0 +1,64 @@
#pragma once
// SyntaxHighlighterElisp.h helpers for SyntaxHighlighter.
static void walkElisp(TSNode node, const std::string& source,
std::vector<HighlightSpan>& spans) {
if (ts_node_is_null(node)) return;
std::string type = nodeType(node);
std::string text = nodeText(node, source);
bool descend = true;
if (type == "comment") {
addSpan(spans, node, TokenCategory::Comment);
descend = false;
} else if (type == "string") {
addSpan(spans, node, TokenCategory::String);
descend = false;
} else if (type == "integer" || type == "float") {
addSpan(spans, node, TokenCategory::Number);
descend = false;
} else if (type == "symbol") {
if (isElispKeyword(text)) {
addSpan(spans, node, TokenCategory::Keyword);
} else if (text.size() > 0 && text[0] == ':') {
// keyword symbol like :test
addSpan(spans, node, TokenCategory::Builtin);
} else if (text == "t" || text == "nil") {
addSpan(spans, node, TokenCategory::Builtin);
} else {
TSNode parent = ts_node_parent(node);
std::string parentType = ts_node_is_null(parent) ? "" : nodeType(parent);
if (parentType == "function_definition" || parentType == "special_form") {
// Check if this is the name position (second element)
TSNode firstSibling = ts_node_named_child(parent, 0);
TSNode secondSibling = ts_node_named_child(parent, 1);
if (!ts_node_is_null(firstSibling) &&
isElispKeyword(nodeText(firstSibling, source)) &&
!ts_node_is_null(secondSibling) &&
ts_node_start_byte(secondSibling) == ts_node_start_byte(node)) {
addSpan(spans, node, TokenCategory::Function);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
}
descend = false;
} else if (!ts_node_is_named(node)) {
if (text == "(" || text == ")" || text == "[" || text == "]") {
addSpan(spans, node, TokenCategory::Punctuation);
} else if (text == "'" || text == "`" || text == ",") {
addSpan(spans, node, TokenCategory::Operator);
}
descend = false;
}
if (descend) {
uint32_t count = ts_node_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
walkElisp(ts_node_child(node, i), source, spans);
}
}
}

View File

@@ -0,0 +1,54 @@
#pragma once
// SyntaxHighlighterGo.h helpers for SyntaxHighlighter.
static void walkGo(TSNode node, const std::string& source,
std::vector<HighlightSpan>& spans) {
if (ts_node_is_null(node)) return;
std::string type = nodeType(node);
std::string text = nodeText(node, source);
bool descend = true;
if (type.find("comment") != std::string::npos) {
addSpan(spans, node, TokenCategory::Comment);
descend = false;
} else if (type == "interpreted_string_literal" || type == "raw_string_literal" ||
type == "rune_literal") {
addSpan(spans, node, TokenCategory::String);
descend = false;
} else if (type.find("int") != std::string::npos || type.find("float") != std::string::npos ||
type == "number") {
addSpan(spans, node, TokenCategory::Number);
descend = false;
} else if (type == "identifier") {
TSNode parent = ts_node_parent(node);
std::string parentType = ts_node_is_null(parent) ? "" : nodeType(parent);
if (parentType == "function_declaration" || parentType == "method_declaration") {
addSpan(spans, node, TokenCategory::Function);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
descend = false;
} else if (type == "type_identifier" || type == "primitive_type") {
addSpan(spans, node, TokenCategory::Type);
descend = false;
} else if (!ts_node_is_named(node)) {
if (isGoKeyword(text)) {
addSpan(spans, node, TokenCategory::Keyword);
} else if (text == "(" || text == ")" || text == "[" || text == "]" ||
text == "{" || text == "}" || text == ";" || text == "," ||
text == ":" ) {
addSpan(spans, node, TokenCategory::Punctuation);
} else {
addSpan(spans, node, TokenCategory::Operator);
}
descend = false;
}
if (descend) {
uint32_t count = ts_node_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
walkGo(ts_node_child(node, i), source, spans);
}
}
}

View File

@@ -0,0 +1,56 @@
#pragma once
// SyntaxHighlighterJava.h helpers for SyntaxHighlighter.
static void walkJava(TSNode node, const std::string& source,
std::vector<HighlightSpan>& spans) {
if (ts_node_is_null(node)) return;
std::string type = nodeType(node);
std::string text = nodeText(node, source);
bool descend = true;
if (type.find("comment") != std::string::npos) {
addSpan(spans, node, TokenCategory::Comment);
descend = false;
} else if (type.find("string") != std::string::npos || type == "character_literal") {
addSpan(spans, node, TokenCategory::String);
descend = false;
} else if (type.find("integer") != std::string::npos || type.find("floating") != std::string::npos ||
type == "decimal_integer_literal" || type == "decimal_floating_point_literal") {
addSpan(spans, node, TokenCategory::Number);
descend = false;
} else if (type == "identifier") {
TSNode parent = ts_node_parent(node);
std::string parentType = ts_node_is_null(parent) ? "" : nodeType(parent);
if (parentType == "method_declaration" || parentType == "constructor_declaration") {
addSpan(spans, node, TokenCategory::Function);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
descend = false;
} else if (type == "type_identifier" || type == "integral_type" || type == "floating_point_type") {
addSpan(spans, node, TokenCategory::Type);
descend = false;
} else if (type == "true" || type == "false" || type == "null_literal") {
addSpan(spans, node, TokenCategory::Builtin);
descend = false;
} else if (!ts_node_is_named(node)) {
if (isJavaKeyword(text)) {
addSpan(spans, node, TokenCategory::Keyword);
} else if (text == "(" || text == ")" || text == "[" || text == "]" ||
text == "{" || text == "}" || text == ";" || text == "," ||
text == "." || text == ":" ) {
addSpan(spans, node, TokenCategory::Punctuation);
} else {
addSpan(spans, node, TokenCategory::Operator);
}
descend = false;
}
if (descend) {
uint32_t count = ts_node_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
walkJava(ts_node_child(node, i), source, spans);
}
}
}

View File

@@ -0,0 +1,57 @@
#pragma once
// SyntaxHighlighterJavaScript.h helpers for SyntaxHighlighter.
static void walkJavaScript(TSNode node, const std::string& source,
std::vector<HighlightSpan>& spans) {
if (ts_node_is_null(node)) return;
std::string type = nodeType(node);
std::string text = nodeText(node, source);
bool descend = true;
if (type.find("comment") != std::string::npos) {
addSpan(spans, node, TokenCategory::Comment);
descend = false;
} else if (type == "string" || type == "string_fragment" || type == "template_string" ||
type == "template_substitution") {
addSpan(spans, node, TokenCategory::String);
descend = false;
} else if (type == "number" || type == "number_literal") {
addSpan(spans, node, TokenCategory::Number);
descend = false;
} else if (type == "identifier" || type == "property_identifier") {
TSNode parent = ts_node_parent(node);
std::string parentType = ts_node_is_null(parent) ? "" : nodeType(parent);
if (parentType == "function_declaration" || parentType == "method_definition") {
addSpan(spans, node, TokenCategory::Function);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
descend = false;
} else if (type == "true" || type == "false" || type == "null" || type == "undefined") {
addSpan(spans, node, TokenCategory::Builtin);
descend = false;
} else if (!ts_node_is_named(node)) {
if (isJsKeyword(text)) {
addSpan(spans, node, TokenCategory::Keyword);
} else if (text == "(" || text == ")" || text == "[" || text == "]" ||
text == "{" || text == "}" || text == ";" || text == "," ||
text == "." || text == ":" ) {
addSpan(spans, node, TokenCategory::Punctuation);
} else {
addSpan(spans, node, TokenCategory::Operator);
}
descend = false;
}
if (descend) {
uint32_t count = ts_node_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
walkJavaScript(ts_node_child(node, i), source, spans);
}
}
}
static void walkTypeScript(TSNode node, const std::string& source,
std::vector<HighlightSpan>& spans) {
walkJavaScript(node, source, spans);
}

View File

@@ -0,0 +1,24 @@
#pragma once
// SyntaxHighlighterOrg.h helpers for SyntaxHighlighter.
static void walkOrgSimple(const std::string& source,
std::vector<HighlightSpan>& spans) {
size_t start = 0;
while (start < source.size()) {
size_t end = source.find('\n', start);
if (end == std::string::npos) end = source.size();
std::string line = source.substr(start, end - start);
std::string trimmed = line;
while (!trimmed.empty() && (trimmed.back() == '\r' || trimmed.back() == '\n')) {
trimmed.pop_back();
}
if (!trimmed.empty() && trimmed[0] == '*') {
spans.push_back({(uint32_t)start, (uint32_t)end, TokenCategory::Keyword});
} else if (trimmed.rfind("#+begin_src", 0) == 0 ||
trimmed.rfind("#+end_src", 0) == 0 ||
trimmed.rfind("#+", 0) == 0) {
spans.push_back({(uint32_t)start, (uint32_t)end, TokenCategory::Comment});
}
start = end + 1;
}
}

View File

@@ -0,0 +1,80 @@
#pragma once
// SyntaxHighlighterPython.h helpers for SyntaxHighlighter.
static void walkPython(TSNode node, const std::string& source,
std::vector<HighlightSpan>& spans) {
if (ts_node_is_null(node)) return;
std::string type = nodeType(node);
std::string text = nodeText(node, source);
bool descend = true;
// Leaf / terminal categorization
if (type == "comment") {
addSpan(spans, node, TokenCategory::Comment);
descend = false;
} else if (type == "string" || type == "concatenated_string") {
addSpan(spans, node, TokenCategory::String);
descend = false;
} else if (type == "integer" || type == "float") {
addSpan(spans, node, TokenCategory::Number);
descend = false;
} else if (type == "identifier") {
// Context-dependent: function name, parameter, builtin, or plain
TSNode parent = ts_node_parent(node);
std::string parentType = ts_node_is_null(parent) ? "" : nodeType(parent);
if (parentType == "function_definition") {
TSNode nameField = ts_node_child_by_field_name(parent, "name", 4);
if (!ts_node_is_null(nameField) &&
ts_node_start_byte(nameField) == ts_node_start_byte(node)) {
addSpan(spans, node, TokenCategory::Function);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
} else if (parentType == "parameters" || parentType == "default_parameter") {
addSpan(spans, node, TokenCategory::Parameter);
} else if (parentType == "call") {
TSNode funcField = ts_node_child_by_field_name(parent, "function", 8);
if (!ts_node_is_null(funcField) &&
ts_node_start_byte(funcField) == ts_node_start_byte(node)) {
addSpan(spans, node, TokenCategory::Function);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
} else if (parentType == "type") {
addSpan(spans, node, TokenCategory::Type);
} else if (isPythonBuiltin(text)) {
addSpan(spans, node, TokenCategory::Builtin);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
descend = false;
} else if (type == "type") {
addSpan(spans, node, TokenCategory::Type);
descend = false;
} else if (!ts_node_is_named(node)) {
// Anonymous/unnamed nodes are operators, keywords, punctuation
if (isPythonKeyword(text)) {
addSpan(spans, node, TokenCategory::Keyword);
} else if (text == "(" || text == ")" || text == "[" || text == "]" ||
text == "{" || text == "}" || text == ":" || text == "," ||
text == "." || text == ";") {
addSpan(spans, node, TokenCategory::Punctuation);
} else if (text == "+" || text == "-" || text == "*" || text == "/" ||
text == "%" || text == "=" || text == "==" || text == "!=" ||
text == "<" || text == ">" || text == "<=" || text == ">=" ||
text == "+=" || text == "-=" || text == "*=" || text == "/=" ||
text == "**" || text == "//" || text == "->" || text == "@") {
addSpan(spans, node, TokenCategory::Operator);
}
descend = false;
}
if (descend) {
uint32_t count = ts_node_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
walkPython(ts_node_child(node, i), source, spans);
}
}
}

View File

@@ -0,0 +1,54 @@
#pragma once
// SyntaxHighlighterRust.h helpers for SyntaxHighlighter.
static void walkRust(TSNode node, const std::string& source,
std::vector<HighlightSpan>& spans) {
if (ts_node_is_null(node)) return;
std::string type = nodeType(node);
std::string text = nodeText(node, source);
bool descend = true;
if (type.find("comment") != std::string::npos) {
addSpan(spans, node, TokenCategory::Comment);
descend = false;
} else if (type == "string_literal" || type == "char_literal" ||
type == "raw_string_literal") {
addSpan(spans, node, TokenCategory::String);
descend = false;
} else if (type.find("integer") != std::string::npos || type.find("float") != std::string::npos ||
type == "number") {
addSpan(spans, node, TokenCategory::Number);
descend = false;
} else if (type == "identifier") {
TSNode parent = ts_node_parent(node);
std::string parentType = ts_node_is_null(parent) ? "" : nodeType(parent);
if (parentType == "function_item") {
addSpan(spans, node, TokenCategory::Function);
} else {
addSpan(spans, node, TokenCategory::Identifier);
}
descend = false;
} else if (type == "type_identifier" || type == "primitive_type") {
addSpan(spans, node, TokenCategory::Type);
descend = false;
} else if (!ts_node_is_named(node)) {
if (isRustKeyword(text)) {
addSpan(spans, node, TokenCategory::Keyword);
} else if (text == "(" || text == ")" || text == "[" || text == "]" ||
text == "{" || text == "}" || text == ";" || text == "," ||
text == ":" ) {
addSpan(spans, node, TokenCategory::Punctuation);
} else {
addSpan(spans, node, TokenCategory::Operator);
}
descend = false;
}
if (descend) {
uint32_t count = ts_node_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
walkRust(ts_node_child(node, i), source, spans);
}
}
}

View File

@@ -0,0 +1,124 @@
#pragma once
// Language keyword helpers for SyntaxHighlighter.
static bool isPythonKeyword(const std::string& text) {
static const char* keywords[] = {
"def", "class", "if", "elif", "else", "for", "while", "return",
"import", "from", "as", "try", "except", "finally", "raise",
"with", "yield", "lambda", "pass", "break", "continue",
"and", "or", "not", "in", "is", "del", "global", "nonlocal",
"assert", "async", "await", nullptr
};
for (const char** k = keywords; *k; ++k) {
if (text == *k) return true;
}
return false;
}
static bool isPythonBuiltin(const std::string& text) {
static const char* builtins[] = {
"True", "False", "None", "print", "len", "range", "int",
"str", "float", "list", "dict", "set", "tuple", "type",
"isinstance", "super", "self", nullptr
};
for (const char** b = builtins; *b; ++b) {
if (text == *b) return true;
}
return false;
}
static bool isCppKeyword(const std::string& text) {
static const char* keywords[] = {
"auto", "break", "case", "catch", "class", "const", "constexpr",
"continue", "default", "delete", "do", "else", "enum", "explicit",
"extern", "for", "friend", "goto", "if", "inline", "mutable",
"namespace", "new", "noexcept", "operator", "private", "protected",
"public", "register", "return", "sizeof", "static", "static_assert",
"static_cast", "struct", "switch", "template", "this", "throw",
"try", "typedef", "typeid", "typename", "union", "using",
"virtual", "volatile", "while", "override", "final",
"co_await", "co_return", "co_yield", "concept", "requires",
"consteval", "constinit", nullptr
};
for (const char** k = keywords; *k; ++k) {
if (text == *k) return true;
}
return false;
}
static bool isCppType(const std::string& text) {
static const char* types[] = {
"void", "bool", "char", "int", "float", "double", "long",
"short", "unsigned", "signed", "size_t", "nullptr_t",
"int8_t", "int16_t", "int32_t", "int64_t",
"uint8_t", "uint16_t", "uint32_t", "uint64_t",
"string", "vector", "map", "set", "array", "tuple",
"unique_ptr", "shared_ptr", "weak_ptr", "optional",
nullptr
};
for (const char** t = types; *t; ++t) {
if (text == *t) return true;
}
return false;
}
static bool isJsKeyword(const std::string& text) {
static const char* keywords[] = {
"function", "return", "if", "else", "for", "while", "class",
"const", "let", "var", "import", "export", "new", "try", "catch",
"finally", "switch", "case", "break", "continue", "throw",
"async", "await", "yield", nullptr
};
for (const char** k = keywords; *k; ++k) {
if (text == *k) return true;
}
return false;
}
static bool isJavaKeyword(const std::string& text) {
static const char* keywords[] = {
"class", "interface", "enum", "public", "private", "protected",
"static", "final", "void", "int", "float", "double", "boolean",
"return", "if", "else", "for", "while", "switch", "case",
"break", "continue", "new", "try", "catch", "finally", "throw",
"extends", "implements", "import", "package", "this", "super",
nullptr
};
for (const char** k = keywords; *k; ++k) {
if (text == *k) return true;
}
return false;
}
static bool isRustKeyword(const std::string& text) {
static const char* keywords[] = {
"fn", "let", "mut", "pub", "impl", "trait", "struct", "enum",
"use", "mod", "crate", "self", "super", "return", "if", "else",
"for", "while", "loop", "match", "break", "continue", "const",
"static", "async", "await", "move", nullptr
};
for (const char** k = keywords; *k; ++k) {
if (text == *k) return true;
}
return false;
}
static bool isGoKeyword(const std::string& text) {
static const char* keywords[] = {
"func", "package", "import", "return", "if", "else", "for",
"switch", "case", "break", "continue", "struct", "interface",
"type", "var", "const", "go", "defer", "range", nullptr
};
for (const char** k = keywords; *k; ++k) {
if (text == *k) return true;
}
return false;
}
static bool isElispKeyword(const std::string& text) {
static const char* keywords[] = {
"defun", "defvar", "defconst", "defmacro", "defcustom",
"let", "let*", "if", "when", "unless", "cond", "while",
"dolist", "dotimes", "progn", "prog1", "prog2",
"lambda", "setq", "setf", "require", "provide",
"interactive", "save-excursion", "save-restriction",
"condition-case", "unwind-protect", "catch", "throw",
nullptr
};
for (const char** k = keywords; *k; ++k) {
if (text == *k) return true;
}
return false;
}

View File

@@ -509,200 +509,4 @@ public:
return kind; // Return as-is if not a common type
}
std::string visitListType(const ListType* type) override {
std::ostringstream oss;
oss << "std::vector<";
auto elementType = type->getChild("elementType");
if (elementType) {
oss << generate(elementType);
} else {
oss << "auto"; // Default if no element type specified
}
oss << ">";
return oss.str();
}
std::string visitSetType(const SetType* type) override {
std::ostringstream oss;
oss << "std::set<";
auto elementType = type->getChild("elementType");
if (elementType) {
oss << generate(elementType);
} else {
oss << "auto"; // Default if no element type specified
}
oss << ">";
return oss.str();
}
std::string visitMapType(const MapType* type) override {
std::ostringstream oss;
oss << "std::map<";
auto keyType = type->getChild("keyType");
auto valueType = type->getChild("valueType");
if (keyType) {
oss << generate(keyType);
} else {
oss << "auto"; // Default if no key type specified
}
oss << ", ";
if (valueType) {
oss << generate(valueType);
} else {
oss << "auto"; // Default if no value type specified
}
oss << ">";
return oss.str();
}
std::string visitTupleType(const TupleType* type) override {
std::ostringstream oss;
oss << "std::tuple<";
auto elementTypes = type->getChildren("elementTypes");
for (size_t i = 0; i < elementTypes.size(); ++i) {
if (i > 0) oss << ", ";
oss << generate(elementTypes[i]);
}
oss << ">";
return oss.str();
}
std::string visitArrayType(const ArrayType* type) override {
std::ostringstream oss;
oss << "std::array<";
auto elementType = type->getChild("elementType");
if (elementType) {
oss << generate(elementType);
} else {
oss << "auto"; // Default if no element type specified
}
oss << ", /* size unknown */>";
return oss.str();
}
std::string visitOptionalType(const OptionalType* type) override {
std::ostringstream oss;
oss << "std::optional<";
auto innerType = type->getChild("innerType");
if (innerType) {
oss << generate(innerType);
} else {
oss << "auto"; // Default if no inner type specified
}
oss << ">";
return oss.str();
}
std::string visitCustomType(const CustomType* type) override {
return type->typeName;
}
std::string visitDerefStrategy(const DerefStrategy* annotation) override {
// Generate C++-style comment for deref strategy
if (annotation->strategy == "batched") {
return "// @deref(batched) - Use batched memory management (smart pointers)";
} else if (annotation->strategy == "streamed") {
return "// @deref(streamed) - Use streamed memory management (RAII)";
} else if (annotation->strategy == "manual") {
return "// @deref(manual) - Use manual memory management (new/delete)";
} else {
return "// @deref(" + annotation->strategy + ") - Memory management strategy";
}
}
std::string visitOptimizationLock(const OptimizationLock* annotation) override {
return "// @lock(" + annotation->lockedBy + ") - Optimization locked: " + annotation->lockReason;
}
std::string visitLangSpecific(const LangSpecific* annotation) override {
return "// @lang_specific(" + annotation->language + ", " + annotation->idiomType + ")";
}
std::string visitDeallocateAnnotation(const DeallocateAnnotation* annotation) override {
return "// @dealloc(" + annotation->strategy + ") - Memory deallocation strategy";
}
std::string visitLifetimeAnnotation(const LifetimeAnnotation* annotation) override {
return "// @lifetime(" + annotation->strategy + ") - Object lifetime management";
}
std::string visitReclaimAnnotation(const ReclaimAnnotation* annotation) override {
return "// @reclaim(" + annotation->strategy + ") - Memory reclamation strategy";
}
std::string visitOwnerAnnotation(const OwnerAnnotation* annotation) override {
return "// @owner(" + annotation->strategy + ") - Ownership management strategy";
}
std::string visitAllocateAnnotation(const AllocateAnnotation* annotation) override {
return "// @allocate(" + annotation->strategy + ") - Memory allocation strategy";
}
std::string visitHotColdAnnotation(const HotColdAnnotation* annotation) override {
if (annotation->hint == "Hot")
return "__attribute__((hot))";
if (annotation->hint == "Cold")
return "__attribute__((cold))";
return "// @hotcold(" + annotation->hint + ")";
}
std::string visitInlineAnnotation(const InlineAnnotation* annotation) override {
if (annotation->mode == "Always")
return "[[gnu::always_inline]] inline";
if (annotation->mode == "Never")
return "__attribute__((noinline))";
return "inline";
}
std::string visitPureAnnotation(const PureAnnotation*) override {
return "[[nodiscard]]";
}
std::string visitConstExprAnnotation(const ConstExprAnnotation*) override {
return "constexpr";
}
private:
// Check enclosing function's memory annotations to determine smart-pointer wrapper
std::string getMemoryTypeWrapper(const Variable* variable) const {
const ASTNode* cur = variable->parent;
while (cur && cur->conceptType != "Function") {
cur = cur->parent;
}
if (!cur) return "";
for (auto* anno : cur->getChildren("annotations")) {
if (anno->conceptType == "ReclaimAnnotation") {
auto* ra = static_cast<const ReclaimAnnotation*>(anno);
if (ra->strategy == "Tracing" || ra->strategy == "Cycle")
return "std::shared_ptr";
}
if (anno->conceptType == "LifetimeAnnotation") {
auto* la = static_cast<const LifetimeAnnotation*>(anno);
if (la->strategy == "RAII")
return "std::unique_ptr";
}
if (anno->conceptType == "OwnerAnnotation") {
auto* oa = static_cast<const OwnerAnnotation*>(anno);
if (oa->strategy == "Shared_ARC") return "std::shared_ptr";
if (oa->strategy == "Single") return "std::unique_ptr";
}
}
return "";
}
};
#include "ast/CppGeneratorTypes.h"

View File

@@ -0,0 +1,199 @@
#pragma once
// CppGenerator type + annotation helpers.
std::string visitListType(const ListType* type) override {
std::ostringstream oss;
oss << "std::vector<";
auto elementType = type->getChild("elementType");
if (elementType) {
oss << generate(elementType);
} else {
oss << "auto"; // Default if no element type specified
}
oss << ">";
return oss.str();
}
std::string visitSetType(const SetType* type) override {
std::ostringstream oss;
oss << "std::set<";
auto elementType = type->getChild("elementType");
if (elementType) {
oss << generate(elementType);
} else {
oss << "auto"; // Default if no element type specified
}
oss << ">";
return oss.str();
}
std::string visitMapType(const MapType* type) override {
std::ostringstream oss;
oss << "std::map<";
auto keyType = type->getChild("keyType");
auto valueType = type->getChild("valueType");
if (keyType) {
oss << generate(keyType);
} else {
oss << "auto"; // Default if no key type specified
}
oss << ", ";
if (valueType) {
oss << generate(valueType);
} else {
oss << "auto"; // Default if no value type specified
}
oss << ">";
return oss.str();
}
std::string visitTupleType(const TupleType* type) override {
std::ostringstream oss;
oss << "std::tuple<";
auto elementTypes = type->getChildren("elementTypes");
for (size_t i = 0; i < elementTypes.size(); ++i) {
if (i > 0) oss << ", ";
oss << generate(elementTypes[i]);
}
oss << ">";
return oss.str();
}
std::string visitArrayType(const ArrayType* type) override {
std::ostringstream oss;
oss << "std::array<";
auto elementType = type->getChild("elementType");
if (elementType) {
oss << generate(elementType);
} else {
oss << "auto"; // Default if no element type specified
}
oss << ", /* size unknown */>";
return oss.str();
}
std::string visitOptionalType(const OptionalType* type) override {
std::ostringstream oss;
oss << "std::optional<";
auto innerType = type->getChild("innerType");
if (innerType) {
oss << generate(innerType);
} else {
oss << "auto"; // Default if no inner type specified
}
oss << ">";
return oss.str();
}
std::string visitCustomType(const CustomType* type) override {
return type->typeName;
}
std::string visitDerefStrategy(const DerefStrategy* annotation) override {
// Generate C++-style comment for deref strategy
if (annotation->strategy == "batched") {
return "// @deref(batched) - Use batched memory management (smart pointers)";
} else if (annotation->strategy == "streamed") {
return "// @deref(streamed) - Use streamed memory management (RAII)";
} else if (annotation->strategy == "manual") {
return "// @deref(manual) - Use manual memory management (new/delete)";
} else {
return "// @deref(" + annotation->strategy + ") - Memory management strategy";
}
}
std::string visitOptimizationLock(const OptimizationLock* annotation) override {
return "// @lock(" + annotation->lockedBy + ") - Optimization locked: " + annotation->lockReason;
}
std::string visitLangSpecific(const LangSpecific* annotation) override {
return "// @lang_specific(" + annotation->language + ", " + annotation->idiomType + ")";
}
std::string visitDeallocateAnnotation(const DeallocateAnnotation* annotation) override {
return "// @dealloc(" + annotation->strategy + ") - Memory deallocation strategy";
}
std::string visitLifetimeAnnotation(const LifetimeAnnotation* annotation) override {
return "// @lifetime(" + annotation->strategy + ") - Object lifetime management";
}
std::string visitReclaimAnnotation(const ReclaimAnnotation* annotation) override {
return "// @reclaim(" + annotation->strategy + ") - Memory reclamation strategy";
}
std::string visitOwnerAnnotation(const OwnerAnnotation* annotation) override {
return "// @owner(" + annotation->strategy + ") - Ownership management strategy";
}
std::string visitAllocateAnnotation(const AllocateAnnotation* annotation) override {
return "// @allocate(" + annotation->strategy + ") - Memory allocation strategy";
}
std::string visitHotColdAnnotation(const HotColdAnnotation* annotation) override {
if (annotation->hint == "Hot")
return "__attribute__((hot))";
if (annotation->hint == "Cold")
return "__attribute__((cold))";
return "// @hotcold(" + annotation->hint + ")";
}
std::string visitInlineAnnotation(const InlineAnnotation* annotation) override {
if (annotation->mode == "Always")
return "[[gnu::always_inline]] inline";
if (annotation->mode == "Never")
return "__attribute__((noinline))";
return "inline";
}
std::string visitPureAnnotation(const PureAnnotation*) override {
return "[[nodiscard]]";
}
std::string visitConstExprAnnotation(const ConstExprAnnotation*) override {
return "constexpr";
}
private:
// Check enclosing function's memory annotations to determine smart-pointer wrapper
std::string getMemoryTypeWrapper(const Variable* variable) const {
const ASTNode* cur = variable->parent;
while (cur && cur->conceptType != "Function") {
cur = cur->parent;
}
if (!cur) return "";
for (auto* anno : cur->getChildren("annotations")) {
if (anno->conceptType == "ReclaimAnnotation") {
auto* ra = static_cast<const ReclaimAnnotation*>(anno);
if (ra->strategy == "Tracing" || ra->strategy == "Cycle")
return "std::shared_ptr";
}
if (anno->conceptType == "LifetimeAnnotation") {
auto* la = static_cast<const LifetimeAnnotation*>(anno);
if (la->strategy == "RAII")
return "std::unique_ptr";
}
if (anno->conceptType == "OwnerAnnotation") {
auto* oa = static_cast<const OwnerAnnotation*>(anno);
if (oa->strategy == "Shared_ARC") return "std::shared_ptr";
if (oa->strategy == "Single") return "std::unique_ptr";
}
}
return "";
}
};

297
editor/src/ast/CppParser.h Normal file
View File

@@ -0,0 +1,297 @@
#pragma once
// TreeSitterParser Cpp support.
public:
// C++
// ---------------------------------------------------------------
static std::unique_ptr<Module> parseCpp(const std::string& source) {
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, tree_sitter_cpp());
TSTree* tree = ts_parser_parse_string(parser, nullptr, source.c_str(), (uint32_t)source.size());
TSNode root = ts_tree_root_node(tree);
auto module = std::make_unique<Module>();
module->id = IdGenerator::next("mod");
module->name = "parsed_cpp_module";
module->targetLanguage = "cpp";
applySpan(module.get(), root);
convertCppTranslationUnit(root, source, module.get());
ts_tree_delete(tree);
ts_parser_delete(parser);
return module;
}
static ParseResult parseCppWithDiagnostics(const std::string& source) {
ParseResult result;
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, tree_sitter_cpp());
TSTree* tree = ts_parser_parse_string(parser, nullptr, source.c_str(), (uint32_t)source.size());
TSNode root = ts_tree_root_node(tree);
result.module = std::make_unique<Module>();
result.module->id = IdGenerator::next("mod");
result.module->name = "parsed_cpp_module";
result.module->targetLanguage = "cpp";
applySpan(result.module.get(), root);
convertCppTranslationUnit(root, source, result.module.get());
collectDiagnostics(root, source, result.diagnostics);
ts_tree_delete(tree);
ts_parser_delete(parser);
return result;
}
// ---------------------------------------------------------------
private:
// C++ CST → AST
// ---------------------------------------------------------------
static void convertCppTranslationUnit(TSNode root, const std::string& source, Module* module) {
uint32_t count = ts_node_named_child_count(root);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(root, i);
std::string type = nodeType(child);
if (type == "function_definition") {
auto* fn = convertCppFunction(child, source);
if (fn) module->addChild("functions", fn);
}
}
}
static Function* convertCppFunction(TSNode node, const std::string& source) {
auto* fn = new Function();
fn->id = IdGenerator::next("fn");
applySpan(fn, node);
// In C++ grammar, the structure is:
// function_definition: type declarator body
// The declarator contains the function name and parameters.
// Return type
TSNode typeNode = childByFieldName(node, "type");
if (!ts_node_is_null(typeNode)) {
std::string typeText = nodeText(typeNode, source);
auto* retType = new PrimitiveType(IdGenerator::next("type"), typeText);
fn->setChild("returnType", retType);
}
// Declarator: function_declarator which has declarator (name) and parameters
TSNode declaratorNode = childByFieldName(node, "declarator");
if (!ts_node_is_null(declaratorNode)) {
extractCppFunctionName(declaratorNode, source, fn);
extractCppParameters(declaratorNode, source, fn);
}
// Body
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
convertCppBody(bodyNode, source, fn);
}
// Memory pattern detection from source text
std::string bodySource = "";
if (!ts_node_is_null(bodyNode)) {
bodySource = nodeText(bodyNode, source);
}
detectCppMemoryPatterns(bodySource, fn);
return fn;
}
static void extractCppFunctionName(TSNode declNode, const std::string& source, Function* fn) {
std::string type = nodeType(declNode);
if (type == "function_declarator") {
TSNode nameNode = childByFieldName(declNode, "declarator");
if (!ts_node_is_null(nameNode)) {
// Could be an identifier directly, or nested further
fn->name = nodeText(nameNode, source);
}
} else if (type == "identifier") {
fn->name = nodeText(declNode, source);
} else {
// Try to find function_declarator among children
uint32_t count = ts_node_named_child_count(declNode);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(declNode, i);
std::string childType = nodeType(child);
if (childType == "function_declarator") {
extractCppFunctionName(child, source, fn);
return;
}
}
// Fallback
fn->name = nodeText(declNode, source);
}
}
static void extractCppParameters(TSNode declNode, const std::string& source, Function* fn) {
std::string type = nodeType(declNode);
if (type == "function_declarator") {
TSNode paramsNode = childByFieldName(declNode, "parameters");
if (!ts_node_is_null(paramsNode)) {
uint32_t count = ts_node_named_child_count(paramsNode);
for (uint32_t i = 0; i < count; ++i) {
TSNode paramChild = ts_node_named_child(paramsNode, i);
std::string paramType = nodeType(paramChild);
if (paramType == "parameter_declaration") {
convertCppParameter(paramChild, source, fn);
}
}
}
} else {
// Search for function_declarator child
uint32_t count = ts_node_named_child_count(declNode);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(declNode, i);
if (nodeType(child) == "function_declarator") {
extractCppParameters(child, source, fn);
return;
}
}
}
}
static void convertCppParameter(TSNode paramNode, const std::string& source, Function* fn) {
auto* param = new Parameter();
param->id = IdGenerator::next("param");
applySpan(param, paramNode);
// parameter_declaration has type and declarator fields
TSNode typeNode = childByFieldName(paramNode, "type");
TSNode declNode = childByFieldName(paramNode, "declarator");
if (!ts_node_is_null(typeNode)) {
std::string typeText = nodeText(typeNode, source);
auto* primType = new PrimitiveType(IdGenerator::next("type"), typeText);
applySpan(primType, typeNode);
param->setChild("type", primType);
}
if (!ts_node_is_null(declNode)) {
param->name = nodeText(declNode, source);
}
fn->addChild("parameters", param);
}
static void convertCppBody(TSNode bodyNode, const std::string& source, Function* fn) {
// bodyNode is a compound_statement { ... }
uint32_t count = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(bodyNode, i);
ASTNode* stmt = convertCppStatement(child, source);
if (stmt) fn->addChild("body", stmt);
}
}
static ASTNode* convertCppStatement(TSNode node, const std::string& source) {
std::string type = nodeType(node);
if (type == "return_statement") {
auto* ret = new Return();
ret->id = IdGenerator::next("ret");
applySpan(ret, node);
uint32_t count = ts_node_named_child_count(node);
if (count > 0) {
ASTNode* val = convertCppExpression(ts_node_named_child(node, 0), source);
if (val) ret->setChild("value", val);
}
return ret;
} else if (type == "expression_statement") {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, node);
uint32_t count = ts_node_named_child_count(node);
if (count > 0) {
ASTNode* expr = convertCppExpression(ts_node_named_child(node, 0), source);
if (expr) exprStmt->setChild("expression", expr);
}
return exprStmt;
} else if (type == "declaration") {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, node);
return exprStmt;
} else if (type == "if_statement") {
auto* ifStmt = new IfStatement();
ifStmt->id = IdGenerator::next("if");
applySpan(ifStmt, node);
return ifStmt;
}
return nullptr;
}
static ASTNode* convertCppExpression(TSNode node, const std::string& source) {
std::string type = nodeType(node);
if (type == "binary_expression") {
auto* binOp = new BinaryOperation();
binOp->id = IdGenerator::next("binop");
applySpan(binOp, node);
TSNode leftNode = childByFieldName(node, "left");
TSNode rightNode = childByFieldName(node, "right");
TSNode opNode = childByFieldName(node, "operator");
if (!ts_node_is_null(opNode)) {
binOp->op = nodeText(opNode, source);
}
if (!ts_node_is_null(leftNode)) {
ASTNode* left = convertCppExpression(leftNode, source);
if (left) binOp->setChild("left", left);
}
if (!ts_node_is_null(rightNode)) {
ASTNode* right = convertCppExpression(rightNode, source);
if (right) binOp->setChild("right", right);
}
return binOp;
} else if (type == "identifier") {
auto* ref = new VariableReference(IdGenerator::next("var"), nodeText(node, source));
applySpan(ref, node);
return ref;
} else if (type == "number_literal") {
std::string text = nodeText(node, source);
int val = 0;
try { val = std::stoi(text); } catch (...) {}
auto* lit = new IntegerLiteral(IdGenerator::next("int"), val);
applySpan(lit, node);
return lit;
} else if (type == "string_literal" || type == "raw_string_literal") {
auto* lit = new StringLiteral(IdGenerator::next("str"), nodeText(node, source));
applySpan(lit, node);
return lit;
} else if (type == "parenthesized_expression") {
uint32_t count = ts_node_named_child_count(node);
if (count > 0) return convertCppExpression(ts_node_named_child(node, 0), source);
}
// Fallback
std::string text = nodeText(node, source);
if (!text.empty()) {
auto* ref = new VariableReference(IdGenerator::next("var"), text);
applySpan(ref, node);
return ref;
}
return nullptr;
}
static void detectCppMemoryPatterns(const std::string& bodySource, Function* fn) {
bool hasUnique = bodySource.find("unique_ptr") != std::string::npos ||
bodySource.find("make_unique") != std::string::npos;
bool hasShared = bodySource.find("shared_ptr") != std::string::npos ||
bodySource.find("make_shared") != std::string::npos;
bool hasNew = bodySource.find("new ") != std::string::npos;
bool hasDelete = bodySource.find("delete ") != std::string::npos ||
bodySource.find("delete;") != std::string::npos;
if (hasUnique) {
auto* anno = new LifetimeAnnotation(IdGenerator::next("anno"), "RAII");
fn->addChild("annotations", anno);
}
if (hasShared) {
auto* anno = new OwnerAnnotation(IdGenerator::next("anno"), "Shared_ARC");
fn->addChild("annotations", anno);
}
if (hasNew && hasDelete) {
auto* anno = new DeallocateAnnotation(IdGenerator::next("anno"), "Explicit");
fn->addChild("annotations", anno);
}
}
// ---------------------------------------------------------------

View File

@@ -0,0 +1,389 @@
#pragma once
// TreeSitterParser Elisp support.
public:
// Elisp
// ---------------------------------------------------------------
static std::unique_ptr<Module> parseElisp(const std::string& source) {
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, tree_sitter_elisp());
TSTree* tree = ts_parser_parse_string(parser, nullptr, source.c_str(), (uint32_t)source.size());
TSNode root = ts_tree_root_node(tree);
auto module = std::make_unique<Module>();
module->id = IdGenerator::next("mod");
module->name = "parsed_elisp_module";
module->targetLanguage = "elisp";
applySpan(module.get(), root);
convertElispSourceFile(root, source, module.get());
ts_tree_delete(tree);
ts_parser_delete(parser);
return module;
}
static ParseResult parseElispWithDiagnostics(const std::string& source) {
ParseResult result;
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, tree_sitter_elisp());
TSTree* tree = ts_parser_parse_string(parser, nullptr, source.c_str(), (uint32_t)source.size());
TSNode root = ts_tree_root_node(tree);
result.module = std::make_unique<Module>();
result.module->id = IdGenerator::next("mod");
result.module->name = "parsed_elisp_module";
result.module->targetLanguage = "elisp";
applySpan(result.module.get(), root);
convertElispSourceFile(root, source, result.module.get());
collectDiagnostics(root, source, result.diagnostics);
ts_tree_delete(tree);
ts_parser_delete(parser);
return result;
}
// ---------------------------------------------------------------
private:
// Elisp CST → AST
// ---------------------------------------------------------------
static void convertElispSourceFile(TSNode root, const std::string& source, Module* module) {
uint32_t count = ts_node_named_child_count(root);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(root, i);
std::string type = nodeType(child);
if (type == "function_definition" || type == "defun") {
auto* fn = convertElispDefun(child, source);
if (fn) module->addChild("functions", fn);
} else if (type == "list") {
auto* fn = tryConvertElispDefunFromList(child, source);
if (fn) module->addChild("functions", fn);
} else if (type == "special_form") {
auto* fn = tryConvertElispDefunFromSpecialForm(child, source);
if (fn) module->addChild("functions", fn);
}
}
}
static Function* convertElispDefun(TSNode node, const std::string& source) {
auto* fn = new Function();
fn->id = IdGenerator::next("fn");
applySpan(fn, node);
// tree-sitter-elisp function_definition has:
// field "name" → symbol (function name)
// field "parameters" → list (arglist)
// remaining named children → body forms (no field name)
TSNode nameNode = childByFieldName(node, "name");
if (!ts_node_is_null(nameNode)) {
fn->name = nodeText(nameNode, source);
}
TSNode paramsNode = childByFieldName(node, "parameters");
if (!ts_node_is_null(paramsNode)) {
convertElispArglist(paramsNode, source, fn);
}
// Body: iterate all named children, skip name and parameters
uint32_t count = ts_node_named_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(node, i);
// Skip the name symbol and parameters list
if (!ts_node_is_null(nameNode) &&
ts_node_start_byte(child) == ts_node_start_byte(nameNode) &&
ts_node_end_byte(child) == ts_node_end_byte(nameNode))
continue;
if (!ts_node_is_null(paramsNode) &&
ts_node_start_byte(child) == ts_node_start_byte(paramsNode) &&
ts_node_end_byte(child) == ts_node_end_byte(paramsNode))
continue;
// This is a body form
ASTNode* expr = convertElispExpression(child, source);
if (expr) {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, child);
exprStmt->setChild("expression", expr);
fn->addChild("body", exprStmt);
}
}
// Fallback: if no name/params fields, try positional approach
if (fn->name.empty()) {
uint32_t nc = ts_node_named_child_count(node);
for (uint32_t i = 0; i < nc; ++i) {
TSNode child = ts_node_named_child(node, i);
std::string childType = nodeType(child);
if (childType == "symbol" && fn->name.empty()) {
fn->name = nodeText(child, source);
} else if (childType == "list" && fn->getChildren("parameters").empty()) {
convertElispArglist(child, source, fn);
}
}
}
// Auto-annotate: Elisp uses tracing GC
auto* reclaim = new ReclaimAnnotation(IdGenerator::next("anno"), "Tracing");
fn->addChild("annotations", reclaim);
return fn;
}
// Handle (defun ...) when parsed as a generic list node
static Function* tryConvertElispDefunFromList(TSNode node, const std::string& source) {
// Check if first child is "defun" symbol
uint32_t count = ts_node_named_child_count(node);
if (count < 3) return nullptr;
TSNode firstChild = ts_node_named_child(node, 0);
std::string firstText = nodeText(firstChild, source);
if (firstText != "defun") return nullptr;
auto* fn = new Function();
fn->id = IdGenerator::next("fn");
// Second child is the name
TSNode nameChild = ts_node_named_child(node, 1);
fn->name = nodeText(nameChild, source);
// Third child is the arglist (a list)
TSNode arglistChild = ts_node_named_child(node, 2);
if (nodeType(arglistChild) == "list") {
convertElispArglist(arglistChild, source, fn);
}
// Remaining children are body forms
for (uint32_t i = 3; i < count; ++i) {
TSNode bodyChild = ts_node_named_child(node, i);
ASTNode* expr = convertElispExpression(bodyChild, source);
if (expr) {
if (i == count - 1) {
// Last form — wrap in ExpressionStatement (implicit return)
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, bodyChild);
exprStmt->setChild("expression", expr);
fn->addChild("body", exprStmt);
} else {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, bodyChild);
exprStmt->setChild("expression", expr);
fn->addChild("body", exprStmt);
}
}
}
// Auto-annotate
auto* reclaim = new ReclaimAnnotation(IdGenerator::next("anno"), "Tracing");
fn->addChild("annotations", reclaim);
return fn;
}
static Function* tryConvertElispDefunFromSpecialForm(TSNode node, const std::string& source) {
// special_form might contain defun
uint32_t count = ts_node_named_child_count(node);
if (count < 3) return nullptr;
TSNode firstChild = ts_node_named_child(node, 0);
std::string firstText = nodeText(firstChild, source);
if (firstText != "defun") return nullptr;
// Same logic as tryConvertElispDefunFromList
auto* fn = new Function();
fn->id = IdGenerator::next("fn");
TSNode nameChild = ts_node_named_child(node, 1);
fn->name = nodeText(nameChild, source);
TSNode arglistChild = ts_node_named_child(node, 2);
std::string argType = nodeType(arglistChild);
if (argType == "list") {
convertElispArglist(arglistChild, source, fn);
}
for (uint32_t i = 3; i < count; ++i) {
TSNode bodyChild = ts_node_named_child(node, i);
ASTNode* expr = convertElispExpression(bodyChild, source);
if (expr) {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, bodyChild);
exprStmt->setChild("expression", expr);
fn->addChild("body", exprStmt);
}
}
auto* reclaim = new ReclaimAnnotation(IdGenerator::next("anno"), "Tracing");
fn->addChild("annotations", reclaim);
return fn;
}
static void convertElispArglist(TSNode node, const std::string& source, Function* fn) {
uint32_t count = ts_node_named_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(node, i);
std::string type = nodeType(child);
if (type == "symbol" || type == "identifier") {
auto* param = new Parameter(IdGenerator::next("param"), nodeText(child, source));
applySpan(param, child);
fn->addChild("parameters", param);
}
}
}
static void convertElispBodyField(TSNode bodyNode, const std::string& source, Function* fn) {
// body might be a single node or we need to iterate children
uint32_t count = ts_node_named_child_count(bodyNode);
if (count > 0) {
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(bodyNode, i);
ASTNode* expr = convertElispExpression(child, source);
if (expr) {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, child);
exprStmt->setChild("expression", expr);
fn->addChild("body", exprStmt);
}
}
} else {
ASTNode* expr = convertElispExpression(bodyNode, source);
if (expr) {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, bodyNode);
exprStmt->setChild("expression", expr);
fn->addChild("body", exprStmt);
}
}
}
static void convertElispBodyFromChildren(TSNode node, const std::string& source, Function* fn) {
// Skip: first named child should be name, second should be arglist, rest is body
uint32_t count = ts_node_named_child_count(node);
int bodyStart = -1;
int arglistSeen = 0;
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(node, i);
std::string type = nodeType(child);
if (type == "symbol" && fn->name.empty()) {
fn->name = nodeText(child, source);
continue;
}
if (type == "list" && fn->getChildren("parameters").empty()) {
convertElispArglist(child, source, fn);
arglistSeen = 1;
continue;
}
if (arglistSeen || (int)i >= 2) {
// Body form
ASTNode* expr = convertElispExpression(child, source);
if (expr) {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, child);
exprStmt->setChild("expression", expr);
fn->addChild("body", exprStmt);
}
}
}
}
static ASTNode* convertElispExpression(TSNode node, const std::string& source) {
std::string type = nodeType(node);
if (type == "list") {
// Check if operator-form: (+ x 1), (- x 1), (* x y), (/ x y)
uint32_t count = ts_node_named_child_count(node);
if (count >= 3) {
TSNode firstChild = ts_node_named_child(node, 0);
std::string firstText = nodeText(firstChild, source);
if (firstText == "+" || firstText == "-" || firstText == "*" || firstText == "/") {
auto* binOp = new BinaryOperation();
binOp->id = IdGenerator::next("binop");
binOp->op = firstText;
applySpan(binOp, node);
ASTNode* left = convertElispExpression(ts_node_named_child(node, 1), source);
ASTNode* right = convertElispExpression(ts_node_named_child(node, 2), source);
if (left) binOp->setChild("left", left);
if (right) binOp->setChild("right", right);
return binOp;
}
}
// Generic list — could be a function call
if (count >= 1) {
auto* call = new FunctionCall();
call->id = IdGenerator::next("call");
applySpan(call, node);
TSNode funcName = ts_node_named_child(node, 0);
call->functionName = nodeText(funcName, source);
for (uint32_t i = 1; i < count; ++i) {
ASTNode* arg = convertElispExpression(ts_node_named_child(node, i), source);
if (arg) call->addChild("arguments", arg);
}
return call;
}
} else if (type == "symbol" || type == "identifier") {
auto* ref = new VariableReference(IdGenerator::next("var"), nodeText(node, source));
applySpan(ref, node);
return ref;
} else if (type == "integer" || type == "number") {
std::string text = nodeText(node, source);
int val = 0;
try { val = std::stoi(text); } catch (...) {}
auto* lit = new IntegerLiteral(IdGenerator::next("int"), val);
applySpan(lit, node);
return lit;
} else if (type == "string") {
auto* lit = new StringLiteral(IdGenerator::next("str"), nodeText(node, source));
applySpan(lit, node);
return lit;
} else if (type == "special_form") {
// Could be (if ...), (let ...), etc.
return convertElispSpecialForm(node, source);
}
// Fallback
std::string text = nodeText(node, source);
if (!text.empty()) {
auto* ref = new VariableReference(IdGenerator::next("var"), text);
applySpan(ref, node);
return ref;
}
return nullptr;
}
static ASTNode* convertElispSpecialForm(TSNode node, const std::string& source) {
uint32_t count = ts_node_named_child_count(node);
if (count < 1) return nullptr;
TSNode firstChild = ts_node_named_child(node, 0);
std::string formName = nodeText(firstChild, source);
if (formName == "if" && count >= 3) {
auto* ifStmt = new IfStatement();
ifStmt->id = IdGenerator::next("if");
applySpan(ifStmt, node);
ASTNode* cond = convertElispExpression(ts_node_named_child(node, 1), source);
if (cond) ifStmt->setChild("condition", cond);
return ifStmt;
}
// Generic: treat as function call
auto* call = new FunctionCall();
call->id = IdGenerator::next("call");
applySpan(call, node);
call->functionName = formName;
for (uint32_t i = 1; i < count; ++i) {
ASTNode* arg = convertElispExpression(ts_node_named_child(node, i), source);
if (arg) call->addChild("arguments", arg);
}
return call;
}
// ---------------------------------------------------------------

489
editor/src/ast/GoParser.h Normal file
View File

@@ -0,0 +1,489 @@
#pragma once
// TreeSitterParser Go support.
public:
// Go
// ---------------------------------------------------------------
static std::unique_ptr<Module> parseGo(const std::string& source) {
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, tree_sitter_go());
TSTree* tree = ts_parser_parse_string(parser, nullptr, source.c_str(), (uint32_t)source.size());
TSNode root = ts_tree_root_node(tree);
auto module = std::make_unique<Module>();
module->id = IdGenerator::next("mod");
module->name = "parsed_go_module";
module->targetLanguage = "go";
applySpan(module.get(), root);
convertGoSourceFile(root, source, module.get());
ts_tree_delete(tree);
ts_parser_delete(parser);
return module;
}
static ParseResult parseGoWithDiagnostics(const std::string& source) {
ParseResult result;
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, tree_sitter_go());
TSTree* tree = ts_parser_parse_string(parser, nullptr, source.c_str(), (uint32_t)source.size());
TSNode root = ts_tree_root_node(tree);
result.module = std::make_unique<Module>();
result.module->id = IdGenerator::next("mod");
result.module->name = "parsed_go_module";
result.module->targetLanguage = "go";
applySpan(result.module.get(), root);
convertGoSourceFile(root, source, result.module.get());
collectDiagnostics(root, source, result.diagnostics);
ts_tree_delete(tree);
ts_parser_delete(parser);
return result;
}
private:
// Go CST -> AST
// ---------------------------------------------------------------
static void convertGoSourceFile(TSNode root,
const std::string& source,
Module* module) {
uint32_t count = ts_node_named_child_count(root);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(root, i);
std::string type = nodeType(child);
if (type == "import_declaration") {
convertGoImport(child, source, module);
} else if (type == "function_declaration") {
auto* fn = convertGoFunction(child, source, "");
if (fn) module->addChild("functions", fn);
} else if (type == "method_declaration") {
auto* fn = convertGoMethod(child, source);
if (fn) module->addChild("functions", fn);
} else if (type == "var_declaration") {
convertGoVarDeclaration(child, source, module);
} else if (type == "type_declaration") {
convertGoTypeDeclaration(child, source, module);
}
}
}
static void convertGoImport(TSNode node,
const std::string& source,
Module* module) {
uint32_t count = ts_node_named_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
TSNode spec = ts_node_named_child(node, i);
if (nodeType(spec) != "import_spec") continue;
TSNode pathNode = childByFieldName(spec, "path");
std::string path = nodeText(pathNode, source);
if (!path.empty()) {
auto* imp = new Import(IdGenerator::next("imp"), path, "module");
module->addChild("imports", imp);
}
}
}
static void convertGoVarDeclaration(TSNode node,
const std::string& source,
Module* module) {
uint32_t count = ts_node_named_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
TSNode spec = ts_node_named_child(node, i);
if (nodeType(spec) != "var_spec") continue;
TSNode nameNode = childByFieldName(spec, "name");
TSNode typeNode = childByFieldName(spec, "type");
TSNode valueNode = childByFieldName(spec, "value");
if (ts_node_is_null(nameNode)) continue;
auto* var = new Variable(IdGenerator::next("var"), nodeText(nameNode, source));
applySpan(var, spec);
if (!ts_node_is_null(typeNode)) {
if (auto* t = convertGoType(typeNode, source)) var->setChild("type", t);
}
if (!ts_node_is_null(valueNode)) {
ASTNode* init = convertGoExpression(valueNode, source);
if (init) var->setChild("initializer", init);
}
module->addChild("variables", var);
}
}
static void convertGoTypeDeclaration(TSNode node,
const std::string& source,
Module* module) {
uint32_t count = ts_node_named_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
TSNode spec = ts_node_named_child(node, i);
if (nodeType(spec) != "type_spec") continue;
TSNode nameNode = childByFieldName(spec, "name");
if (ts_node_is_null(nameNode)) continue;
auto* var = new Variable(IdGenerator::next("var"), nodeText(nameNode, source));
module->addChild("variables", var);
}
}
static Function* convertGoFunction(TSNode node,
const std::string& source,
const std::string& receiverType) {
TSNode nameNode = childByFieldName(node, "name");
if (ts_node_is_null(nameNode)) return nullptr;
auto* fn = new Function();
fn->id = IdGenerator::next("fn");
applySpan(fn, node);
std::string name = nodeText(nameNode, source);
fn->name = receiverType.empty() ? name : (receiverType + "." + name);
TSNode paramsNode = childByFieldName(node, "parameters");
if (!ts_node_is_null(paramsNode)) {
convertGoParameters(paramsNode, source, fn);
}
TSNode resultNode = childByFieldName(node, "result");
if (!ts_node_is_null(resultNode)) {
if (auto* t = convertGoType(resultNode, source)) fn->setChild("returnType", t);
}
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
convertGoBlock(bodyNode, source, fn);
}
auto* reclaim = new ReclaimAnnotation(IdGenerator::next("anno"), "Escape");
fn->addChild("annotations", reclaim);
return fn;
}
static Function* convertGoMethod(TSNode node,
const std::string& source) {
TSNode recvNode = childByFieldName(node, "receiver");
std::string receiverType;
if (!ts_node_is_null(recvNode)) {
TSNode typeNode = findDescendantByField(recvNode, "type");
if (!ts_node_is_null(typeNode)) {
receiverType = nodeText(typeNode, source);
}
if (receiverType.empty()) {
TSNode nameNode = findDescendantByField(recvNode, "name");
if (!ts_node_is_null(nameNode)) {
receiverType = nodeText(nameNode, source);
}
}
receiverType = trimPointerPrefix(receiverType);
}
return convertGoFunction(node, source, receiverType);
}
static void convertGoParameters(TSNode paramsNode,
const std::string& source,
Function* fn) {
uint32_t count = ts_node_named_child_count(paramsNode);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(paramsNode, i);
if (nodeType(child) != "parameter_declaration" &&
nodeType(child) != "variadic_parameter_declaration") {
continue;
}
TSNode nameNode = childByFieldName(child, "name");
TSNode typeNode = childByFieldName(child, "type");
if (ts_node_is_null(nameNode)) continue;
auto* param = new Parameter(IdGenerator::next("param"), nodeText(nameNode, source));
applySpan(param, child);
if (!ts_node_is_null(typeNode)) {
if (auto* t = convertGoType(typeNode, source)) param->setChild("type", t);
}
fn->addChild("parameters", param);
}
}
static void convertGoBlock(TSNode blockNode,
const std::string& source,
Function* fn) {
uint32_t count = ts_node_named_child_count(blockNode);
for (uint32_t i = 0; i < count; ++i) {
ASTNode* stmt = convertGoStatement(ts_node_named_child(blockNode, i), source);
if (stmt) fn->addChild("body", stmt);
}
}
static ASTNode* convertGoStatement(TSNode node,
const std::string& source) {
std::string type = nodeType(node);
if (type == "return_statement") {
auto* ret = new Return();
ret->id = IdGenerator::next("ret");
applySpan(ret, node);
TSNode valueNode = childByFieldName(node, "value");
if (!ts_node_is_null(valueNode)) {
ASTNode* val = convertGoExpression(valueNode, source);
if (val) ret->setChild("value", val);
}
return ret;
} else if (type == "short_var_declaration") {
auto* assign = new Assignment();
assign->id = IdGenerator::next("assign");
applySpan(assign, node);
TSNode leftNode = childByFieldName(node, "left");
TSNode rightNode = childByFieldName(node, "right");
if (!ts_node_is_null(leftNode)) {
ASTNode* target = convertGoExpression(leftNode, source);
if (target) assign->setChild("target", target);
}
if (!ts_node_is_null(rightNode)) {
ASTNode* value = convertGoExpression(rightNode, source);
if (value) assign->setChild("value", value);
}
return assign;
} else if (type == "assignment_statement") {
auto* assign = new Assignment();
assign->id = IdGenerator::next("assign");
applySpan(assign, node);
TSNode leftNode = childByFieldName(node, "left");
TSNode rightNode = childByFieldName(node, "right");
if (!ts_node_is_null(leftNode)) {
ASTNode* target = convertGoExpression(leftNode, source);
if (target) assign->setChild("target", target);
}
if (!ts_node_is_null(rightNode)) {
ASTNode* value = convertGoExpression(rightNode, source);
if (value) assign->setChild("value", value);
}
return assign;
} else if (type == "expression_statement") {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, node);
if (ts_node_named_child_count(node) > 0) {
ASTNode* expr = convertGoExpression(ts_node_named_child(node, 0), source);
if (expr) exprStmt->setChild("expression", expr);
}
return exprStmt;
} else if (type == "if_statement") {
auto* ifStmt = new IfStatement();
ifStmt->id = IdGenerator::next("if");
applySpan(ifStmt, node);
TSNode condNode = childByFieldName(node, "condition");
if (!ts_node_is_null(condNode)) {
ASTNode* cond = convertGoExpression(condNode, source);
if (cond) ifStmt->setChild("condition", cond);
}
TSNode consNode = childByFieldName(node, "consequence");
if (!ts_node_is_null(consNode)) {
uint32_t cc = ts_node_named_child_count(consNode);
for (uint32_t i = 0; i < cc; ++i) {
ASTNode* s = convertGoStatement(ts_node_named_child(consNode, i), source);
if (s) ifStmt->addChild("thenBranch", s);
}
}
TSNode altNode = childByFieldName(node, "alternative");
if (!ts_node_is_null(altNode)) {
uint32_t ac = ts_node_named_child_count(altNode);
for (uint32_t i = 0; i < ac; ++i) {
ASTNode* s = convertGoStatement(ts_node_named_child(altNode, i), source);
if (s) ifStmt->addChild("elseBranch", s);
}
}
return ifStmt;
} else if (type == "for_statement") {
auto* loop = new ForLoop();
loop->id = IdGenerator::next("for");
applySpan(loop, node);
TSNode rangeNode = childByFieldName(node, "right");
if (!ts_node_is_null(rangeNode)) {
ASTNode* iter = convertGoExpression(rangeNode, source);
if (iter) loop->setChild("iterable", iter);
}
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
uint32_t bc = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < bc; ++i) {
ASTNode* s = convertGoStatement(ts_node_named_child(bodyNode, i), source);
if (s) loop->addChild("body", s);
}
}
return loop;
} else if (type == "block") {
auto* block = new Block();
block->id = IdGenerator::next("block");
applySpan(block, node);
uint32_t bc = ts_node_named_child_count(node);
for (uint32_t i = 0; i < bc; ++i) {
ASTNode* s = convertGoStatement(ts_node_named_child(node, i), source);
if (s) block->addChild("statements", s);
}
return block;
}
ASTNode* expr = convertGoExpression(node, source);
if (expr) {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, node);
exprStmt->setChild("expression", expr);
return exprStmt;
}
return nullptr;
}
static ASTNode* convertGoExpression(TSNode node,
const std::string& source) {
std::string type = nodeType(node);
if (type == "binary_expression") {
auto* binOp = new BinaryOperation();
binOp->id = IdGenerator::next("binop");
applySpan(binOp, node);
TSNode leftNode = childByFieldName(node, "left");
TSNode rightNode = childByFieldName(node, "right");
TSNode opNode = childByFieldName(node, "operator");
if (!ts_node_is_null(opNode)) binOp->op = nodeText(opNode, source);
if (!ts_node_is_null(leftNode)) {
ASTNode* left = convertGoExpression(leftNode, source);
if (left) binOp->setChild("left", left);
}
if (!ts_node_is_null(rightNode)) {
ASTNode* right = convertGoExpression(rightNode, source);
if (right) binOp->setChild("right", right);
}
return binOp;
} else if (type == "call_expression") {
auto* call = new FunctionCall();
call->id = IdGenerator::next("call");
applySpan(call, node);
TSNode funcNode = childByFieldName(node, "function");
if (!ts_node_is_null(funcNode)) {
call->functionName = nodeText(funcNode, source);
}
TSNode argsNode = childByFieldName(node, "arguments");
if (!ts_node_is_null(argsNode)) {
uint32_t count = ts_node_named_child_count(argsNode);
for (uint32_t i = 0; i < count; ++i) {
ASTNode* arg = convertGoExpression(ts_node_named_child(argsNode, i), source);
if (arg) call->addChild("arguments", arg);
}
}
return call;
} else if (type == "selector_expression") {
auto* mem = new MemberAccess();
mem->id = IdGenerator::next("member");
applySpan(mem, node);
TSNode operandNode = childByFieldName(node, "operand");
TSNode fieldNode = childByFieldName(node, "field");
if (!ts_node_is_null(fieldNode)) {
mem->memberName = nodeText(fieldNode, source);
}
if (!ts_node_is_null(operandNode)) {
ASTNode* target = convertGoExpression(operandNode, source);
if (target) mem->setChild("target", target);
}
return mem;
} else if (type == "index_expression") {
auto* access = new IndexAccess();
access->id = IdGenerator::next("index");
applySpan(access, node);
TSNode operandNode = childByFieldName(node, "operand");
TSNode indexNode = childByFieldName(node, "index");
if (!ts_node_is_null(operandNode)) {
ASTNode* target = convertGoExpression(operandNode, source);
if (target) access->setChild("target", target);
}
if (!ts_node_is_null(indexNode)) {
ASTNode* idx = convertGoExpression(indexNode, source);
if (idx) access->setChild("index", idx);
}
return access;
} else if (type == "identifier") {
auto* ref = new VariableReference(IdGenerator::next("var"), nodeText(node, source));
applySpan(ref, node);
return ref;
} else if (type == "int_literal") {
std::string text = nodeText(node, source);
int val = 0;
try { val = std::stoi(text); } catch (...) {}
auto* lit = new IntegerLiteral(IdGenerator::next("int"), val);
applySpan(lit, node);
return lit;
} else if (type == "float_literal") {
auto* lit = new FloatLiteral(IdGenerator::next("float"), nodeText(node, source));
applySpan(lit, node);
return lit;
} else if (type == "raw_string_literal" || type == "interpreted_string_literal") {
auto* lit = new StringLiteral(IdGenerator::next("str"), nodeText(node, source));
applySpan(lit, node);
return lit;
} else if (type == "true" || type == "false") {
auto* lit = new BooleanLiteral(IdGenerator::next("bool"), type == "true");
applySpan(lit, node);
return lit;
} else if (type == "nil") {
auto* lit = new NullLiteral();
lit->id = IdGenerator::next("null");
applySpan(lit, node);
return lit;
} else if (type == "parenthesized_expression") {
if (ts_node_named_child_count(node) > 0) {
return convertGoExpression(ts_node_named_child(node, 0), source);
}
}
std::string text = nodeText(node, source);
if (!text.empty()) {
auto* ref = new VariableReference(IdGenerator::next("var"), text);
applySpan(ref, node);
return ref;
}
return nullptr;
}
static Type* convertGoType(TSNode node, const std::string& source) {
std::string type = nodeType(node);
if (type == "qualified_type") {
auto* custom = new CustomType();
custom->id = IdGenerator::next("type");
custom->typeName = nodeText(node, source);
return custom;
} else if (type == "pointer_type") {
TSNode elemNode = childByFieldName(node, "type");
if (!ts_node_is_null(elemNode)) {
if (auto* inner = convertGoType(elemNode, source)) {
auto* opt = new OptionalType();
opt->id = IdGenerator::next("type");
opt->setChild("innerType", inner);
return opt;
}
}
} else if (type == "array_type" || type == "slice_type") {
auto* arr = new ArrayType();
arr->id = IdGenerator::next("type");
TSNode elemNode = childByFieldName(node, "element");
if (!ts_node_is_null(elemNode)) {
if (auto* et = convertGoType(elemNode, source)) arr->setChild("elementType", et);
}
return arr;
} else if (type == "map_type") {
auto* map = new MapType();
map->id = IdGenerator::next("type");
TSNode keyNode = childByFieldName(node, "key");
TSNode valNode = childByFieldName(node, "value");
if (!ts_node_is_null(keyNode)) {
if (auto* kt = convertGoType(keyNode, source)) map->setChild("keyType", kt);
}
if (!ts_node_is_null(valNode)) {
if (auto* vt = convertGoType(valNode, source)) map->setChild("valueType", vt);
}
return map;
} else if (type == "struct_type") {
auto* custom = new CustomType();
custom->id = IdGenerator::next("type");
custom->typeName = "struct";
return custom;
} else if (type == "interface_type") {
auto* custom = new CustomType();
custom->id = IdGenerator::next("type");
custom->typeName = "interface{}";
return custom;
}
auto* custom = new CustomType();
custom->id = IdGenerator::next("type");
custom->typeName = nodeText(node, source);
return custom;
}
// ---------------------------------------------------------------

577
editor/src/ast/JavaParser.h Normal file
View File

@@ -0,0 +1,577 @@
#pragma once
// TreeSitterParser Java support.
public:
// Java
// ---------------------------------------------------------------
static std::unique_ptr<Module> parseJava(const std::string& source) {
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, tree_sitter_java());
TSTree* tree = ts_parser_parse_string(parser, nullptr, source.c_str(), (uint32_t)source.size());
TSNode root = ts_tree_root_node(tree);
auto module = std::make_unique<Module>();
module->id = IdGenerator::next("mod");
module->name = "parsed_java_module";
module->targetLanguage = "java";
applySpan(module.get(), root);
convertJavaCompilationUnit(root, source, module.get());
ts_tree_delete(tree);
ts_parser_delete(parser);
return module;
}
static ParseResult parseJavaWithDiagnostics(const std::string& source) {
ParseResult result;
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, tree_sitter_java());
TSTree* tree = ts_parser_parse_string(parser, nullptr, source.c_str(), (uint32_t)source.size());
TSNode root = ts_tree_root_node(tree);
result.module = std::make_unique<Module>();
result.module->id = IdGenerator::next("mod");
result.module->name = "parsed_java_module";
result.module->targetLanguage = "java";
applySpan(result.module.get(), root);
convertJavaCompilationUnit(root, source, result.module.get());
collectDiagnostics(root, source, result.diagnostics);
ts_tree_delete(tree);
ts_parser_delete(parser);
return result;
}
// ---------------------------------------------------------------
private:
// Java CST -> AST
// ---------------------------------------------------------------
static void convertJavaCompilationUnit(TSNode root,
const std::string& source,
Module* module) {
uint32_t count = ts_node_named_child_count(root);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(root, i);
std::string type = nodeType(child);
if (type == "import_declaration") {
std::string importName;
uint32_t ic = ts_node_named_child_count(child);
for (uint32_t j = 0; j < ic; ++j) {
TSNode part = ts_node_named_child(child, j);
std::string pType = nodeType(part);
if (pType == "scoped_identifier" || pType == "identifier") {
importName = nodeText(part, source);
break;
}
}
if (importName.empty()) {
importName = nodeText(child, source);
}
if (!importName.empty()) {
auto* imp = new Import(IdGenerator::next("imp"), importName, "module");
module->addChild("imports", imp);
}
} else if (type == "class_declaration" || type == "interface_declaration" ||
type == "enum_declaration" || type == "record_declaration" ||
type == "annotation_type_declaration") {
convertJavaTypeDeclaration(child, source, module);
}
}
}
static void convertJavaTypeDeclaration(TSNode node,
const std::string& source,
Module* module) {
TSNode nameNode = childByFieldName(node, "name");
std::string className = nodeText(nameNode, source);
TSNode bodyNode = childByFieldName(node, "body");
if (ts_node_is_null(bodyNode)) return;
uint32_t count = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(bodyNode, i);
std::string type = nodeType(child);
if (type == "method_declaration") {
auto* fn = convertJavaMethod(child, source, className);
if (fn) module->addChild("functions", fn);
} else if (type == "constructor_declaration" || type == "compact_constructor_declaration") {
auto* fn = convertJavaConstructor(child, source, className);
if (fn) module->addChild("functions", fn);
} else if (type == "field_declaration") {
convertJavaFieldDeclaration(child, source, module, className);
} else if (type == "class_declaration" || type == "interface_declaration" ||
type == "enum_declaration" || type == "record_declaration" ||
type == "annotation_type_declaration") {
convertJavaTypeDeclaration(child, source, module);
}
}
}
static void convertJavaFieldDeclaration(TSNode node,
const std::string& source,
Module* module,
const std::string& className) {
TSNode typeNode = childByFieldName(node, "type");
uint32_t count = ts_node_named_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(node, i);
if (nodeType(child) != "variable_declarator") continue;
TSNode nameNode = childByFieldName(child, "name");
if (ts_node_is_null(nameNode)) continue;
std::string varName = nodeText(nameNode, source);
if (!className.empty()) varName = className + "." + varName;
auto* var = new Variable(IdGenerator::next("var"), varName);
applySpan(var, child);
if (!ts_node_is_null(typeNode)) {
if (auto* t = convertJavaType(typeNode, source)) var->setChild("type", t);
}
TSNode valueNode = childByFieldName(child, "value");
if (!ts_node_is_null(valueNode)) {
ASTNode* init = convertJavaExpression(valueNode, source);
if (init) var->setChild("initializer", init);
}
module->addChild("variables", var);
}
}
static Function* convertJavaMethod(TSNode node,
const std::string& source,
const std::string& className) {
TSNode nameNode = childByFieldName(node, "name");
if (ts_node_is_null(nameNode)) return nullptr;
auto* fn = new Function();
fn->id = IdGenerator::next("fn");
applySpan(fn, node);
std::string methodName = nodeText(nameNode, source);
fn->name = className.empty() ? methodName : (className + "." + methodName);
TSNode typeNode = childByFieldName(node, "type");
if (!ts_node_is_null(typeNode)) {
if (auto* t = convertJavaType(typeNode, source)) fn->setChild("returnType", t);
}
TSNode paramsNode = childByFieldName(node, "parameters");
if (!ts_node_is_null(paramsNode)) {
convertJavaParameters(paramsNode, source, fn);
}
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
convertJavaBlockStatements(bodyNode, source, fn);
}
attachJavaAnnotations(node, source, fn);
auto* reclaim = new ReclaimAnnotation(IdGenerator::next("anno"), "Tracing");
fn->addChild("annotations", reclaim);
return fn;
}
static Function* convertJavaConstructor(TSNode node,
const std::string& source,
const std::string& className) {
if (className.empty()) return nullptr;
auto* fn = new Function();
fn->id = IdGenerator::next("fn");
applySpan(fn, node);
fn->name = className + ".constructor";
TSNode paramsNode = childByFieldName(node, "parameters");
if (!ts_node_is_null(paramsNode)) {
convertJavaParameters(paramsNode, source, fn);
}
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
convertJavaBlockStatements(bodyNode, source, fn);
}
attachJavaAnnotations(node, source, fn);
auto* reclaim = new ReclaimAnnotation(IdGenerator::next("anno"), "Tracing");
fn->addChild("annotations", reclaim);
return fn;
}
static void convertJavaParameters(TSNode paramsNode,
const std::string& source,
Function* fn) {
uint32_t count = ts_node_named_child_count(paramsNode);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(paramsNode, i);
std::string type = nodeType(child);
if (type == "formal_parameter" || type == "spread_parameter") {
TSNode nameNode = childByFieldName(child, "name");
if (ts_node_is_null(nameNode)) {
nameNode = findDescendantByType(child, "_variable_declarator_id");
if (!ts_node_is_null(nameNode)) {
TSNode innerName = childByFieldName(nameNode, "name");
if (!ts_node_is_null(innerName)) nameNode = innerName;
}
}
if (ts_node_is_null(nameNode)) continue;
std::string name = nodeText(nameNode, source);
if (type == "spread_parameter") name = "..." + name;
auto* param = new Parameter(IdGenerator::next("param"), name);
applySpan(param, child);
TSNode typeNode = childByFieldName(child, "type");
if (!ts_node_is_null(typeNode)) {
if (auto* t = convertJavaType(typeNode, source)) param->setChild("type", t);
}
fn->addChild("parameters", param);
}
}
}
static void convertJavaBlockStatements(TSNode bodyNode,
const std::string& source,
Function* fn) {
uint32_t count = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < count; ++i) {
ASTNode* stmt = convertJavaStatement(ts_node_named_child(bodyNode, i), source);
if (stmt) fn->addChild("body", stmt);
}
}
static ASTNode* convertJavaStatement(TSNode node,
const std::string& source) {
std::string type = nodeType(node);
if (type == "return_statement") {
auto* ret = new Return();
ret->id = IdGenerator::next("ret");
applySpan(ret, node);
if (ts_node_named_child_count(node) > 0) {
ASTNode* val = convertJavaExpression(ts_node_named_child(node, 0), source);
if (val) ret->setChild("value", val);
}
return ret;
} else if (type == "expression_statement") {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, node);
if (ts_node_named_child_count(node) > 0) {
ASTNode* expr = convertJavaExpression(ts_node_named_child(node, 0), source);
if (expr) exprStmt->setChild("expression", expr);
}
return exprStmt;
} else if (type == "local_variable_declaration") {
uint32_t count = ts_node_named_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(node, i);
if (nodeType(child) != "variable_declarator") continue;
TSNode nameNode = childByFieldName(child, "name");
if (ts_node_is_null(nameNode)) continue;
auto* assign = new Assignment();
assign->id = IdGenerator::next("assign");
applySpan(assign, child);
auto* target = new VariableReference(IdGenerator::next("var"), nodeText(nameNode, source));
applySpan(target, nameNode);
assign->setChild("target", target);
TSNode valueNode = childByFieldName(child, "value");
if (!ts_node_is_null(valueNode)) {
ASTNode* val = convertJavaExpression(valueNode, source);
if (val) assign->setChild("value", val);
}
return assign;
}
} else if (type == "if_statement") {
auto* ifStmt = new IfStatement();
ifStmt->id = IdGenerator::next("if");
applySpan(ifStmt, node);
TSNode condNode = childByFieldName(node, "condition");
if (!ts_node_is_null(condNode)) {
ASTNode* cond = convertJavaExpression(condNode, source);
if (cond) ifStmt->setChild("condition", cond);
}
TSNode consNode = childByFieldName(node, "consequence");
if (!ts_node_is_null(consNode)) {
uint32_t cc = ts_node_named_child_count(consNode);
for (uint32_t i = 0; i < cc; ++i) {
ASTNode* s = convertJavaStatement(ts_node_named_child(consNode, i), source);
if (s) ifStmt->addChild("thenBranch", s);
}
}
TSNode altNode = childByFieldName(node, "alternative");
if (!ts_node_is_null(altNode)) {
uint32_t ac = ts_node_named_child_count(altNode);
for (uint32_t i = 0; i < ac; ++i) {
ASTNode* s = convertJavaStatement(ts_node_named_child(altNode, i), source);
if (s) ifStmt->addChild("elseBranch", s);
}
}
return ifStmt;
} else if (type == "while_statement") {
auto* loop = new WhileLoop();
loop->id = IdGenerator::next("while");
applySpan(loop, node);
TSNode condNode = childByFieldName(node, "condition");
if (!ts_node_is_null(condNode)) {
ASTNode* cond = convertJavaExpression(condNode, source);
if (cond) loop->setChild("condition", cond);
}
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
uint32_t bc = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < bc; ++i) {
ASTNode* s = convertJavaStatement(ts_node_named_child(bodyNode, i), source);
if (s) loop->addChild("body", s);
}
}
return loop;
} else if (type == "for_statement") {
auto* loop = new WhileLoop();
loop->id = IdGenerator::next("for");
applySpan(loop, node);
TSNode condNode = childByFieldName(node, "condition");
if (!ts_node_is_null(condNode)) {
ASTNode* cond = convertJavaExpression(condNode, source);
if (cond) loop->setChild("condition", cond);
}
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
uint32_t bc = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < bc; ++i) {
ASTNode* s = convertJavaStatement(ts_node_named_child(bodyNode, i), source);
if (s) loop->addChild("body", s);
}
}
return loop;
} else if (type == "enhanced_for_statement") {
auto* loop = new ForLoop();
loop->id = IdGenerator::next("for");
applySpan(loop, node);
TSNode nameNode = childByFieldName(node, "name");
if (!ts_node_is_null(nameNode)) {
loop->iteratorName = nodeText(nameNode, source);
}
TSNode valueNode = childByFieldName(node, "value");
if (!ts_node_is_null(valueNode)) {
ASTNode* iter = convertJavaExpression(valueNode, source);
if (iter) loop->setChild("iterable", iter);
}
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
uint32_t bc = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < bc; ++i) {
ASTNode* s = convertJavaStatement(ts_node_named_child(bodyNode, i), source);
if (s) loop->addChild("body", s);
}
}
return loop;
} else if (type == "block") {
auto* block = new Block();
block->id = IdGenerator::next("block");
applySpan(block, node);
uint32_t bc = ts_node_named_child_count(node);
for (uint32_t i = 0; i < bc; ++i) {
ASTNode* s = convertJavaStatement(ts_node_named_child(node, i), source);
if (s) block->addChild("statements", s);
}
return block;
} else if (type == "try_statement") {
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
auto* block = new Block();
block->id = IdGenerator::next("block");
applySpan(block, bodyNode);
uint32_t bc = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < bc; ++i) {
ASTNode* s = convertJavaStatement(ts_node_named_child(bodyNode, i), source);
if (s) block->addChild("statements", s);
}
return block;
}
}
ASTNode* expr = convertJavaExpression(node, source);
if (expr) {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, node);
exprStmt->setChild("expression", expr);
return exprStmt;
}
return nullptr;
}
static ASTNode* convertJavaExpression(TSNode node,
const std::string& source) {
std::string type = nodeType(node);
if (type == "binary_expression") {
auto* binOp = new BinaryOperation();
binOp->id = IdGenerator::next("binop");
applySpan(binOp, node);
TSNode leftNode = childByFieldName(node, "left");
TSNode rightNode = childByFieldName(node, "right");
TSNode opNode = childByFieldName(node, "operator");
if (!ts_node_is_null(opNode)) binOp->op = nodeText(opNode, source);
if (!ts_node_is_null(leftNode)) {
ASTNode* left = convertJavaExpression(leftNode, source);
if (left) binOp->setChild("left", left);
}
if (!ts_node_is_null(rightNode)) {
ASTNode* right = convertJavaExpression(rightNode, source);
if (right) binOp->setChild("right", right);
}
return binOp;
} else if (type == "assignment_expression") {
auto* assign = new Assignment();
assign->id = IdGenerator::next("assign");
applySpan(assign, node);
TSNode leftNode = childByFieldName(node, "left");
TSNode rightNode = childByFieldName(node, "right");
if (!ts_node_is_null(leftNode)) {
ASTNode* target = convertJavaExpression(leftNode, source);
if (target) assign->setChild("target", target);
}
if (!ts_node_is_null(rightNode)) {
ASTNode* value = convertJavaExpression(rightNode, source);
if (value) assign->setChild("value", value);
}
return assign;
} else if (type == "method_invocation") {
auto* call = new FunctionCall();
call->id = IdGenerator::next("call");
applySpan(call, node);
TSNode nameNode = childByFieldName(node, "name");
TSNode objectNode = childByFieldName(node, "object");
if (!ts_node_is_null(objectNode) && !ts_node_is_null(nameNode)) {
call->functionName = nodeText(objectNode, source) + "." + nodeText(nameNode, source);
} else if (!ts_node_is_null(nameNode)) {
call->functionName = nodeText(nameNode, source);
} else {
call->functionName = nodeText(node, source);
}
TSNode argsNode = childByFieldName(node, "arguments");
if (!ts_node_is_null(argsNode)) {
uint32_t count = ts_node_named_child_count(argsNode);
for (uint32_t i = 0; i < count; ++i) {
ASTNode* arg = convertJavaExpression(ts_node_named_child(argsNode, i), source);
if (arg) call->addChild("arguments", arg);
}
}
return call;
} else if (type == "field_access") {
auto* mem = new MemberAccess();
mem->id = IdGenerator::next("member");
applySpan(mem, node);
TSNode objNode = childByFieldName(node, "object");
TSNode fieldNode = childByFieldName(node, "field");
if (!ts_node_is_null(fieldNode)) {
mem->memberName = nodeText(fieldNode, source);
}
if (!ts_node_is_null(objNode)) {
ASTNode* target = convertJavaExpression(objNode, source);
if (target) mem->setChild("target", target);
}
return mem;
} else if (type == "array_access") {
auto* access = new IndexAccess();
access->id = IdGenerator::next("index");
applySpan(access, node);
TSNode arrayNode = childByFieldName(node, "array");
TSNode indexNode = childByFieldName(node, "index");
if (!ts_node_is_null(arrayNode)) {
ASTNode* target = convertJavaExpression(arrayNode, source);
if (target) access->setChild("target", target);
}
if (!ts_node_is_null(indexNode)) {
ASTNode* idx = convertJavaExpression(indexNode, source);
if (idx) access->setChild("index", idx);
}
return access;
} else if (type == "identifier") {
auto* ref = new VariableReference(IdGenerator::next("var"), nodeText(node, source));
applySpan(ref, node);
return ref;
} else if (type == "integer_literal") {
std::string text = nodeText(node, source);
int val = 0;
try { val = std::stoi(text); } catch (...) {}
auto* lit = new IntegerLiteral(IdGenerator::next("int"), val);
applySpan(lit, node);
return lit;
} else if (type == "floating_point_literal") {
auto* lit = new FloatLiteral(IdGenerator::next("float"), nodeText(node, source));
applySpan(lit, node);
return lit;
} else if (type == "string_literal" || type == "character_literal") {
auto* lit = new StringLiteral(IdGenerator::next("str"), nodeText(node, source));
applySpan(lit, node);
return lit;
} else if (type == "true" || type == "false") {
auto* lit = new BooleanLiteral(IdGenerator::next("bool"), type == "true");
applySpan(lit, node);
return lit;
} else if (type == "null_literal") {
auto* lit = new NullLiteral();
lit->id = IdGenerator::next("null");
applySpan(lit, node);
return lit;
} else if (type == "parenthesized_expression") {
if (ts_node_named_child_count(node) > 0) {
return convertJavaExpression(ts_node_named_child(node, 0), source);
}
} else if (type == "lambda_expression") {
auto* ref = new VariableReference(IdGenerator::next("var"), nodeText(node, source));
applySpan(ref, node);
return ref;
}
std::string text = nodeText(node, source);
if (!text.empty()) {
auto* ref = new VariableReference(IdGenerator::next("var"), text);
applySpan(ref, node);
return ref;
}
return nullptr;
}
static Type* convertJavaType(TSNode node, const std::string& source) {
std::string type = nodeType(node);
if (type == "integral_type" || type == "floating_point_type" ||
type == "boolean_type" || type == "void_type") {
auto* prim = new PrimitiveType();
prim->id = IdGenerator::next("type");
prim->kind = nodeText(node, source);
return prim;
} else if (type == "array_type") {
auto* arr = new ArrayType();
arr->id = IdGenerator::next("type");
TSNode element = childByFieldName(node, "element");
if (!ts_node_is_null(element)) {
if (auto* et = convertJavaType(element, source)) arr->setChild("elementType", et);
}
return arr;
}
auto* custom = new CustomType();
custom->id = IdGenerator::next("type");
custom->typeName = nodeText(node, source);
return custom;
}
static void attachJavaAnnotations(TSNode node,
const std::string& source,
ASTNode* target) {
if (!target) return;
uint32_t count = ts_node_named_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(node, i);
if (nodeType(child) != "modifiers") continue;
uint32_t mc = ts_node_named_child_count(child);
for (uint32_t j = 0; j < mc; ++j) {
TSNode modChild = ts_node_named_child(child, j);
std::string mType = nodeType(modChild);
if (mType == "marker_annotation" || mType == "annotation") {
auto* anno = new LangSpecific();
anno->id = IdGenerator::next("anno");
anno->language = "java";
TSNode nameNode = childByFieldName(modChild, "name");
anno->idiomType = ts_node_is_null(nameNode) ? "" : nodeText(nameNode, source);
anno->rawSyntax = nodeText(modChild, source);
target->addChild("annotations", anno);
}
}
}
}
// ---------------------------------------------------------------

View File

@@ -0,0 +1,476 @@
#pragma once
// TreeSitterParser JavaScript support.
public:
// JavaScript
// ---------------------------------------------------------------
static std::unique_ptr<Module> parseJavaScript(const std::string& source) {
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, tree_sitter_javascript());
TSTree* tree = ts_parser_parse_string(parser, nullptr, source.c_str(), (uint32_t)source.size());
TSNode root = ts_tree_root_node(tree);
auto module = std::make_unique<Module>();
module->id = IdGenerator::next("mod");
module->name = "parsed_js_module";
module->targetLanguage = "javascript";
applySpan(module.get(), root);
convertJavaScriptModule(root, source, module.get(), "javascript");
ts_tree_delete(tree);
ts_parser_delete(parser);
return module;
}
static ParseResult parseJavaScriptWithDiagnostics(const std::string& source) {
ParseResult result;
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, tree_sitter_javascript());
TSTree* tree = ts_parser_parse_string(parser, nullptr, source.c_str(), (uint32_t)source.size());
TSNode root = ts_tree_root_node(tree);
result.module = std::make_unique<Module>();
result.module->id = IdGenerator::next("mod");
result.module->name = "parsed_js_module";
result.module->targetLanguage = "javascript";
applySpan(result.module.get(), root);
convertJavaScriptModule(root, source, result.module.get(), "javascript");
collectDiagnostics(root, source, result.diagnostics);
ts_tree_delete(tree);
ts_parser_delete(parser);
return result;
}
// ---------------------------------------------------------------
private:
// JavaScript / TypeScript CST -> AST
// ---------------------------------------------------------------
static void convertJavaScriptModule(TSNode root,
const std::string& source,
Module* module,
const std::string& language) {
uint32_t count = ts_node_named_child_count(root);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(root, i);
std::string type = nodeType(child);
if (type == "function_declaration") {
auto* fn = convertJavaScriptFunction(child, source, language);
if (fn) module->addChild("functions", fn);
} else if (type == "class_declaration") {
convertJavaScriptClass(child, source, module, language);
} else if (type == "lexical_declaration" || type == "variable_declaration") {
convertJavaScriptVariableFunctions(child, source, module, language);
} else if (type == "export_statement") {
TSNode decl = ts_node_named_child(child, 0);
std::string declType = nodeType(decl);
if (declType == "function_declaration") {
auto* fn = convertJavaScriptFunction(decl, source, language);
if (fn) module->addChild("functions", fn);
} else if (declType == "class_declaration") {
convertJavaScriptClass(decl, source, module, language);
} else if (declType == "lexical_declaration" || declType == "variable_declaration") {
convertJavaScriptVariableFunctions(decl, source, module, language);
}
}
}
}
static void convertJavaScriptClass(TSNode node,
const std::string& source,
Module* module,
const std::string& language) {
TSNode nameNode = childByFieldName(node, "name");
std::string className = nodeText(nameNode, source);
TSNode bodyNode = childByFieldName(node, "body");
if (ts_node_is_null(bodyNode)) return;
uint32_t count = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(bodyNode, i);
std::string type = nodeType(child);
if (type == "method_definition") {
auto* fn = convertJavaScriptMethod(child, source, language, className);
if (fn) module->addChild("functions", fn);
}
}
}
static void convertJavaScriptVariableFunctions(TSNode node,
const std::string& source,
Module* module,
const std::string& language) {
uint32_t count = ts_node_named_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(node, i);
if (nodeType(child) != "variable_declarator") continue;
TSNode nameNode = childByFieldName(child, "name");
TSNode valueNode = childByFieldName(child, "value");
if (ts_node_is_null(nameNode) || ts_node_is_null(valueNode)) continue;
std::string valueType = nodeType(valueNode);
if (valueType == "arrow_function" || valueType == "function" ||
valueType == "function_expression") {
auto* fn = convertJavaScriptFunctionExpression(valueNode, source,
language, nodeText(nameNode, source));
if (fn) module->addChild("functions", fn);
}
}
}
static Function* convertJavaScriptFunction(TSNode node,
const std::string& source,
const std::string& language) {
TSNode nameNode = childByFieldName(node, "name");
if (ts_node_is_null(nameNode)) return nullptr;
auto* fn = new Function();
fn->id = IdGenerator::next("fn");
applySpan(fn, node);
fn->name = nodeText(nameNode, source);
TSNode paramsNode = childByFieldName(node, "parameters");
if (!ts_node_is_null(paramsNode)) {
convertJavaScriptParameters(paramsNode, source, fn, language);
}
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
convertJavaScriptBody(bodyNode, source, fn, language);
}
auto* reclaim = new ReclaimAnnotation(IdGenerator::next("anno"), "Tracing");
fn->addChild("annotations", reclaim);
return fn;
}
static Function* convertJavaScriptMethod(TSNode node,
const std::string& source,
const std::string& language,
const std::string& className) {
TSNode nameNode = childByFieldName(node, "name");
if (ts_node_is_null(nameNode)) return nullptr;
auto* fn = new Function();
fn->id = IdGenerator::next("fn");
applySpan(fn, node);
std::string name = nodeText(nameNode, source);
fn->name = className.empty() ? name : (className + "." + name);
TSNode paramsNode = childByFieldName(node, "parameters");
if (!ts_node_is_null(paramsNode)) {
convertJavaScriptParameters(paramsNode, source, fn, language);
}
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
convertJavaScriptBody(bodyNode, source, fn, language);
}
auto* reclaim = new ReclaimAnnotation(IdGenerator::next("anno"), "Tracing");
fn->addChild("annotations", reclaim);
return fn;
}
static Function* convertJavaScriptFunctionExpression(TSNode node,
const std::string& source,
const std::string& language,
const std::string& nameOverride) {
auto* fn = new Function();
fn->id = IdGenerator::next("fn");
applySpan(fn, node);
fn->name = nameOverride;
TSNode paramsNode = childByFieldName(node, "parameters");
if (!ts_node_is_null(paramsNode)) {
convertJavaScriptParameters(paramsNode, source, fn, language);
} else if (nodeType(node) == "arrow_function") {
TSNode paramNode = childByFieldName(node, "parameter");
if (!ts_node_is_null(paramNode)) {
auto* param = new Parameter(IdGenerator::next("param"), nodeText(paramNode, source));
applySpan(param, paramNode);
fn->addChild("parameters", param);
}
}
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
convertJavaScriptBody(bodyNode, source, fn, language);
}
auto* reclaim = new ReclaimAnnotation(IdGenerator::next("anno"), "Tracing");
fn->addChild("annotations", reclaim);
return fn;
}
static void convertJavaScriptParameters(TSNode paramsNode,
const std::string& source,
Function* fn,
const std::string& language) {
uint32_t count = ts_node_named_child_count(paramsNode);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(paramsNode, i);
std::string type = nodeType(child);
if (type == "identifier" || type == "pattern" || type == "rest_pattern") {
auto* param = new Parameter(IdGenerator::next("param"), nodeText(child, source));
applySpan(param, child);
fn->addChild("parameters", param);
} else if (type == "required_parameter" || type == "optional_parameter" ||
type == "formal_parameter") {
TSNode nameNode = childByFieldName(child, "pattern");
if (ts_node_is_null(nameNode)) nameNode = childByFieldName(child, "name");
if (!ts_node_is_null(nameNode)) {
auto* param = new Parameter(IdGenerator::next("param"), nodeText(nameNode, source));
applySpan(param, child);
TSNode typeNode = childByFieldName(child, "type");
if (ts_node_is_null(typeNode)) typeNode = childByFieldName(child, "type_annotation");
if (!ts_node_is_null(typeNode) && language == "typescript") {
auto* typeAnno = new CustomType();
typeAnno->id = IdGenerator::next("type");
typeAnno->typeName = nodeText(typeNode, source);
param->setChild("type", typeAnno);
}
fn->addChild("parameters", param);
}
} else if (type == "assignment_pattern") {
TSNode left = childByFieldName(child, "left");
TSNode right = childByFieldName(child, "right");
if (!ts_node_is_null(left)) {
auto* param = new Parameter(IdGenerator::next("param"), nodeText(left, source));
applySpan(param, child);
if (!ts_node_is_null(right)) {
ASTNode* defVal = convertJavaScriptExpression(right, source, language);
if (defVal) param->setChild("defaultValue", defVal);
}
fn->addChild("parameters", param);
}
}
}
}
static void convertJavaScriptBody(TSNode bodyNode,
const std::string& source,
Function* fn,
const std::string& language) {
std::string type = nodeType(bodyNode);
if (type == "statement_block" || type == "statement_block") {
uint32_t count = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < count; ++i) {
ASTNode* stmt = convertJavaScriptStatement(ts_node_named_child(bodyNode, i), source, language);
if (stmt) fn->addChild("body", stmt);
}
} else {
ASTNode* expr = convertJavaScriptExpression(bodyNode, source, language);
if (expr) {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, bodyNode);
exprStmt->setChild("expression", expr);
fn->addChild("body", exprStmt);
}
}
}
static ASTNode* convertJavaScriptStatement(TSNode node,
const std::string& source,
const std::string& language) {
std::string type = nodeType(node);
if (type == "return_statement") {
auto* ret = new Return();
ret->id = IdGenerator::next("ret");
applySpan(ret, node);
if (ts_node_named_child_count(node) > 0) {
ASTNode* val = convertJavaScriptExpression(ts_node_named_child(node, 0), source, language);
if (val) ret->setChild("value", val);
}
return ret;
} else if (type == "expression_statement") {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, node);
if (ts_node_named_child_count(node) > 0) {
ASTNode* expr = convertJavaScriptExpression(ts_node_named_child(node, 0), source, language);
if (expr) exprStmt->setChild("expression", expr);
}
return exprStmt;
} else if (type == "lexical_declaration" || type == "variable_declaration") {
uint32_t count = ts_node_named_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(node, i);
if (nodeType(child) != "variable_declarator") continue;
TSNode nameNode = childByFieldName(child, "name");
TSNode valueNode = childByFieldName(child, "value");
if (ts_node_is_null(nameNode)) continue;
auto* assign = new Assignment();
assign->id = IdGenerator::next("assign");
applySpan(assign, child);
auto* target = new VariableReference(IdGenerator::next("var"), nodeText(nameNode, source));
applySpan(target, nameNode);
assign->setChild("target", target);
if (!ts_node_is_null(valueNode)) {
ASTNode* val = convertJavaScriptExpression(valueNode, source, language);
if (val) assign->setChild("value", val);
}
return assign;
}
} else if (type == "if_statement") {
auto* ifStmt = new IfStatement();
ifStmt->id = IdGenerator::next("if");
applySpan(ifStmt, node);
TSNode condNode = childByFieldName(node, "condition");
if (!ts_node_is_null(condNode)) {
ASTNode* cond = convertJavaScriptExpression(condNode, source, language);
if (cond) ifStmt->setChild("condition", cond);
}
TSNode consNode = childByFieldName(node, "consequence");
if (!ts_node_is_null(consNode)) {
uint32_t cc = ts_node_named_child_count(consNode);
for (uint32_t i = 0; i < cc; ++i) {
ASTNode* s = convertJavaScriptStatement(ts_node_named_child(consNode, i), source, language);
if (s) ifStmt->addChild("thenBranch", s);
}
}
TSNode altNode = childByFieldName(node, "alternative");
if (!ts_node_is_null(altNode)) {
uint32_t ac = ts_node_named_child_count(altNode);
for (uint32_t i = 0; i < ac; ++i) {
ASTNode* s = convertJavaScriptStatement(ts_node_named_child(altNode, i), source, language);
if (s) ifStmt->addChild("elseBranch", s);
}
}
return ifStmt;
}
ASTNode* expr = convertJavaScriptExpression(node, source, language);
if (expr) {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, node);
exprStmt->setChild("expression", expr);
return exprStmt;
}
return nullptr;
}
static ASTNode* convertJavaScriptExpression(TSNode node,
const std::string& source,
const std::string& language) {
std::string type = nodeType(node);
if (type == "binary_expression" || type == "logical_expression") {
auto* binOp = new BinaryOperation();
binOp->id = IdGenerator::next("binop");
applySpan(binOp, node);
TSNode leftNode = childByFieldName(node, "left");
TSNode rightNode = childByFieldName(node, "right");
TSNode opNode = childByFieldName(node, "operator");
if (!ts_node_is_null(opNode)) binOp->op = nodeText(opNode, source);
if (!ts_node_is_null(leftNode)) {
ASTNode* left = convertJavaScriptExpression(leftNode, source, language);
if (left) binOp->setChild("left", left);
}
if (!ts_node_is_null(rightNode)) {
ASTNode* right = convertJavaScriptExpression(rightNode, source, language);
if (right) binOp->setChild("right", right);
}
return binOp;
} else if (type == "assignment_expression") {
auto* assign = new Assignment();
assign->id = IdGenerator::next("assign");
applySpan(assign, node);
TSNode leftNode = childByFieldName(node, "left");
TSNode rightNode = childByFieldName(node, "right");
if (!ts_node_is_null(leftNode)) {
ASTNode* target = convertJavaScriptExpression(leftNode, source, language);
if (target) assign->setChild("target", target);
}
if (!ts_node_is_null(rightNode)) {
ASTNode* value = convertJavaScriptExpression(rightNode, source, language);
if (value) assign->setChild("value", value);
}
return assign;
} else if (type == "call_expression") {
auto* call = new FunctionCall();
call->id = IdGenerator::next("call");
applySpan(call, node);
TSNode funcNode = childByFieldName(node, "function");
if (!ts_node_is_null(funcNode)) {
call->functionName = nodeText(funcNode, source);
}
TSNode argsNode = childByFieldName(node, "arguments");
if (!ts_node_is_null(argsNode)) {
uint32_t count = ts_node_named_child_count(argsNode);
for (uint32_t i = 0; i < count; ++i) {
ASTNode* arg = convertJavaScriptExpression(ts_node_named_child(argsNode, i), source, language);
if (arg) call->addChild("arguments", arg);
}
}
return call;
} else if (type == "member_expression") {
auto* mem = new MemberAccess();
mem->id = IdGenerator::next("member");
applySpan(mem, node);
TSNode objNode = childByFieldName(node, "object");
TSNode propNode = childByFieldName(node, "property");
if (!ts_node_is_null(propNode)) {
mem->memberName = nodeText(propNode, source);
}
if (!ts_node_is_null(objNode)) {
ASTNode* target = convertJavaScriptExpression(objNode, source, language);
if (target) mem->setChild("target", target);
}
return mem;
} else if (type == "subscript_expression") {
auto* access = new IndexAccess();
access->id = IdGenerator::next("index");
applySpan(access, node);
TSNode objNode = childByFieldName(node, "object");
TSNode idxNode = childByFieldName(node, "index");
if (!ts_node_is_null(objNode)) {
ASTNode* target = convertJavaScriptExpression(objNode, source, language);
if (target) access->setChild("target", target);
}
if (!ts_node_is_null(idxNode)) {
ASTNode* idx = convertJavaScriptExpression(idxNode, source, language);
if (idx) access->setChild("index", idx);
}
return access;
} else if (type == "identifier") {
auto* ref = new VariableReference(IdGenerator::next("var"), nodeText(node, source));
applySpan(ref, node);
return ref;
} else if (type == "number") {
std::string text = nodeText(node, source);
if (text.find('.') != std::string::npos) {
auto* lit = new FloatLiteral(IdGenerator::next("float"), text);
applySpan(lit, node);
return lit;
}
int val = 0;
try { val = std::stoi(text); } catch (...) {}
auto* lit = new IntegerLiteral(IdGenerator::next("int"), val);
applySpan(lit, node);
return lit;
} else if (type == "string" || type == "string_fragment" || type == "template_string") {
auto* lit = new StringLiteral(IdGenerator::next("str"), nodeText(node, source));
applySpan(lit, node);
return lit;
} else if (type == "true" || type == "false") {
auto* lit = new BooleanLiteral(IdGenerator::next("bool"), type == "true");
applySpan(lit, node);
return lit;
} else if (type == "null") {
auto* lit = new NullLiteral();
lit->id = IdGenerator::next("null");
applySpan(lit, node);
return lit;
} else if (type == "parenthesized_expression") {
if (ts_node_named_child_count(node) > 0) {
return convertJavaScriptExpression(ts_node_named_child(node, 0), source, language);
}
}
std::string text = nodeText(node, source);
if (!text.empty()) {
auto* ref = new VariableReference(IdGenerator::next("var"), text);
applySpan(ref, node);
return ref;
}
return nullptr;
}
// ---------------------------------------------------------------

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,312 @@
#pragma once
// TreeSitterParser Python support.
public:
// Python
// ---------------------------------------------------------------
static std::unique_ptr<Module> parsePython(const std::string& source) {
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, tree_sitter_python());
TSTree* tree = ts_parser_parse_string(parser, nullptr, source.c_str(), (uint32_t)source.size());
TSNode root = ts_tree_root_node(tree);
auto module = std::make_unique<Module>();
module->id = IdGenerator::next("mod");
module->name = "parsed_python_module";
module->targetLanguage = "python";
applySpan(module.get(), root);
convertPythonModule(root, source, module.get());
ts_tree_delete(tree);
ts_parser_delete(parser);
return module;
}
static ParseResult parsePythonWithDiagnostics(const std::string& source) {
ParseResult result;
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, tree_sitter_python());
TSTree* tree = ts_parser_parse_string(parser, nullptr, source.c_str(), (uint32_t)source.size());
TSNode root = ts_tree_root_node(tree);
result.module = std::make_unique<Module>();
result.module->id = IdGenerator::next("mod");
result.module->name = "parsed_python_module";
result.module->targetLanguage = "python";
applySpan(result.module.get(), root);
convertPythonModule(root, source, result.module.get());
collectDiagnostics(root, source, result.diagnostics);
ts_tree_delete(tree);
ts_parser_delete(parser);
return result;
}
// ---------------------------------------------------------------
private:
// Python CST → AST
// ---------------------------------------------------------------
static void convertPythonModule(TSNode root, const std::string& source, Module* module) {
uint32_t count = ts_node_named_child_count(root);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(root, i);
std::string type = nodeType(child);
if (type == "function_definition") {
auto* fn = convertPythonFunction(child, source);
if (fn) module->addChild("functions", fn);
}
}
}
static Function* convertPythonFunction(TSNode node, const std::string& source) {
// Get function name
TSNode nameNode = childByFieldName(node, "name");
if (ts_node_is_null(nameNode)) return nullptr;
auto* fn = new Function();
fn->id = IdGenerator::next("fn");
applySpan(fn, node);
applySpan(fn, node);
fn->name = nodeText(nameNode, source);
applySpan(fn, node);
// Parameters
TSNode paramsNode = childByFieldName(node, "parameters");
if (!ts_node_is_null(paramsNode)) {
convertPythonParameters(paramsNode, source, fn);
}
// Body
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
convertPythonBody(bodyNode, source, fn);
}
// Auto-annotate: Python uses tracing GC
auto* reclaim = new ReclaimAnnotation(IdGenerator::next("anno"), "Tracing");
fn->addChild("annotations", reclaim);
return fn;
}
static void convertPythonParameters(TSNode paramsNode, const std::string& source, Function* fn) {
uint32_t count = ts_node_named_child_count(paramsNode);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(paramsNode, i);
std::string type = nodeType(child);
if (type == "identifier") {
auto* param = new Parameter(IdGenerator::next("param"), nodeText(child, source));
applySpan(param, child);
fn->addChild("parameters", param);
} else if (type == "default_parameter") {
// def f(x=10) → Parameter with defaultValue
TSNode nameN = childByFieldName(child, "name");
TSNode valueN = childByFieldName(child, "value");
if (!ts_node_is_null(nameN)) {
auto* param = new Parameter(IdGenerator::next("param"), nodeText(nameN, source));
applySpan(param, child);
if (!ts_node_is_null(valueN)) {
ASTNode* defVal = convertPythonExpression(valueN, source);
if (defVal) param->setChild("defaultValue", defVal);
}
fn->addChild("parameters", param);
}
}
}
}
static void convertPythonBody(TSNode bodyNode, const std::string& source, Function* fn) {
// bodyNode is typically a "block" node
uint32_t count = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(bodyNode, i);
ASTNode* stmt = convertPythonStatement(child, source);
if (stmt) fn->addChild("body", stmt);
}
}
static ASTNode* convertPythonStatement(TSNode node, const std::string& source) {
std::string type = nodeType(node);
if (type == "return_statement") {
auto* ret = new Return();
ret->id = IdGenerator::next("ret");
applySpan(ret, node);
// The return value is the first named child (if any)
uint32_t count = ts_node_named_child_count(node);
if (count > 0) {
TSNode valNode = ts_node_named_child(node, 0);
ASTNode* val = convertPythonExpression(valNode, source);
if (val) ret->setChild("value", val);
}
return ret;
} else if (type == "if_statement") {
auto* ifStmt = new IfStatement();
ifStmt->id = IdGenerator::next("if");
applySpan(ifStmt, node);
TSNode condNode = childByFieldName(node, "condition");
if (!ts_node_is_null(condNode)) {
ASTNode* cond = convertPythonExpression(condNode, source);
if (cond) ifStmt->setChild("condition", cond);
}
TSNode conseq = childByFieldName(node, "consequence");
if (!ts_node_is_null(conseq)) {
uint32_t cc = ts_node_named_child_count(conseq);
for (uint32_t i = 0; i < cc; ++i) {
ASTNode* s = convertPythonStatement(ts_node_named_child(conseq, i), source);
if (s) ifStmt->addChild("thenBranch", s);
}
}
return ifStmt;
} else if (type == "for_statement") {
auto* forLoop = new ForLoop();
forLoop->id = IdGenerator::next("for");
applySpan(forLoop, node);
TSNode leftNode = childByFieldName(node, "left");
if (!ts_node_is_null(leftNode)) {
forLoop->iteratorName = nodeText(leftNode, source);
}
TSNode rightNode = childByFieldName(node, "right");
if (!ts_node_is_null(rightNode)) {
ASTNode* iter = convertPythonExpression(rightNode, source);
if (iter) forLoop->setChild("iterable", iter);
}
TSNode body = childByFieldName(node, "body");
if (!ts_node_is_null(body)) {
uint32_t cc = ts_node_named_child_count(body);
for (uint32_t i = 0; i < cc; ++i) {
ASTNode* s = convertPythonStatement(ts_node_named_child(body, i), source);
if (s) forLoop->addChild("body", s);
}
}
return forLoop;
} else if (type == "expression_statement") {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, node);
uint32_t count = ts_node_named_child_count(node);
if (count > 0) {
ASTNode* expr = convertPythonExpression(ts_node_named_child(node, 0), source);
if (expr) exprStmt->setChild("expression", expr);
}
return exprStmt;
}
// Fallback: wrap as expression statement
ASTNode* expr = convertPythonExpression(node, source);
if (expr) {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, node);
exprStmt->setChild("expression", expr);
return exprStmt;
}
return nullptr;
}
static ASTNode* convertPythonExpression(TSNode node, const std::string& source) {
std::string type = nodeType(node);
if (type == "binary_operator") {
auto* binOp = new BinaryOperation();
binOp->id = IdGenerator::next("binop");
applySpan(binOp, node);
TSNode opNode = childByFieldName(node, "operator");
if (!ts_node_is_null(opNode)) {
binOp->op = nodeText(opNode, source);
}
TSNode leftNode = childByFieldName(node, "left");
TSNode rightNode = childByFieldName(node, "right");
if (!ts_node_is_null(leftNode)) {
ASTNode* left = convertPythonExpression(leftNode, source);
if (left) binOp->setChild("left", left);
}
if (!ts_node_is_null(rightNode)) {
ASTNode* right = convertPythonExpression(rightNode, source);
if (right) binOp->setChild("right", right);
}
return binOp;
} else if (type == "identifier") {
auto* ref = new VariableReference(IdGenerator::next("var"), nodeText(node, source));
applySpan(ref, node);
return ref;
} else if (type == "integer") {
std::string text = nodeText(node, source);
int val = 0;
try { val = std::stoi(text); } catch (...) {}
auto* lit = new IntegerLiteral(IdGenerator::next("int"), val);
applySpan(lit, node);
return lit;
} else if (type == "string" || type == "concatenated_string") {
auto* lit = new StringLiteral(IdGenerator::next("str"), nodeText(node, source));
applySpan(lit, node);
return lit;
} else if (type == "comparison_operator" || type == "boolean_operator") {
// Treat like binary op
auto* binOp = new BinaryOperation();
binOp->id = IdGenerator::next("binop");
applySpan(binOp, node);
uint32_t count = ts_node_named_child_count(node);
if (count >= 2) {
ASTNode* left = convertPythonExpression(ts_node_named_child(node, 0), source);
if (left) binOp->setChild("left", left);
ASTNode* right = convertPythonExpression(ts_node_named_child(node, count - 1), source);
if (right) binOp->setChild("right", right);
}
// Operator is a non-named child between the named ones
uint32_t totalCount = ts_node_child_count(node);
for (uint32_t i = 0; i < totalCount; ++i) {
TSNode c = ts_node_child(node, i);
if (!ts_node_is_named(c)) {
std::string opText = nodeText(c, source);
if (!opText.empty() && opText != "(" && opText != ")") {
binOp->op = opText;
break;
}
}
}
return binOp;
} else if (type == "call") {
auto* call = new FunctionCall();
call->id = IdGenerator::next("call");
applySpan(call, node);
TSNode funcNode = childByFieldName(node, "function");
if (!ts_node_is_null(funcNode)) {
call->functionName = nodeText(funcNode, source);
}
TSNode argsNode = childByFieldName(node, "arguments");
if (!ts_node_is_null(argsNode)) {
uint32_t count = ts_node_named_child_count(argsNode);
for (uint32_t i = 0; i < count; ++i) {
ASTNode* arg = convertPythonExpression(ts_node_named_child(argsNode, i), source);
if (arg) call->addChild("arguments", arg);
}
}
return call;
} else if (type == "parenthesized_expression") {
uint32_t count = ts_node_named_child_count(node);
if (count > 0) return convertPythonExpression(ts_node_named_child(node, 0), source);
} else if (type == "unary_operator") {
auto* unOp = new UnaryOperation();
unOp->id = IdGenerator::next("unop");
applySpan(unOp, node);
TSNode opNode = childByFieldName(node, "operator");
if (!ts_node_is_null(opNode)) {
unOp->op = nodeText(opNode, source);
}
TSNode operandNode = childByFieldName(node, "operand");
if (!ts_node_is_null(operandNode)) {
ASTNode* operand = convertPythonExpression(operandNode, source);
if (operand) unOp->setChild("operand", operand);
}
return unOp;
}
// Fallback: treat as variable reference with raw text
std::string text = nodeText(node, source);
if (!text.empty()) {
auto* ref = new VariableReference(IdGenerator::next("var"), text);
applySpan(ref, node);
return ref;
}
return nullptr;
}
// ---------------------------------------------------------------

465
editor/src/ast/RustParser.h Normal file
View File

@@ -0,0 +1,465 @@
#pragma once
// TreeSitterParser Rust support.
public:
// Rust
// ---------------------------------------------------------------
static std::unique_ptr<Module> parseRust(const std::string& source) {
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, tree_sitter_rust());
TSTree* tree = ts_parser_parse_string(parser, nullptr, source.c_str(), (uint32_t)source.size());
TSNode root = ts_tree_root_node(tree);
auto module = std::make_unique<Module>();
module->id = IdGenerator::next("mod");
module->name = "parsed_rust_module";
module->targetLanguage = "rust";
applySpan(module.get(), root);
convertRustCrate(root, source, module.get());
ts_tree_delete(tree);
ts_parser_delete(parser);
return module;
}
static ParseResult parseRustWithDiagnostics(const std::string& source) {
ParseResult result;
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, tree_sitter_rust());
TSTree* tree = ts_parser_parse_string(parser, nullptr, source.c_str(), (uint32_t)source.size());
TSNode root = ts_tree_root_node(tree);
result.module = std::make_unique<Module>();
result.module->id = IdGenerator::next("mod");
result.module->name = "parsed_rust_module";
result.module->targetLanguage = "rust";
applySpan(result.module.get(), root);
convertRustCrate(root, source, result.module.get());
collectDiagnostics(root, source, result.diagnostics);
ts_tree_delete(tree);
ts_parser_delete(parser);
return result;
}
// ---------------------------------------------------------------
private:
// Rust CST -> AST
// ---------------------------------------------------------------
static void convertRustCrate(TSNode root,
const std::string& source,
Module* module) {
uint32_t count = ts_node_named_child_count(root);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(root, i);
std::string type = nodeType(child);
if (type == "use_declaration") {
TSNode arg = childByFieldName(child, "argument");
std::string importName = nodeText(arg, source);
if (importName.empty()) importName = nodeText(child, source);
if (!importName.empty()) {
auto* imp = new Import(IdGenerator::next("imp"), importName, "module");
module->addChild("imports", imp);
}
} else if (type == "function_item") {
auto* fn = convertRustFunction(child, source, "");
if (fn) module->addChild("functions", fn);
} else if (type == "impl_item") {
convertRustImpl(child, source, module);
} else if (type == "struct_item" || type == "enum_item" || type == "trait_item") {
// Record type name as a custom type variable for visibility.
TSNode nameNode = childByFieldName(child, "name");
if (!ts_node_is_null(nameNode)) {
auto* var = new Variable(IdGenerator::next("var"), nodeText(nameNode, source));
module->addChild("variables", var);
}
}
}
}
static void convertRustImpl(TSNode node,
const std::string& source,
Module* module) {
TSNode typeNode = childByFieldName(node, "type");
std::string typeName = nodeText(typeNode, source);
TSNode bodyNode = childByFieldName(node, "body");
if (ts_node_is_null(bodyNode)) return;
uint32_t count = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(bodyNode, i);
if (nodeType(child) == "function_item" || nodeType(child) == "function_signature_item") {
auto* fn = convertRustFunction(child, source, typeName);
if (fn) module->addChild("functions", fn);
}
}
}
static Function* convertRustFunction(TSNode node,
const std::string& source,
const std::string& receiverType) {
TSNode nameNode = childByFieldName(node, "name");
if (ts_node_is_null(nameNode)) return nullptr;
auto* fn = new Function();
fn->id = IdGenerator::next("fn");
applySpan(fn, node);
std::string name = nodeText(nameNode, source);
if (!receiverType.empty()) {
fn->name = receiverType + "." + name;
} else {
fn->name = name;
}
TSNode paramsNode = childByFieldName(node, "parameters");
if (!ts_node_is_null(paramsNode)) {
convertRustParameters(paramsNode, source, fn);
}
TSNode retNode = childByFieldName(node, "return_type");
if (!ts_node_is_null(retNode)) {
if (auto* t = convertRustType(retNode, source)) fn->setChild("returnType", t);
}
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
convertRustBlock(bodyNode, source, fn);
}
auto* lifetime = new LifetimeAnnotation(IdGenerator::next("anno"), "RAII");
fn->addChild("annotations", lifetime);
return fn;
}
static void convertRustParameters(TSNode paramsNode,
const std::string& source,
Function* fn) {
uint32_t count = ts_node_named_child_count(paramsNode);
for (uint32_t i = 0; i < count; ++i) {
TSNode child = ts_node_named_child(paramsNode, i);
std::string type = nodeType(child);
if (type == "self_parameter") {
auto* param = new Parameter(IdGenerator::next("param"), "self");
applySpan(param, child);
fn->addChild("parameters", param);
} else if (type == "parameter") {
TSNode patternNode = childByFieldName(child, "pattern");
TSNode typeNode = childByFieldName(child, "type");
if (ts_node_is_null(patternNode)) continue;
auto* param = new Parameter(IdGenerator::next("param"), nodeText(patternNode, source));
applySpan(param, child);
if (!ts_node_is_null(typeNode)) {
if (auto* t = convertRustType(typeNode, source)) param->setChild("type", t);
}
fn->addChild("parameters", param);
}
}
}
static void convertRustBlock(TSNode blockNode,
const std::string& source,
Function* fn) {
uint32_t count = ts_node_named_child_count(blockNode);
for (uint32_t i = 0; i < count; ++i) {
ASTNode* stmt = convertRustStatement(ts_node_named_child(blockNode, i), source);
if (stmt) fn->addChild("body", stmt);
}
}
static ASTNode* convertRustStatement(TSNode node,
const std::string& source) {
std::string type = nodeType(node);
if (type == "return_expression") {
auto* ret = new Return();
ret->id = IdGenerator::next("ret");
applySpan(ret, node);
if (ts_node_named_child_count(node) > 0) {
ASTNode* val = convertRustExpression(ts_node_named_child(node, 0), source);
if (val) ret->setChild("value", val);
}
return ret;
} else if (type == "let_declaration") {
auto* assign = new Assignment();
assign->id = IdGenerator::next("assign");
applySpan(assign, node);
TSNode patternNode = childByFieldName(node, "pattern");
TSNode valueNode = childByFieldName(node, "value");
if (!ts_node_is_null(patternNode)) {
auto* target = new VariableReference(IdGenerator::next("var"), nodeText(patternNode, source));
assign->setChild("target", target);
}
if (!ts_node_is_null(valueNode)) {
ASTNode* val = convertRustExpression(valueNode, source);
if (val) assign->setChild("value", val);
}
return assign;
} else if (type == "expression_statement") {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, node);
if (ts_node_named_child_count(node) > 0) {
ASTNode* expr = convertRustExpression(ts_node_named_child(node, 0), source);
if (expr) exprStmt->setChild("expression", expr);
}
return exprStmt;
} else if (type == "if_expression") {
auto* ifStmt = new IfStatement();
ifStmt->id = IdGenerator::next("if");
applySpan(ifStmt, node);
TSNode condNode = childByFieldName(node, "condition");
if (!ts_node_is_null(condNode)) {
ASTNode* cond = convertRustExpression(condNode, source);
if (cond) ifStmt->setChild("condition", cond);
}
TSNode consNode = childByFieldName(node, "consequence");
if (!ts_node_is_null(consNode)) {
uint32_t cc = ts_node_named_child_count(consNode);
for (uint32_t i = 0; i < cc; ++i) {
ASTNode* s = convertRustStatement(ts_node_named_child(consNode, i), source);
if (s) ifStmt->addChild("thenBranch", s);
}
}
TSNode altNode = childByFieldName(node, "alternative");
if (!ts_node_is_null(altNode)) {
uint32_t ac = ts_node_named_child_count(altNode);
for (uint32_t i = 0; i < ac; ++i) {
ASTNode* s = convertRustStatement(ts_node_named_child(altNode, i), source);
if (s) ifStmt->addChild("elseBranch", s);
}
}
return ifStmt;
} else if (type == "while_expression") {
auto* loop = new WhileLoop();
loop->id = IdGenerator::next("while");
applySpan(loop, node);
TSNode condNode = childByFieldName(node, "condition");
if (!ts_node_is_null(condNode)) {
ASTNode* cond = convertRustExpression(condNode, source);
if (cond) loop->setChild("condition", cond);
}
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
uint32_t bc = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < bc; ++i) {
ASTNode* s = convertRustStatement(ts_node_named_child(bodyNode, i), source);
if (s) loop->addChild("body", s);
}
}
return loop;
} else if (type == "for_expression") {
auto* loop = new ForLoop();
loop->id = IdGenerator::next("for");
applySpan(loop, node);
TSNode patternNode = childByFieldName(node, "pattern");
if (!ts_node_is_null(patternNode)) {
loop->iteratorName = nodeText(patternNode, source);
}
TSNode valueNode = childByFieldName(node, "value");
if (!ts_node_is_null(valueNode)) {
ASTNode* iter = convertRustExpression(valueNode, source);
if (iter) loop->setChild("iterable", iter);
}
TSNode bodyNode = childByFieldName(node, "body");
if (!ts_node_is_null(bodyNode)) {
uint32_t bc = ts_node_named_child_count(bodyNode);
for (uint32_t i = 0; i < bc; ++i) {
ASTNode* s = convertRustStatement(ts_node_named_child(bodyNode, i), source);
if (s) loop->addChild("body", s);
}
}
return loop;
} else if (type == "block") {
auto* block = new Block();
block->id = IdGenerator::next("block");
applySpan(block, node);
uint32_t bc = ts_node_named_child_count(node);
for (uint32_t i = 0; i < bc; ++i) {
ASTNode* s = convertRustStatement(ts_node_named_child(node, i), source);
if (s) block->addChild("statements", s);
}
return block;
}
ASTNode* expr = convertRustExpression(node, source);
if (expr) {
auto* exprStmt = new ExpressionStatement();
exprStmt->id = IdGenerator::next("exprstmt");
applySpan(exprStmt, node);
exprStmt->setChild("expression", expr);
return exprStmt;
}
return nullptr;
}
static ASTNode* convertRustExpression(TSNode node,
const std::string& source) {
std::string type = nodeType(node);
if (type == "binary_expression") {
auto* binOp = new BinaryOperation();
binOp->id = IdGenerator::next("binop");
applySpan(binOp, node);
TSNode leftNode = childByFieldName(node, "left");
TSNode rightNode = childByFieldName(node, "right");
TSNode opNode = childByFieldName(node, "operator");
if (!ts_node_is_null(opNode)) binOp->op = nodeText(opNode, source);
if (!ts_node_is_null(leftNode)) {
ASTNode* left = convertRustExpression(leftNode, source);
if (left) binOp->setChild("left", left);
}
if (!ts_node_is_null(rightNode)) {
ASTNode* right = convertRustExpression(rightNode, source);
if (right) binOp->setChild("right", right);
}
return binOp;
} else if (type == "assignment_expression" || type == "compound_assignment_expr") {
auto* assign = new Assignment();
assign->id = IdGenerator::next("assign");
applySpan(assign, node);
TSNode leftNode = childByFieldName(node, "left");
TSNode rightNode = childByFieldName(node, "right");
if (!ts_node_is_null(leftNode)) {
ASTNode* target = convertRustExpression(leftNode, source);
if (target) assign->setChild("target", target);
}
if (!ts_node_is_null(rightNode)) {
ASTNode* value = convertRustExpression(rightNode, source);
if (value) assign->setChild("value", value);
}
return assign;
} else if (type == "call_expression") {
auto* call = new FunctionCall();
call->id = IdGenerator::next("call");
applySpan(call, node);
TSNode funcNode = childByFieldName(node, "function");
if (!ts_node_is_null(funcNode)) {
call->functionName = nodeText(funcNode, source);
}
TSNode argsNode = childByFieldName(node, "arguments");
if (!ts_node_is_null(argsNode)) {
uint32_t count = ts_node_named_child_count(argsNode);
for (uint32_t i = 0; i < count; ++i) {
ASTNode* arg = convertRustExpression(ts_node_named_child(argsNode, i), source);
if (arg) call->addChild("arguments", arg);
}
}
return call;
} else if (type == "field_expression") {
auto* mem = new MemberAccess();
mem->id = IdGenerator::next("member");
applySpan(mem, node);
TSNode valueNode = childByFieldName(node, "value");
TSNode fieldNode = childByFieldName(node, "field");
if (!ts_node_is_null(fieldNode)) {
mem->memberName = nodeText(fieldNode, source);
}
if (!ts_node_is_null(valueNode)) {
ASTNode* target = convertRustExpression(valueNode, source);
if (target) mem->setChild("target", target);
}
return mem;
} else if (type == "index_expression") {
auto* access = new IndexAccess();
access->id = IdGenerator::next("index");
applySpan(access, node);
if (ts_node_named_child_count(node) >= 2) {
ASTNode* target = convertRustExpression(ts_node_named_child(node, 0), source);
ASTNode* idx = convertRustExpression(ts_node_named_child(node, 1), source);
if (target) access->setChild("target", target);
if (idx) access->setChild("index", idx);
}
return access;
} else if (type == "identifier") {
auto* ref = new VariableReference(IdGenerator::next("var"), nodeText(node, source));
applySpan(ref, node);
return ref;
} else if (type == "integer_literal") {
std::string text = nodeText(node, source);
int val = 0;
try { val = std::stoi(text); } catch (...) {}
auto* lit = new IntegerLiteral(IdGenerator::next("int"), val);
applySpan(lit, node);
return lit;
} else if (type == "float_literal") {
auto* lit = new FloatLiteral(IdGenerator::next("float"), nodeText(node, source));
applySpan(lit, node);
return lit;
} else if (type == "string_literal" || type == "char_literal") {
auto* lit = new StringLiteral(IdGenerator::next("str"), nodeText(node, source));
applySpan(lit, node);
return lit;
} else if (type == "true" || type == "false") {
auto* lit = new BooleanLiteral(IdGenerator::next("bool"), type == "true");
applySpan(lit, node);
return lit;
} else if (type == "unit_expression") {
auto* lit = new NullLiteral();
lit->id = IdGenerator::next("null");
applySpan(lit, node);
return lit;
} else if (type == "parenthesized_expression") {
if (ts_node_named_child_count(node) > 0) {
return convertRustExpression(ts_node_named_child(node, 0), source);
}
} else if (type == "array_expression") {
auto* list = new ListLiteral();
list->id = IdGenerator::next("list");
applySpan(list, node);
uint32_t count = ts_node_named_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
ASTNode* elem = convertRustExpression(ts_node_named_child(node, i), source);
if (elem) list->addChild("elements", elem);
}
return list;
}
std::string text = nodeText(node, source);
if (!text.empty()) {
auto* ref = new VariableReference(IdGenerator::next("var"), text);
applySpan(ref, node);
return ref;
}
return nullptr;
}
static Type* convertRustType(TSNode node, const std::string& source) {
std::string type = nodeType(node);
if (type == "primitive_type") {
auto* prim = new PrimitiveType();
prim->id = IdGenerator::next("type");
prim->kind = nodeText(node, source);
return prim;
} else if (type == "reference_type") {
TSNode valueNode = childByFieldName(node, "value");
if (!ts_node_is_null(valueNode)) {
if (auto* inner = convertRustType(valueNode, source)) {
auto* opt = new OptionalType();
opt->id = IdGenerator::next("type");
opt->setChild("innerType", inner);
return opt;
}
}
} else if (type == "array_type") {
auto* arr = new ArrayType();
arr->id = IdGenerator::next("type");
TSNode elemNode = childByFieldName(node, "element");
if (!ts_node_is_null(elemNode)) {
if (auto* et = convertRustType(elemNode, source)) arr->setChild("elementType", et);
}
return arr;
} else if (type == "tuple_type") {
auto* tuple = new TupleType();
tuple->id = IdGenerator::next("type");
uint32_t count = ts_node_named_child_count(node);
for (uint32_t i = 0; i < count; ++i) {
if (auto* t = convertRustType(ts_node_named_child(node, i), source)) {
tuple->addChild("elementTypes", t);
}
}
return tuple;
}
auto* custom = new CustomType();
custom->id = IdGenerator::next("type");
custom->typeName = nodeText(node, source);
return custom;
}
// ---------------------------------------------------------------

View File

@@ -0,0 +1,47 @@
#pragma once
// TreeSitterParser TypeScript support.
public:
// TypeScript
// ---------------------------------------------------------------
static std::unique_ptr<Module> parseTypeScript(const std::string& source) {
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, tree_sitter_typescript());
TSTree* tree = ts_parser_parse_string(parser, nullptr, source.c_str(), (uint32_t)source.size());
TSNode root = ts_tree_root_node(tree);
auto module = std::make_unique<Module>();
module->id = IdGenerator::next("mod");
module->name = "parsed_ts_module";
module->targetLanguage = "typescript";
applySpan(module.get(), root);
convertJavaScriptModule(root, source, module.get(), "typescript");
ts_tree_delete(tree);
ts_parser_delete(parser);
return module;
}
static ParseResult parseTypeScriptWithDiagnostics(const std::string& source) {
ParseResult result;
TSParser* parser = ts_parser_new();
ts_parser_set_language(parser, tree_sitter_typescript());
TSTree* tree = ts_parser_parse_string(parser, nullptr, source.c_str(), (uint32_t)source.size());
TSNode root = ts_tree_root_node(tree);
result.module = std::make_unique<Module>();
result.module->id = IdGenerator::next("mod");
result.module->name = "parsed_ts_module";
result.module->targetLanguage = "typescript";
applySpan(result.module.get(), root);
convertJavaScriptModule(root, source, result.module.get(), "typescript");
collectDiagnostics(root, source, result.diagnostics);
ts_tree_delete(tree);
ts_parser_delete(parser);
return result;
}
// ---------------------------------------------------------------
private:

View File

@@ -46,16 +46,12 @@ int main() {
int headerCount = 0;
int overs = 0;
std::vector<std::string> allowlist = {
(srcDir / "ast" / "CppGenerator.h").string(),
(srcDir / "ast" / "JavaGenerator.h").string(),
(srcDir / "ast" / "JavaScriptGenerator.h").string(),
(srcDir / "ast" / "Parser.h").string(),
(srcDir / "CodeEditorWidget.h").string(),
(srcDir / "EditorState.h").string(),
(srcDir / "EditorUtils.h").string(),
(srcDir / "panels" / "BottomPanel.h").string(),
(srcDir / "panels" / "EditorPanel.h").string(),
(srcDir / "SyntaxHighlighter.h").string()
(srcDir / "panels" / "EditorPanel.h").string()
};
for (const auto& entry : fs::recursive_directory_iterator(srcDir)) {
if (!entry.is_regular_file()) continue;

View File

@@ -0,0 +1,34 @@
#include <iostream>
#include <string>
#include "ast/Parser.h"
#include "SyntaxHighlighter.h"
static void expect(bool cond, const std::string& name, int& passed, int& failed) {
if (cond) {
std::cout << "Test " << (passed + failed + 1) << " PASS: " << name << "\n";
++passed;
} else {
std::cout << "Test " << (passed + failed + 1) << " FAIL: " << name << "\n";
++failed;
}
}
int main() {
int passed = 0;
int failed = 0;
const std::string py = "def f(x):\n return x + 1\n";
auto result = TreeSitterParser::parsePythonWithDiagnostics(py);
expect(result.module != nullptr, "parsePythonWithDiagnostics returns module", passed, failed);
expect(!result.hasErrors(), "parsePythonWithDiagnostics no errors", passed, failed);
auto spans = SyntaxHighlighter::highlight("def f():\n return 1\n", "python");
expect(!spans.empty(), "syntax highlighter returns spans", passed, failed);
expect(std::string(SyntaxHighlighter::categoryName(TokenCategory::Keyword)) == "keyword",
"categoryName keyword", passed, failed);
std::cout << "\n=== Step 168 Integration Results: " << passed << " passed, "
<< failed << " failed ===\n";
return failed == 0 ? 0 : 1;
}

View File

@@ -0,0 +1,83 @@
#include <filesystem>
#include <fstream>
#include <iostream>
#include <string>
#include <vector>
static void expect(bool cond, const std::string& name, int& passed, int& failed) {
if (cond) {
std::cout << "Test " << (passed + failed + 1) << " PASS: " << name << "\n";
++passed;
} else {
std::cout << "Test " << (passed + failed + 1) << " FAIL: " << name << "\n";
++failed;
}
}
static int countLines(const std::string& path) {
std::ifstream in(path);
if (!in.is_open()) return -1;
int lines = 0;
std::string line;
while (std::getline(in, line)) {
++lines;
}
return lines;
}
int main() {
int passed = 0;
int failed = 0;
const int maxHeaderLines = 600;
namespace fs = std::filesystem;
fs::path root = fs::current_path();
fs::path srcDir = root / "editor" / "src";
if (!fs::exists(srcDir)) {
srcDir = root / "src";
}
std::vector<fs::path> headers = {
srcDir / "CodeEditorWidget.h",
srcDir / "CodeEditorRendering.h",
srcDir / "CodeEditorRenderHelpers.h",
srcDir / "CodeEditorSelection.h",
srcDir / "SyntaxHighlighter.h",
srcDir / "SyntaxLanguages.h",
srcDir / "SyntaxHighlighterPython.h",
srcDir / "SyntaxHighlighterCpp.h",
srcDir / "SyntaxHighlighterJavaScript.h",
srcDir / "SyntaxHighlighterJava.h",
srcDir / "SyntaxHighlighterRust.h",
srcDir / "SyntaxHighlighterGo.h",
srcDir / "SyntaxHighlighterElisp.h",
srcDir / "SyntaxHighlighterOrg.h",
srcDir / "ast" / "Parser.h",
srcDir / "ast" / "PythonParser.h",
srcDir / "ast" / "CppParser.h",
srcDir / "ast" / "ElispParser.h",
srcDir / "ast" / "JavaScriptParser.h",
srcDir / "ast" / "TypeScriptParser.h",
srcDir / "ast" / "JavaParser.h",
srcDir / "ast" / "RustParser.h",
srcDir / "ast" / "GoParser.h",
srcDir / "ast" / "CppGenerator.h",
srcDir / "ast" / "CppGeneratorTypes.h",
};
for (const auto& path : headers) {
expect(fs::exists(path), "exists " + path.string(), passed, failed);
int lines = countLines(path.string());
expect(lines > 0, "read " + path.string(), passed, failed);
if (lines > 0) {
expect(lines <= maxHeaderLines,
"line limit " + path.string() + " (" + std::to_string(lines) + ")",
passed, failed);
}
}
std::cout << "\n=== Step 168 Results: " << passed << " passed, "
<< failed << " failed ===\n";
return failed == 0 ? 0 : 1;
}

View File

@@ -51,7 +51,7 @@ first — all UX work becomes easier when panels are modular.
400 lines, each sub-state under 200 lines.
*Modifies:* `EditorState.h`. *New:* sub-state headers or inline structs.
- [ ] **Step 168: Split oversized component headers**
- [x] **Step 168: Split oversized component headers**
Enforce the 600-line limit on remaining violators:
- `CodeEditorWidget.h` (1112 lines): extract rendering helpers into
`CodeEditorRendering.h`, selection logic into `CodeEditorSelection.h`