1696b92bb8
Sprints 46-58 implement the cross-language porting foundation: language-to-IR adapters, equivalence checking, gate validation, legacy ingestion, managed/dynamic families, low-level/logic-actor semantics, debug workflow tooling, AST-native family tools, Rust/CPP raising tools, system-level orchestration, query family, and porting gates. Sprint 59 adds the governance layer (policy packs, review boards, waiver packets, ambiguity triage, decision ledger) with the whetstone_review_porting_decision MCP tool. Also includes: sprint plans 46-130, MCP taskitem pipeline scripts, CLAUDE.md, docs, and full test matrix (steps 689-828). Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
242 lines
8.2 KiB
C++
242 lines
8.2 KiB
C++
#pragma once
|
|
// Step 52: Syntax highlighting via tree-sitter
|
|
//
|
|
// SyntaxHighlighter parses source text with tree-sitter and walks the CST
|
|
// to produce a list of colored spans. Each span has a byte range and a
|
|
// token category (keyword, string, comment, number, identifier, operator,
|
|
// type, punctuation, function, parameter).
|
|
//
|
|
// The highlight data is language-agnostic at the output level — consumers
|
|
// map categories to colors via a theme.
|
|
|
|
#include <string>
|
|
#include <vector>
|
|
#include <algorithm>
|
|
#include <cstring>
|
|
#include <cctype>
|
|
#include <tree_sitter/api.h>
|
|
|
|
extern "C" {
|
|
const TSLanguage* tree_sitter_python();
|
|
const TSLanguage* tree_sitter_cpp();
|
|
const TSLanguage* tree_sitter_elisp();
|
|
const TSLanguage* tree_sitter_javascript();
|
|
const TSLanguage* tree_sitter_typescript();
|
|
const TSLanguage* tree_sitter_java();
|
|
const TSLanguage* tree_sitter_rust();
|
|
const TSLanguage* tree_sitter_go();
|
|
#if !defined(WHETSTONE_ENABLE_ORG)
|
|
#define WHETSTONE_ENABLE_ORG 0
|
|
#endif
|
|
#if WHETSTONE_ENABLE_ORG
|
|
const TSLanguage* tree_sitter_org();
|
|
#endif
|
|
}
|
|
|
|
enum class TokenCategory {
|
|
Plain,
|
|
Keyword,
|
|
String,
|
|
Comment,
|
|
Number,
|
|
Identifier,
|
|
Operator,
|
|
Type,
|
|
Punctuation,
|
|
Function,
|
|
Parameter,
|
|
Builtin
|
|
};
|
|
|
|
struct HighlightSpan {
|
|
uint32_t start; // byte offset
|
|
uint32_t end; // byte offset (exclusive)
|
|
TokenCategory category;
|
|
};
|
|
|
|
class SyntaxHighlighter {
|
|
public:
|
|
static std::vector<HighlightSpan> highlight(const std::string& source,
|
|
const std::string& language) {
|
|
std::vector<HighlightSpan> spans;
|
|
if (source.empty()) return spans;
|
|
if (language == "markdown") {
|
|
walkMarkdownSimple(source, spans);
|
|
std::sort(spans.begin(), spans.end(),
|
|
[](const HighlightSpan& a, const HighlightSpan& b) {
|
|
return a.start < b.start;
|
|
});
|
|
return spans;
|
|
}
|
|
|
|
TSParser* parser = ts_parser_new();
|
|
const TSLanguage* lang = nullptr;
|
|
|
|
if (language == "python") lang = tree_sitter_python();
|
|
else if (language == "cpp") lang = tree_sitter_cpp();
|
|
else if (language == "elisp") lang = tree_sitter_elisp();
|
|
else if (language == "javascript") lang = tree_sitter_javascript();
|
|
else if (language == "typescript") lang = tree_sitter_typescript();
|
|
else if (language == "java") lang = tree_sitter_java();
|
|
else if (language == "rust") lang = tree_sitter_rust();
|
|
else if (language == "go") lang = tree_sitter_go();
|
|
#if WHETSTONE_ENABLE_ORG
|
|
else if (language == "org") lang = tree_sitter_org();
|
|
#endif
|
|
|
|
if (!lang) {
|
|
ts_parser_delete(parser);
|
|
return spans;
|
|
}
|
|
|
|
ts_parser_set_language(parser, lang);
|
|
TSTree* tree = ts_parser_parse_string(parser, nullptr,
|
|
source.c_str(),
|
|
(uint32_t)source.size());
|
|
TSNode root = ts_tree_root_node(tree);
|
|
|
|
if (language == "python") walkPython(root, source, spans);
|
|
else if (language == "cpp") walkCpp(root, source, spans);
|
|
else if (language == "elisp") walkElisp(root, source, spans);
|
|
else if (language == "javascript") walkJavaScript(root, source, spans);
|
|
else if (language == "typescript") walkTypeScript(root, source, spans);
|
|
else if (language == "java") walkJava(root, source, spans);
|
|
else if (language == "rust") walkRust(root, source, spans);
|
|
else if (language == "go") walkGo(root, source, spans);
|
|
#if WHETSTONE_ENABLE_ORG
|
|
else if (language == "org") walkOrgSimple(source, spans);
|
|
#endif
|
|
|
|
ts_tree_delete(tree);
|
|
ts_parser_delete(parser);
|
|
|
|
// Sort by start position
|
|
std::sort(spans.begin(), spans.end(),
|
|
[](const HighlightSpan& a, const HighlightSpan& b) {
|
|
return a.start < b.start;
|
|
});
|
|
|
|
return spans;
|
|
}
|
|
|
|
static const char* categoryName(TokenCategory cat) {
|
|
switch (cat) {
|
|
case TokenCategory::Plain: return "plain";
|
|
case TokenCategory::Keyword: return "keyword";
|
|
case TokenCategory::String: return "string";
|
|
case TokenCategory::Comment: return "comment";
|
|
case TokenCategory::Number: return "number";
|
|
case TokenCategory::Identifier: return "identifier";
|
|
case TokenCategory::Operator: return "operator";
|
|
case TokenCategory::Type: return "type";
|
|
case TokenCategory::Punctuation: return "punctuation";
|
|
case TokenCategory::Function: return "function";
|
|
case TokenCategory::Parameter: return "parameter";
|
|
case TokenCategory::Builtin: return "builtin";
|
|
}
|
|
return "plain";
|
|
}
|
|
|
|
private:
|
|
static std::string nodeType(TSNode node) {
|
|
return ts_node_type(node);
|
|
}
|
|
|
|
static std::string nodeText(TSNode node, const std::string& source) {
|
|
uint32_t s = ts_node_start_byte(node);
|
|
uint32_t e = ts_node_end_byte(node);
|
|
if (s >= source.size() || e > source.size()) return "";
|
|
return source.substr(s, e - s);
|
|
}
|
|
|
|
static void addSpan(std::vector<HighlightSpan>& spans, TSNode node,
|
|
TokenCategory cat) {
|
|
uint32_t s = ts_node_start_byte(node);
|
|
uint32_t e = ts_node_end_byte(node);
|
|
if (s < e) {
|
|
spans.push_back({s, e, cat});
|
|
}
|
|
}
|
|
|
|
static void walkMarkdownSimple(const std::string& source,
|
|
std::vector<HighlightSpan>& spans) {
|
|
size_t lineStart = 0;
|
|
bool inFence = false;
|
|
while (lineStart <= source.size()) {
|
|
size_t lineEnd = source.find('\n', lineStart);
|
|
if (lineEnd == std::string::npos) lineEnd = source.size();
|
|
std::string line = source.substr(lineStart, lineEnd - lineStart);
|
|
std::string trimmed = line;
|
|
while (!trimmed.empty() && std::isspace((unsigned char)trimmed.front())) {
|
|
trimmed.erase(trimmed.begin());
|
|
}
|
|
|
|
if (trimmed.rfind("```", 0) == 0 || trimmed.rfind("~~~", 0) == 0) {
|
|
spans.push_back({(uint32_t)lineStart, (uint32_t)lineEnd, TokenCategory::Keyword});
|
|
inFence = !inFence;
|
|
} else if (inFence) {
|
|
spans.push_back({(uint32_t)lineStart, (uint32_t)lineEnd, TokenCategory::String});
|
|
} else if (!trimmed.empty() && trimmed[0] == '#') {
|
|
spans.push_back({(uint32_t)lineStart, (uint32_t)lineEnd, TokenCategory::Keyword});
|
|
} else {
|
|
size_t tickStart = line.find('`');
|
|
while (tickStart != std::string::npos) {
|
|
size_t tickEnd = line.find('`', tickStart + 1);
|
|
if (tickEnd == std::string::npos) break;
|
|
spans.push_back({
|
|
(uint32_t)(lineStart + tickStart),
|
|
(uint32_t)(lineStart + tickEnd + 1),
|
|
TokenCategory::Builtin
|
|
});
|
|
tickStart = line.find('`', tickEnd + 1);
|
|
}
|
|
}
|
|
if (lineEnd == source.size()) break;
|
|
lineStart = lineEnd + 1;
|
|
}
|
|
}
|
|
|
|
#include "SyntaxLanguages.h"
|
|
#include "SyntaxHighlighterPython.h"
|
|
#include "SyntaxHighlighterCpp.h"
|
|
#include "SyntaxHighlighterJavaScript.h"
|
|
#include "SyntaxHighlighterJava.h"
|
|
#include "SyntaxHighlighterRust.h"
|
|
#include "SyntaxHighlighterGo.h"
|
|
#include "SyntaxHighlighterElisp.h"
|
|
#include "SyntaxHighlighterOrg.h"
|
|
// --- Python --------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
// --- C++ -----------------------------------------------------------
|
|
|
|
|
|
|
|
// --- JavaScript / TypeScript --------------------------------------
|
|
|
|
|
|
|
|
|
|
// --- Java -----------------------------------------------------------
|
|
|
|
|
|
|
|
// --- Rust -----------------------------------------------------------
|
|
|
|
|
|
|
|
// --- Go -------------------------------------------------------------
|
|
|
|
|
|
|
|
|
|
// --- Elisp ---------------------------------------------------------
|
|
|
|
|
|
|
|
// --- Org -----------------------------------------------------------
|
|
|
|
};
|