Complete step461 x86 assembly parser with tests

This commit is contained in:
Bill
2026-02-16 20:10:19 -07:00
parent 95922f1ec6
commit 5b0044cabb
4 changed files with 531 additions and 0 deletions

View File

@@ -3028,4 +3028,13 @@ target_link_libraries(step460_test PRIVATE
tree_sitter_javascript tree_sitter_typescript
tree_sitter_java tree_sitter_rust tree_sitter_go)
add_executable(step461_test tests/step461_test.cpp)
target_include_directories(step461_test PRIVATE src)
target_link_libraries(step461_test PRIVATE
nlohmann_json::nlohmann_json
unofficial::tree-sitter::tree-sitter
tree_sitter_python tree_sitter_cpp tree_sitter_elisp
tree_sitter_javascript tree_sitter_typescript
tree_sitter_java tree_sitter_rust tree_sitter_go)
# Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies)

View File

@@ -0,0 +1,299 @@
#pragma once
// Step 461: x86 Assembly Parser
// Supports Intel and AT&T syntax for core instruction/directive/label forms.
#include "AssemblyNodes.h"
#include "EnumNamespaceNodes.h"
#include "Function.h"
#include "Module.h"
#include "Parser.h"
#include <algorithm>
#include <cctype>
#include <regex>
#include <set>
#include <sstream>
#include <string>
#include <vector>
enum class X86SyntaxMode {
Intel,
ATT
};
class X86AssemblyParser {
public:
static std::unique_ptr<Module> parseX86(const std::string& source,
X86SyntaxMode mode = X86SyntaxMode::Intel) {
auto result = parseX86WithDiagnostics(source, mode);
return std::move(result.module);
}
static ParseResult parseX86WithDiagnostics(const std::string& source,
X86SyntaxMode mode = X86SyntaxMode::Intel) {
ParseResult out;
auto module = std::make_unique<Module>();
module->id = IdGenerator::next("mod");
module->name = "parsed_x86_module";
module->targetLanguage = "x86_asm";
std::set<std::string> globalLabels;
NamespaceDeclaration* currentSection = nullptr;
Function* currentFunction = nullptr;
std::istringstream in(source);
std::string raw;
int lineNo = 0;
while (std::getline(in, raw)) {
++lineNo;
std::string line = stripComment(raw, mode);
line = trim(line);
if (line.empty()) continue;
if (isDirective(line)) {
auto dir = parseDirective(line);
if (dir.type == AssemblyDirectiveType::Global) {
std::string g = trim(dir.value);
if (!g.empty()) globalLabels.insert(stripSigils(g));
}
if (dir.type == AssemblyDirectiveType::Text ||
dir.type == AssemblyDirectiveType::Data ||
dir.type == AssemblyDirectiveType::Section) {
std::string sectionName = dir.type == AssemblyDirectiveType::Section
? trim(dir.value) : directiveTypeToString(dir.type);
currentSection = new NamespaceDeclaration(IdGenerator::next("ns"), sectionName);
module->addChild("statements", currentSection);
currentFunction = nullptr;
}
auto* d = new AssemblyDirective(dir.type, dir.value);
if (currentSection) currentSection->addChild("body", d);
else module->addChild("statements", d);
continue;
}
if (isLabel(line)) {
std::string labelName = parseLabelName(line);
bool isGlobal = globalLabels.count(labelName) > 0;
auto* label = new AssemblyLabel(labelName, isGlobal);
currentFunction = new Function(IdGenerator::next("fn"), labelName);
currentFunction->addChild("body", label);
module->addChild("functions", currentFunction);
continue;
}
auto inst = parseInstruction(line, mode);
if (inst.opcode.empty()) {
out.diagnostics.push_back({lineNo, 1, "Unrecognized assembly line: " + line, "warning"});
continue;
}
if (!isKnownOpcode(inst.opcode)) {
out.diagnostics.push_back({lineNo, 1, "Unknown opcode: " + inst.opcode, "warning"});
}
auto* node = new AssemblyInstruction(inst.opcode, inst.operands);
node->registers = std::move(inst.registers);
node->memoryOperands = std::move(inst.memoryOperands);
if (!currentFunction) {
currentFunction = new Function(IdGenerator::next("fn"), "_entry");
module->addChild("functions", currentFunction);
}
currentFunction->addChild("body", node);
}
out.module = std::move(module);
return out;
}
private:
static std::string trim(const std::string& s) {
size_t start = s.find_first_not_of(" \t\r\n");
if (start == std::string::npos) return "";
size_t end = s.find_last_not_of(" \t\r\n");
return s.substr(start, end - start + 1);
}
static std::string stripComment(const std::string& line, X86SyntaxMode mode) {
size_t pos = line.find(';');
size_t pos2 = std::string::npos;
if (mode == X86SyntaxMode::ATT) pos2 = line.find('#');
size_t cut = std::string::npos;
if (pos != std::string::npos) cut = pos;
if (pos2 != std::string::npos) cut = (cut == std::string::npos) ? pos2 : std::min(cut, pos2);
return cut == std::string::npos ? line : line.substr(0, cut);
}
static bool isDirective(const std::string& line) {
return !line.empty() && line[0] == '.';
}
static bool isLabel(const std::string& line) {
return !line.empty() && line.back() == ':';
}
static std::string parseLabelName(const std::string& line) {
std::string n = line.substr(0, line.size() - 1);
return stripSigils(trim(n));
}
static std::string stripSigils(const std::string& token) {
std::string out = token;
while (!out.empty() && (out[0] == '%' || out[0] == '$')) out.erase(out.begin());
return out;
}
static AssemblyDirective parseDirective(const std::string& line) {
size_t sp = line.find_first_of(" \t");
std::string head = sp == std::string::npos ? line : line.substr(0, sp);
std::string tail = sp == std::string::npos ? "" : trim(line.substr(sp + 1));
return AssemblyDirective(directiveTypeFromString(head), tail);
}
static std::vector<std::string> splitOperands(const std::string& s) {
std::vector<std::string> out;
std::string cur;
int bracketDepth = 0;
int parenDepth = 0;
for (char c : s) {
if (c == '[') ++bracketDepth;
if (c == ']') --bracketDepth;
if (c == '(') ++parenDepth;
if (c == ')') --parenDepth;
if (c == ',' && bracketDepth == 0 && parenDepth == 0) {
out.push_back(trim(cur));
cur.clear();
} else {
cur.push_back(c);
}
}
if (!trim(cur).empty()) out.push_back(trim(cur));
return out;
}
static bool isRegisterName(const std::string& raw) {
std::string r = stripSigils(raw);
static const std::set<std::string> regs = {
"rax","rbx","rcx","rdx","rsi","rdi","rsp","rbp","rip",
"eax","ebx","ecx","edx","esi","edi","esp","ebp","eip",
"ax","bx","cx","dx","si","di","sp","bp","ip",
"al","bl","cl","dl","sil","dil","spl","bpl",
"r8","r9","r10","r11","r12","r13","r14","r15",
"r8d","r9d","r10d","r11d","r12d","r13d","r14d","r15d",
"r8w","r9w","r10w","r11w","r12w","r13w","r14w","r15w",
"r8b","r9b","r10b","r11b","r12b","r13b","r14b","r15b"
};
return regs.count(r) > 0;
}
static int registerSizeBits(const std::string& raw) {
std::string r = stripSigils(raw);
if (r.empty()) return 0;
if (r == "rax" || r == "rbx" || r == "rcx" || r == "rdx" || r == "rsi" ||
r == "rdi" || r == "rsp" || r == "rbp" || r == "rip" ||
(r.size() >= 2 && r[0] == 'r' && std::isdigit(static_cast<unsigned char>(r[1])) &&
(r.size() == 2 || (r[2] >= '0' && r[2] <= '9'))))
return 64;
if (r == "eax" || r == "ebx" || r == "ecx" || r == "edx" || r == "esi" ||
r == "edi" || r == "esp" || r == "ebp" || r == "eip" ||
(r.size() >= 3 && r[0] == 'r' && std::isdigit(static_cast<unsigned char>(r[1])) && r.back() == 'd'))
return 32;
if (r == "ax" || r == "bx" || r == "cx" || r == "dx" || r == "si" || r == "di" ||
r == "sp" || r == "bp" || r == "ip" ||
(r.size() >= 3 && r[0] == 'r' && std::isdigit(static_cast<unsigned char>(r[1])) && r.back() == 'w'))
return 16;
return 8;
}
static int parseInt(const std::string& raw) {
std::string t = trim(raw);
if (t.empty()) return 0;
try {
size_t idx = 0;
int base = 10;
if (t.size() > 2 && t[0] == '0' && (t[1] == 'x' || t[1] == 'X')) base = 16;
return std::stoi(t, &idx, base);
} catch (...) {
return 0;
}
}
static bool parseIntelMemory(const std::string& op, AssemblyMemoryOperand& out) {
if (op.size() < 2 || op.front() != '[' || op.back() != ']') return false;
std::string inner = trim(op.substr(1, op.size() - 2));
std::string token;
std::string normalized;
for (char c : inner) normalized.push_back(c == '-' ? '+' : c);
std::stringstream ss(normalized);
std::vector<std::string> parts;
while (std::getline(ss, token, '+')) {
token = trim(token);
if (!token.empty()) parts.push_back(token);
}
bool sawNegative = inner.find('-') != std::string::npos;
for (const auto& p : parts) {
if (isRegisterName(p)) {
if (out.baseRegister.empty()) out.baseRegister = stripSigils(p);
else out.indexRegister = stripSigils(p);
} else if (p.find('*') != std::string::npos) {
size_t star = p.find('*');
out.indexRegister = stripSigils(trim(p.substr(0, star)));
out.scale = parseInt(trim(p.substr(star + 1)));
} else {
out.offset = parseInt(p);
}
}
if (sawNegative && out.offset > 0) out.offset = -out.offset;
return !out.baseRegister.empty() || !out.indexRegister.empty() || out.offset != 0;
}
static bool parseATTMemory(const std::string& op, AssemblyMemoryOperand& out) {
// form: -8(%rbp,%rbx,4) or (%rax)
std::regex rx(R"(^\s*([-+]?\d+)?\s*\(\s*%?([A-Za-z0-9]+)\s*(?:,\s*%?([A-Za-z0-9]+)\s*(?:,\s*(\d+)\s*)?)?\)\s*$)");
std::smatch m;
if (!std::regex_match(op, m, rx)) return false;
out.offset = m[1].matched ? parseInt(m[1].str()) : 0;
out.baseRegister = stripSigils(m[2].str());
if (m[3].matched) out.indexRegister = stripSigils(m[3].str());
if (m[4].matched) out.scale = parseInt(m[4].str());
return true;
}
static AssemblyInstruction parseInstruction(const std::string& line, X86SyntaxMode mode) {
size_t sp = line.find_first_of(" \t");
std::string op = trim(sp == std::string::npos ? line : line.substr(0, sp));
std::string ops = sp == std::string::npos ? "" : trim(line.substr(sp + 1));
AssemblyInstruction inst(op, splitOperands(ops));
for (const auto& operand : inst.operands) {
if (isRegisterName(operand)) {
inst.registers.emplace_back(stripSigils(operand), registerSizeBits(operand));
continue;
}
AssemblyMemoryOperand mem;
bool ok = (mode == X86SyntaxMode::Intel)
? parseIntelMemory(operand, mem)
: parseATTMemory(operand, mem);
if (ok) inst.memoryOperands.push_back(mem);
}
return inst;
}
static bool isKnownOpcode(const std::string& opRaw) {
std::string op = opRaw;
std::transform(op.begin(), op.end(), op.begin(),
[](unsigned char c){ return static_cast<char>(std::tolower(c)); });
// Normalize AT&T suffixes (movq, addl, etc.).
if (op.size() > 3 && (op.back() == 'b' || op.back() == 'w' || op.back() == 'l' || op.back() == 'q')) {
std::string trimmed = op.substr(0, op.size() - 1);
if (trimmed == "mov" || trimmed == "add" || trimmed == "sub" || trimmed == "cmp" ||
trimmed == "push" || trimmed == "pop") op = trimmed;
}
static const std::set<std::string> known = {
"mov", "add", "sub", "cmp", "jmp", "call", "ret", "push", "pop",
"lea", "test", "and", "or", "xor", "nop"
};
return known.count(op) > 0;
}
};

View File

@@ -0,0 +1,186 @@
// Step 461: x86 Assembly Parser Tests (12 tests)
#include "ast/AssemblyNodes.h"
#include "ast/EnumNamespaceNodes.h"
#include "ast/X86AssemblyParser.h"
#include <iostream>
static int passed = 0, failed = 0;
#define TEST(name) { std::cout << " " << #name << "... "; }
#define PASS() { std::cout << "PASS\n"; ++passed; }
#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; }
#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {}
void test_parse_intel_basic_instructions() {
TEST(parse_intel_basic_instructions);
std::string src = ".text\nmain:\n mov rax, rbx\n add rax, 1\n ret\n";
auto mod = X86AssemblyParser::parseX86(src, X86SyntaxMode::Intel);
CHECK(mod->getChildren("functions").size() == 1, "expected one function");
auto* fn = static_cast<Function*>(mod->getChildren("functions")[0]);
CHECK(fn->name == "main", "expected function name main");
CHECK(fn->getChildren("body").size() >= 3, "expected label + instructions");
PASS();
}
void test_parse_att_basic_instructions() {
TEST(parse_att_basic_instructions);
std::string src = ".text\nmain:\n movq %rax, %rbx\n ret\n";
auto mod = X86AssemblyParser::parseX86(src, X86SyntaxMode::ATT);
auto* fn = static_cast<Function*>(mod->getChildren("functions")[0]);
bool sawMov = false;
for (auto* n : fn->getChildren("body")) {
if (n->conceptType == "AssemblyInstruction") {
auto* i = static_cast<AssemblyInstruction*>(n);
if (i->opcode == "movq") sawMov = true;
}
}
CHECK(sawMov, "expected movq opcode");
PASS();
}
void test_parse_labels_and_global_flags() {
TEST(parse_labels_and_global_flags);
std::string src = ".global main\n.text\nmain:\n ret\n";
auto mod = X86AssemblyParser::parseX86(src);
auto* fn = static_cast<Function*>(mod->getChildren("functions")[0]);
auto* lbl = static_cast<AssemblyLabel*>(fn->getChildren("body")[0]);
CHECK(lbl->conceptType == "AssemblyLabel", "expected label node");
CHECK(lbl->isGlobal, "expected global label");
PASS();
}
void test_parse_directives_and_sections_as_namespaces() {
TEST(parse_directives_and_sections_as_namespaces);
std::string src = ".text\nmain:\n ret\n.data\nx: .word 1\n";
auto mod = X86AssemblyParser::parseX86(src);
int nsCount = 0;
for (auto* s : mod->getChildren("statements")) {
if (s->conceptType == "NamespaceDeclaration") ++nsCount;
}
CHECK(nsCount >= 2, "expected namespace-like sections");
PASS();
}
void test_register_recognition_sizes() {
TEST(register_recognition_sizes);
std::string src = ".text\nmain:\n mov rax, eax\n mov ax, al\n ret\n";
auto mod = X86AssemblyParser::parseX86(src);
auto* fn = static_cast<Function*>(mod->getChildren("functions")[0]);
int has64 = 0, has32 = 0, has16 = 0, has8 = 0;
for (auto* n : fn->getChildren("body")) {
if (n->conceptType != "AssemblyInstruction") continue;
auto* i = static_cast<AssemblyInstruction*>(n);
for (const auto& r : i->registers) {
if (r.sizeBits == 64) ++has64;
if (r.sizeBits == 32) ++has32;
if (r.sizeBits == 16) ++has16;
if (r.sizeBits == 8) ++has8;
}
}
CHECK(has64 > 0 && has32 > 0 && has16 > 0 && has8 > 0, "expected all register sizes");
PASS();
}
void test_parse_intel_memory_base_offset() {
TEST(parse_intel_memory_base_offset);
std::string src = ".text\nmain:\n mov rax, [rbp-8]\n ret\n";
auto mod = X86AssemblyParser::parseX86(src, X86SyntaxMode::Intel);
auto* fn = static_cast<Function*>(mod->getChildren("functions")[0]);
auto* i = static_cast<AssemblyInstruction*>(fn->getChildren("body")[1]);
CHECK(i->memoryOperands.size() == 1, "expected one memory operand");
CHECK(i->memoryOperands[0].baseRegister == "rbp", "expected base rbp");
CHECK(i->memoryOperands[0].offset == -8, "expected offset -8");
PASS();
}
void test_parse_intel_memory_base_index_scale_offset() {
TEST(parse_intel_memory_base_index_scale_offset);
std::string src = ".text\nmain:\n mov rax, [rax+rbx*4+8]\n ret\n";
auto mod = X86AssemblyParser::parseX86(src);
auto* fn = static_cast<Function*>(mod->getChildren("functions")[0]);
auto* i = static_cast<AssemblyInstruction*>(fn->getChildren("body")[1]);
CHECK(i->memoryOperands.size() == 1, "expected one memory operand");
CHECK(i->memoryOperands[0].baseRegister == "rax", "expected base rax");
CHECK(i->memoryOperands[0].indexRegister == "rbx", "expected index rbx");
CHECK(i->memoryOperands[0].scale == 4, "expected scale 4");
CHECK(i->memoryOperands[0].offset == 8, "expected offset 8");
PASS();
}
void test_parse_att_memory_addressing() {
TEST(parse_att_memory_addressing);
std::string src = ".text\nmain:\n movq -8(%rbp,%rbx,4), %rax\n ret\n";
auto mod = X86AssemblyParser::parseX86(src, X86SyntaxMode::ATT);
auto* fn = static_cast<Function*>(mod->getChildren("functions")[0]);
auto* i = static_cast<AssemblyInstruction*>(fn->getChildren("body")[1]);
CHECK(i->memoryOperands.size() == 1, "expected one memory operand");
CHECK(i->memoryOperands[0].baseRegister == "rbp", "expected base rbp");
CHECK(i->memoryOperands[0].indexRegister == "rbx", "expected index rbx");
CHECK(i->memoryOperands[0].scale == 4, "expected scale 4");
CHECK(i->memoryOperands[0].offset == -8, "expected offset -8");
PASS();
}
void test_parse_ignores_comments() {
TEST(parse_ignores_comments);
std::string src = ".text ; section\nmain: ; label\n mov rax, rbx ; comment\n ret\n";
auto mod = X86AssemblyParser::parseX86(src);
auto* fn = static_cast<Function*>(mod->getChildren("functions")[0]);
CHECK(fn->getChildren("body").size() >= 3, "expected parsed nodes despite comments");
PASS();
}
void test_known_opcode_set_covers_control_flow_and_stack() {
TEST(known_opcode_set_covers_control_flow_and_stack);
std::string src = ".text\nmain:\n push rax\n call foo\n jmp done\n pop rax\n ret\nfoo:\n ret\ndone:\n ret\n";
auto res = X86AssemblyParser::parseX86WithDiagnostics(src);
CHECK(res.diagnostics.empty(), "expected no unknown opcode warnings");
PASS();
}
void test_unknown_opcode_emits_warning() {
TEST(unknown_opcode_emits_warning);
std::string src = ".text\nmain:\n frobnicate rax, rbx\n ret\n";
auto res = X86AssemblyParser::parseX86WithDiagnostics(src);
CHECK(!res.diagnostics.empty(), "expected warning diagnostics");
CHECK(res.diagnostics[0].severity == "warning", "expected warning severity");
PASS();
}
void test_parse_text_data_bss_sections() {
TEST(parse_text_data_bss_sections);
std::string src = ".text\nmain:\n ret\n.data\nx: .word 1\n.bss\ny: .byte 0\n";
auto mod = X86AssemblyParser::parseX86(src);
bool sawText = false, sawData = false;
for (auto* s : mod->getChildren("statements")) {
if (s->conceptType != "NamespaceDeclaration") continue;
auto* ns = static_cast<NamespaceDeclaration*>(s);
if (ns->name == ".text") sawText = true;
if (ns->name == ".data") sawData = true;
}
CHECK(sawText, "expected .text namespace");
CHECK(sawData, "expected .data namespace");
PASS();
}
int main() {
std::cout << "Step 461: x86 Assembly Parser Tests\n";
test_parse_intel_basic_instructions(); // 1
test_parse_att_basic_instructions(); // 2
test_parse_labels_and_global_flags(); // 3
test_parse_directives_and_sections_as_namespaces(); // 4
test_register_recognition_sizes(); // 5
test_parse_intel_memory_base_offset(); // 6
test_parse_intel_memory_base_index_scale_offset(); // 7
test_parse_att_memory_addressing(); // 8
test_parse_ignores_comments(); // 9
test_known_opcode_set_covers_control_flow_and_stack(); // 10
test_unknown_opcode_emits_warning(); // 11
test_parse_text_data_bss_sections(); // 12
std::cout << "\nResults: " << passed << "/" << (passed + failed)
<< " passed\n";
return failed == 0 ? 0 : 1;
}

View File

@@ -6065,3 +6065,40 @@ memory operands.
**Architecture gate check:**
- `editor/src/ast/AssemblyNodes.h` within header-size limit (`200` <= `600`)
- `editor/tests/step460_test.cpp` within test-file size guidance (`160` lines)
### Step 461: x86 Assembly Parser
**Status:** PASS (12/12 tests)
Implements x86 assembly parsing with Intel/AT&T syntax mode support, including
labels, directives, core instructions, register recognition, and memory
addressing extraction.
**Files added:**
- `editor/src/ast/X86AssemblyParser.h` — x86 parser:
- syntax-mode support (`Intel`, `ATT`)
- parses directives (`.section`, `.global`, `.data`, `.text`, etc.)
- parses labels into function shells (`label + instruction body`)
- parses instructions with operand splitting respecting address delimiters
- register recognition + inferred register size (8/16/32/64)
- Intel memory addressing parse (`[rbp-8]`, `[rax+rbx*4+8]`)
- AT&T memory addressing parse (`-8(%rbp,%rbx,4)`)
- warning diagnostics for unknown/unrecognized opcode lines
- section directives represented as namespace-like nodes (`NamespaceDeclaration`)
- `editor/tests/step461_test.cpp` — 12 tests covering:
- Intel and AT&T instruction parsing
- global label detection
- directive/section parsing behavior
- register-size recognition across register families
- Intel + AT&T memory addressing extraction
- comment stripping behavior
- known-opcode vs unknown-opcode diagnostics
- `editor/CMakeLists.txt``step461_test` target
**Verification run:**
- `cmake --build editor/build-native --target step461_test` — PASS
- `./editor/build-native/step461_test` — PASS (12/12)
- `./editor/build-native/step460_test` — PASS (12/12) regression coverage
**Architecture gate check:**
- `editor/src/ast/X86AssemblyParser.h` within header-size limit (`299` <= `600`)
- `editor/tests/step461_test.cpp` within test-file size guidance (`186` lines)