diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 57de58b..541ea72 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -2578,4 +2578,13 @@ target_link_libraries(step410_test PRIVATE tree_sitter_javascript tree_sitter_typescript tree_sitter_java tree_sitter_rust tree_sitter_go) +add_executable(step411_test tests/step411_test.cpp) +target_include_directories(step411_test PRIVATE src) +target_link_libraries(step411_test PRIVATE + nlohmann_json::nlohmann_json + unofficial::tree-sitter::tree-sitter + tree_sitter_python tree_sitter_cpp tree_sitter_elisp + tree_sitter_javascript tree_sitter_typescript + tree_sitter_java tree_sitter_rust tree_sitter_go) + # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies) diff --git a/editor/src/Pipeline.h b/editor/src/Pipeline.h index 468e637..77e5253 100644 --- a/editor/src/Pipeline.h +++ b/editor/src/Pipeline.h @@ -126,6 +126,10 @@ public: auto pr = VBNetParser::parseVBNetWithDiagnostics(source); diags = std::move(pr.diagnostics); return std::move(pr.module); + } else if (language == "postgresql" || language == "postgres") { + auto pr = PostgreSQLParser::parsePostgreSQLWithDiagnostics(source); + diags = std::move(pr.diagnostics); + return std::move(pr.module); } else if (language == "c") { auto pr = CParser::parseCWithDiagnostics(source); diags = std::move(pr.diagnostics); diff --git a/editor/src/ast/Parser.h b/editor/src/ast/Parser.h index b0ffcee..0cbcec8 100644 --- a/editor/src/ast/Parser.h +++ b/editor/src/ast/Parser.h @@ -166,6 +166,7 @@ private: #include "ast/CSharpParser.h" #include "ast/FSharpParser.h" #include "ast/VBNetParser.h" +#include "ast/PostgreSQLParser.h" #include "ast/CParser.h" #include "ast/WatParser.h" #include "ast/CommonLispParser.h" diff --git a/editor/src/ast/PostgreSQLParser.h b/editor/src/ast/PostgreSQLParser.h new file mode 100644 index 0000000..64f8f24 --- /dev/null +++ b/editor/src/ast/PostgreSQLParser.h @@ -0,0 +1,288 @@ +#pragma once + +#include "ASTNode.h" +#include "Module.h" +#include "Function.h" +#include "SqlNodes.h" +#include "../ast/Parser.h" + +#include +#include +#include +#include + +class PostgreSQLParser { +public: + static std::unique_ptr parsePostgreSQL(const std::string& source) { + auto module = std::make_unique(); + module->id = IdGenerator::next("mod"); + module->name = "parsed_postgresql_module"; + module->targetLanguage = "postgresql"; + parseStatements(source, module.get()); + return module; + } + + static ParseResult parsePostgreSQLWithDiagnostics(const std::string& source) { + ParseResult result; + result.module = parsePostgreSQL(source); + return result; + } + +private: + static void parseStatements(const std::string& source, Module* module) { + for (const auto& stmtRaw : splitStatements(source)) { + std::string stmt = trim(stmtRaw); + if (stmt.empty()) continue; + std::string lower = toLower(stmt); + + if (startsWith(lower, "create table ")) { + parseCreateTable(stmt, module); + continue; + } + if (startsWith(lower, "create index ") || startsWith(lower, "create unique index ")) { + parseCreateIndex(stmt, module); + continue; + } + if (startsWith(lower, "select ")) { + parseSelect(stmt, module); + continue; + } + if (startsWith(lower, "insert into ")) { + parseInsert(stmt, module); + continue; + } + if (startsWith(lower, "update ")) { + parseUpdate(stmt, module); + continue; + } + if (startsWith(lower, "delete from ")) { + parseDelete(stmt, module); + continue; + } + if (startsWith(lower, "do $$") || startsWith(lower, "create function ")) { + auto* fn = new Function(); + fn->id = IdGenerator::next("fn"); + if (startsWith(lower, "do $$")) { + fn->name = "do_block"; + } else { + std::string raw = extractAfterKeyword(stmt, "FUNCTION "); + fn->name = trim(untilDelimiter(raw, "(")); + } + module->addChild("functions", fn); + continue; + } + } + } + + static void parseCreateTable(const std::string& stmt, Module* module) { + std::string tableName = extractAfterKeyword(stmt, "TABLE "); + tableName = trim(untilDelimiter(tableName, "(")); + auto* table = new TableDeclaration(); + table->id = IdGenerator::next("table"); + table->name = tableName; + + auto dotPos = tableName.find('.'); + if (dotPos != std::string::npos) { + table->schema = tableName.substr(0, dotPos); + } + + auto lp = stmt.find('('); + auto rp = stmt.rfind(')'); + if (lp != std::string::npos && rp != std::string::npos && rp > lp) { + std::string cols = stmt.substr(lp + 1, rp - lp - 1); + std::stringstream ss(cols); + std::string item; + while (std::getline(ss, item, ',')) { + std::string colLine = trim(item); + if (colLine.empty()) continue; + if (startsWith(toLower(colLine), "constraint ") || + startsWith(toLower(colLine), "primary key") || + startsWith(toLower(colLine), "foreign key")) continue; + + std::vector parts = splitWhitespace(colLine); + if (parts.size() < 2) continue; + auto* col = new ColumnDefinition(); + col->id = IdGenerator::next("col"); + col->name = parts[0]; + col->dataType = parts[1]; + col->nullable = (toLower(colLine).find("not null") == std::string::npos); + table->addChild("columns", col); + } + } + module->addChild("statements", table); + } + + static void parseCreateIndex(const std::string& stmt, Module* module) { + std::string lower = toLower(stmt); + bool unique = startsWith(lower, "create unique index "); + std::string idxName = unique ? extractAfterKeyword(stmt, "INDEX ") : extractAfterKeyword(stmt, "INDEX "); + std::string tableName = extractAfterKeyword(stmt, "ON "); + + auto* idx = new IndexDefinition(); + idx->id = IdGenerator::next("idx"); + idx->name = trim(untilDelimiter(idxName, " ON")); + idx->tableName = trim(untilDelimiter(tableName, "(")); + idx->unique = unique; + module->addChild("statements", idx); + } + + static void parseSelect(const std::string& stmt, Module* module) { + std::string lower = toLower(stmt); + auto* q = new SelectQuery(); + q->id = IdGenerator::next("sel"); + q->distinct = (lower.find("select distinct") == 0); + + auto fromPos = lower.find(" from "); + if (fromPos != std::string::npos) { + std::string afterFrom = trim(stmt.substr(fromPos + 6)); + std::string fromName = firstToken(afterFrom); + auto* from = new TableDeclaration(); + from->id = IdGenerator::next("tbl"); + from->name = fromName; + q->setChild("from", from); + } + + size_t search = 0; + while (true) { + auto joinPos = lower.find(" join ", search); + if (joinPos == std::string::npos) break; + auto tableStart = joinPos + 6; + std::string joinRest = stmt.substr(tableStart); + std::string tableName = firstToken(joinRest); + + auto* j = new JoinClause(); + j->id = IdGenerator::next("join"); + j->joinType = detectJoinType(lower, joinPos); + j->tableName = tableName; + q->addChild("joins", j); + search = tableStart + 1; + } + + auto wherePos = lower.find(" where "); + if (wherePos != std::string::npos) { + auto* w = new WhereClause(); + w->id = IdGenerator::next("where"); + w->expression = trim(stmt.substr(wherePos + 7)); + q->setChild("where", w); + } + module->addChild("statements", q); + } + + static void parseInsert(const std::string& stmt, Module* module) { + auto* ins = new InsertStatement(); + ins->id = IdGenerator::next("ins"); + ins->tableName = extractAfterKeyword(stmt, "INTO "); + ins->tableName = trim(untilDelimiter(ins->tableName, "(")); + module->addChild("statements", ins); + } + + static void parseUpdate(const std::string& stmt, Module* module) { + auto* up = new UpdateStatement(); + up->id = IdGenerator::next("upd"); + up->tableName = trim(untilDelimiter(extractAfterKeyword(stmt, "UPDATE "), " SET")); + std::string lower = toLower(stmt); + auto wherePos = lower.find(" where "); + if (wherePos != std::string::npos) { + auto* w = new WhereClause(); + w->id = IdGenerator::next("where"); + w->expression = trim(stmt.substr(wherePos + 7)); + up->setChild("where", w); + } + module->addChild("statements", up); + } + + static void parseDelete(const std::string& stmt, Module* module) { + auto* del = new DeleteStatement(); + del->id = IdGenerator::next("del"); + del->tableName = trim(untilDelimiter(extractAfterKeyword(stmt, "FROM "), " WHERE")); + std::string lower = toLower(stmt); + auto wherePos = lower.find(" where "); + if (wherePos != std::string::npos) { + auto* w = new WhereClause(); + w->id = IdGenerator::next("where"); + w->expression = trim(stmt.substr(wherePos + 7)); + del->setChild("where", w); + } + module->addChild("statements", del); + } + + static std::string detectJoinType(const std::string& lower, size_t joinPos) { + std::string prefix = lower.substr((joinPos > 8 ? joinPos - 8 : 0), 8); + if (prefix.find("left") != std::string::npos) return "LEFT"; + if (prefix.find("right") != std::string::npos) return "RIGHT"; + if (prefix.find("full") != std::string::npos) return "FULL"; + return "INNER"; + } + + static std::vector splitStatements(const std::string& source) { + std::vector out; + std::string cur; + int dollarDepth = 0; + for (size_t i = 0; i < source.size(); ++i) { + if (i + 1 < source.size() && source[i] == '$' && source[i + 1] == '$') { + dollarDepth = 1 - dollarDepth; + cur += "$$"; + ++i; + continue; + } + if (source[i] == ';' && dollarDepth == 0) { + out.push_back(cur); + cur.clear(); + continue; + } + cur.push_back(source[i]); + } + if (!trim(cur).empty()) out.push_back(cur); + return out; + } + + static std::string extractAfterKeyword(const std::string& s, const std::string& keywordUpper) { + std::string lower = toLower(s); + std::string kw = toLower(keywordUpper); + auto pos = lower.find(kw); + if (pos == std::string::npos) return ""; + pos += kw.size(); + return trim(s.substr(pos)); + } + + static std::string firstToken(const std::string& s) { + std::string t = trim(s); + auto end = t.find_first_of(" \t\r\n"); + if (end == std::string::npos) return t; + return t.substr(0, end); + } + + static std::string untilDelimiter(const std::string& s, const std::string& delimiterUpper) { + std::string lower = toLower(s); + std::string d = toLower(delimiterUpper); + auto pos = lower.find(d); + if (pos == std::string::npos) return s; + return s.substr(0, pos); + } + + static std::vector splitWhitespace(const std::string& s) { + std::vector out; + std::stringstream ss(s); + std::string token; + while (ss >> token) out.push_back(token); + return out; + } + + static bool startsWith(const std::string& s, const std::string& prefix) { + return s.rfind(prefix, 0) == 0; + } + + static std::string toLower(const std::string& s) { + std::string out = s; + std::transform(out.begin(), out.end(), out.begin(), + [](unsigned char c){ return static_cast(std::tolower(c)); }); + return out; + } + + static std::string trim(const std::string& s) { + auto start = s.find_first_not_of(" \t\r\n"); + if (start == std::string::npos) return ""; + auto end = s.find_last_not_of(" \t\r\n"); + return s.substr(start, end - start + 1); + } +}; diff --git a/editor/tests/step411_test.cpp b/editor/tests/step411_test.cpp new file mode 100644 index 0000000..b0a836d --- /dev/null +++ b/editor/tests/step411_test.cpp @@ -0,0 +1,185 @@ +// Step 411: PostgreSQL Parser Tests (12 tests) + +#include "ast/PostgreSQLParser.h" +#include "Pipeline.h" +#include "ast/SqlNodes.h" +#include "ast/Function.h" + +#include +#include +#include + +static int passed = 0, failed = 0; +#define TEST(name) { std::cout << " " << #name << "... "; } +#define PASS() { std::cout << "PASS\n"; ++passed; } +#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; } +#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {} + +template +T* firstStatementOf(const Module* mod, const std::string& conceptName) { + for (auto* s : mod->getChildren("statements")) { + if (s->conceptType == conceptName) return static_cast(s); + } + return nullptr; +} + +void test_parse_create_table() { + TEST(parse_create_table); + auto mod = PostgreSQLParser::parsePostgreSQL( + "CREATE TABLE users (id SERIAL PRIMARY KEY, name TEXT NOT NULL);\n"); + CHECK(mod != nullptr, "module is null"); + auto* t = firstStatementOf(mod.get(), "TableDeclaration"); + CHECK(t != nullptr, "expected table declaration"); + CHECK(t->name == "users", "expected table users"); + CHECK(t->getChildren("columns").size() >= 2, "expected parsed columns"); + PASS(); +} + +void test_parse_select_query() { + TEST(parse_select_query); + auto mod = PostgreSQLParser::parsePostgreSQL("SELECT id, name FROM users;\n"); + auto* q = firstStatementOf(mod.get(), "SelectQuery"); + CHECK(q != nullptr, "expected select query"); + CHECK(q->getChild("from") != nullptr, "expected from child"); + PASS(); +} + +void test_parse_insert_statement() { + TEST(parse_insert_statement); + auto mod = PostgreSQLParser::parsePostgreSQL( + "INSERT INTO users (id, name) VALUES (1, 'a');\n"); + auto* ins = firstStatementOf(mod.get(), "InsertStatement"); + CHECK(ins != nullptr, "expected insert"); + CHECK(ins->tableName == "users", "expected insert table users"); + PASS(); +} + +void test_parse_update_statement() { + TEST(parse_update_statement); + auto mod = PostgreSQLParser::parsePostgreSQL( + "UPDATE users SET name = 'b' WHERE id = 1;\n"); + auto* up = firstStatementOf(mod.get(), "UpdateStatement"); + CHECK(up != nullptr, "expected update"); + CHECK(up->tableName == "users", "expected update table users"); + CHECK(up->getChild("where") != nullptr, "expected where clause"); + PASS(); +} + +void test_parse_delete_statement() { + TEST(parse_delete_statement); + auto mod = PostgreSQLParser::parsePostgreSQL("DELETE FROM users WHERE id = 2;\n"); + auto* del = firstStatementOf(mod.get(), "DeleteStatement"); + CHECK(del != nullptr, "expected delete"); + CHECK(del->tableName == "users", "expected delete table users"); + CHECK(del->getChild("where") != nullptr, "expected where"); + PASS(); +} + +void test_parse_join_and_where() { + TEST(parse_join_and_where); + auto mod = PostgreSQLParser::parsePostgreSQL( + "SELECT u.id FROM users u LEFT JOIN orders o ON u.id = o.user_id WHERE u.active = true;\n"); + auto* q = firstStatementOf(mod.get(), "SelectQuery"); + CHECK(q != nullptr, "expected select"); + CHECK(!q->getChildren("joins").empty(), "expected join clause"); + CHECK(q->getChild("where") != nullptr, "expected where clause"); + PASS(); +} + +void test_parse_postgres_types_serial_jsonb_array() { + TEST(parse_postgres_types_serial_jsonb_array); + auto mod = PostgreSQLParser::parsePostgreSQL( + "CREATE TABLE events (id SERIAL, payload JSONB, tags TEXT[]);\n"); + auto* t = firstStatementOf(mod.get(), "TableDeclaration"); + CHECK(t != nullptr, "expected table"); + auto cols = t->getChildren("columns"); + CHECK(cols.size() == 3, "expected 3 columns"); + CHECK(static_cast(cols[0])->dataType == "SERIAL", "expected SERIAL"); + CHECK(static_cast(cols[1])->dataType == "JSONB", "expected JSONB"); + CHECK(static_cast(cols[2])->dataType == "TEXT[]", "expected TEXT[]"); + PASS(); +} + +void test_parse_schema_qualified_names() { + TEST(parse_schema_qualified_names); + auto mod = PostgreSQLParser::parsePostgreSQL( + "CREATE TABLE public.accounts (id BIGINT, email TEXT);\n"); + auto* t = firstStatementOf(mod.get(), "TableDeclaration"); + CHECK(t != nullptr, "expected table"); + CHECK(t->name == "public.accounts", "expected schema-qualified name"); + CHECK(t->schema == "public", "expected schema public"); + PASS(); +} + +void test_parse_create_index() { + TEST(parse_create_index); + auto mod = PostgreSQLParser::parsePostgreSQL( + "CREATE UNIQUE INDEX idx_users_email ON users (email);\n"); + auto* idx = firstStatementOf(mod.get(), "IndexDefinition"); + CHECK(idx != nullptr, "expected index"); + CHECK(idx->name == "idx_users_email", "expected index name"); + CHECK(idx->tableName == "users", "expected table name"); + CHECK(idx->unique, "expected unique index"); + PASS(); +} + +void test_parse_do_block_and_plpgsql_function_markers() { + TEST(parse_do_block_and_plpgsql_function_markers); + auto mod = PostgreSQLParser::parsePostgreSQL( + "DO $$ BEGIN RAISE NOTICE 'x'; END $$;\n" + "CREATE FUNCTION add_one(x INTEGER) RETURNS INTEGER AS $$ BEGIN RETURN x + 1; END; $$ LANGUAGE plpgsql;\n"); + auto fns = mod->getChildren("functions"); + CHECK(fns.size() >= 2, "expected function markers for DO and CREATE FUNCTION"); + bool hasDo = false, hasFn = false; + for (auto* f : fns) { + auto* fn = static_cast(f); + if (fn->name == "do_block") hasDo = true; + if (fn->name == "add_one") hasFn = true; + } + CHECK(hasDo && hasFn, "expected both markers"); + PASS(); +} + +void test_parse_with_diagnostics_entrypoint() { + TEST(parse_with_diagnostics_entrypoint); + auto result = PostgreSQLParser::parsePostgreSQLWithDiagnostics( + "SELECT DISTINCT id FROM users;\n"); + CHECK(result.module != nullptr, "module null"); + auto* q = firstStatementOf(result.module.get(), "SelectQuery"); + CHECK(q != nullptr, "expected select"); + CHECK(q->distinct, "expected distinct flag"); + PASS(); +} + +void test_pipeline_parse_routing_aliases() { + TEST(pipeline_parse_routing_aliases); + Pipeline p; + std::vector d1, d2; + auto m1 = p.parse("SELECT id FROM users;\n", "postgresql", d1); + auto m2 = p.parse("SELECT id FROM users;\n", "postgres", d2); + CHECK(m1 != nullptr && m2 != nullptr, "expected parse success"); + CHECK(m1->targetLanguage == "postgresql", "expected postgresql target"); + CHECK(m2->targetLanguage == "postgresql", "expected alias target"); + PASS(); +} + +int main() { + std::cout << "Step 411: PostgreSQL Parser Tests\n"; + + test_parse_create_table(); // 1 + test_parse_select_query(); // 2 + test_parse_insert_statement(); // 3 + test_parse_update_statement(); // 4 + test_parse_delete_statement(); // 5 + test_parse_join_and_where(); // 6 + test_parse_postgres_types_serial_jsonb_array(); // 7 + test_parse_schema_qualified_names(); // 8 + test_parse_create_index(); // 9 + test_parse_do_block_and_plpgsql_function_markers(); // 10 + test_parse_with_diagnostics_entrypoint(); // 11 + test_pipeline_parse_routing_aliases(); // 12 + + std::cout << "\nResults: " << passed << "/" << (passed + failed) + << " passed\n"; + return failed == 0 ? 0 : 1; +} diff --git a/progress.md b/progress.md index c370319..221219a 100644 --- a/progress.md +++ b/progress.md @@ -3915,6 +3915,55 @@ pipeline tooling. - `editor/src/MCPServer.h` (`1679` > `600`) - `editor/src/HeadlessAgentRPCHandler.h` (`2623` > `600`) +### Step 411: PostgreSQL Parser +**Status:** PASS (12/12 tests) + +Added a standalone PostgreSQL parser with SQL-statement extraction and mapping +to the new SQL AST nodes. Covers DDL/DML basics plus PostgreSQL-specific type +and procedural patterns. + +**Files created:** +- `editor/src/ast/PostgreSQLParser.h` — PostgreSQL parse support: + - `parsePostgreSQL(...)` and `parsePostgreSQLWithDiagnostics(...)` + - statement splitting with `$$ ... $$` block awareness + - `CREATE TABLE` -> `TableDeclaration` + `ColumnDefinition` + - `SELECT` -> `SelectQuery` with `from`/`joins`/`where` and `distinct` + - `INSERT`/`UPDATE`/`DELETE` mappings + - `CREATE INDEX`/`CREATE UNIQUE INDEX` -> `IndexDefinition` + - `DO $$` and `CREATE FUNCTION` markers -> `Function` nodes +- `editor/tests/step411_test.cpp` — 12 tests covering: + 1. CREATE TABLE parse + 2. SELECT parse + 3. INSERT parse + 4. UPDATE parse + 5. DELETE parse + 6. JOIN + WHERE parse + 7. PostgreSQL types (`SERIAL`, `JSONB`, `TEXT[]`) + 8. schema-qualified table names + 9. CREATE UNIQUE INDEX parse + 10. DO block + PL/pgSQL function marker parse + 11. diagnostics entry point + 12. pipeline parse routing aliases (`postgresql`, `postgres`) + +**Files modified:** +- `editor/src/ast/Parser.h` — include `ast/PostgreSQLParser.h` +- `editor/src/Pipeline.h` — parse routing for `postgresql`, `postgres` +- `editor/CMakeLists.txt` — `step411_test` target + +**Verification run:** +- `step411_test` — PASS (12/12) new step coverage +- `step410_test` — PASS (12/12) regression coverage +- `step409_test` — PASS (8/8) regression coverage + +**Architecture gate check:** +- `editor/src/ast/PostgreSQLParser.h` within header-size limit (`288` <= `600`) +- `editor/tests/step411_test.cpp` within test-file size guidance (`185` lines) +- `editor/src/Pipeline.h` within header-size limit (`251` <= `600`) +- Legacy oversized headers persist: + - `editor/src/ast/Serialization.h` (`1427` > `600`) + - `editor/src/MCPServer.h` (`1679` > `600`) + - `editor/src/HeadlessAgentRPCHandler.h` (`2623` > `600`) + # Roadmap Planning — Sprints 12-25+ ## Status: Planning Complete (Sprints 12-19 detailed, 20-25 in roadmap.md)