bill/whetstone_DSL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Step 411: add standalone PostgreSQL parser and routing

Browse Source
This commit is contained in:
Bill
2026-02-16 15:40:06 -07:00
parent c0f7770dcf
commit 158e46895b
6 changed files with 536 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
editor/CMakeLists.txt
View File

@@ -2578,4 +2578,13 @@ target_link_libraries(step410_test PRIVATE
tree_sitter_javascript tree_sitter_typescript
tree_sitter_java tree_sitter_rust tree_sitter_go)
add_executable(step411_test tests/step411_test.cpp)
target_include_directories(step411_test PRIVATE src)
target_link_libraries(step411_test PRIVATE
nlohmann_json::nlohmann_json
unofficial::tree-sitter::tree-sitter
tree_sitter_python tree_sitter_cpp tree_sitter_elisp
tree_sitter_javascript tree_sitter_typescript
tree_sitter_java tree_sitter_rust tree_sitter_go)
# Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies)

4
editor/src/Pipeline.h
View File

@@ -126,6 +126,10 @@ public:
auto pr = VBNetParser::parseVBNetWithDiagnostics(source);
diags = std::move(pr.diagnostics);
return std::move(pr.module);
} else if (language == "postgresql" || language == "postgres") {
auto pr = PostgreSQLParser::parsePostgreSQLWithDiagnostics(source);
diags = std::move(pr.diagnostics);
return std::move(pr.module);
} else if (language == "c") {
auto pr = CParser::parseCWithDiagnostics(source);
diags = std::move(pr.diagnostics);

1
editor/src/ast/Parser.h
View File

@@ -166,6 +166,7 @@ private:
#include "ast/CSharpParser.h"
#include "ast/FSharpParser.h"
#include "ast/VBNetParser.h"
#include "ast/PostgreSQLParser.h"
#include "ast/CParser.h"
#include "ast/WatParser.h"
#include "ast/CommonLispParser.h"

288
editor/src/ast/PostgreSQLParser.h Normal file
View File

@@ -0,0 +1,288 @@
#pragma once
#include "ASTNode.h"
#include "Module.h"
#include "Function.h"
#include "SqlNodes.h"
#include "../ast/Parser.h"
#include <string>
#include <sstream>
#include <vector>
#include <algorithm>
class PostgreSQLParser {
public:
static std::unique_ptr<Module> parsePostgreSQL(const std::string& source) {
auto module = std::make_unique<Module>();
module->id = IdGenerator::next("mod");
module->name = "parsed_postgresql_module";
module->targetLanguage = "postgresql";
parseStatements(source, module.get());
return module;
}
static ParseResult parsePostgreSQLWithDiagnostics(const std::string& source) {
ParseResult result;
result.module = parsePostgreSQL(source);
return result;
}
private:
static void parseStatements(const std::string& source, Module* module) {
for (const auto& stmtRaw : splitStatements(source)) {
std::string stmt = trim(stmtRaw);
if (stmt.empty()) continue;
std::string lower = toLower(stmt);
if (startsWith(lower, "create table ")) {
parseCreateTable(stmt, module);
continue;
}
if (startsWith(lower, "create index ") || startsWith(lower, "create unique index ")) {
parseCreateIndex(stmt, module);
continue;
}
if (startsWith(lower, "select ")) {
parseSelect(stmt, module);
continue;
}
if (startsWith(lower, "insert into ")) {
parseInsert(stmt, module);
continue;
}
if (startsWith(lower, "update ")) {
parseUpdate(stmt, module);
continue;
}
if (startsWith(lower, "delete from ")) {
parseDelete(stmt, module);
continue;
}
if (startsWith(lower, "do $$") || startsWith(lower, "create function ")) {
auto* fn = new Function();
fn->id = IdGenerator::next("fn");
if (startsWith(lower, "do $$")) {
fn->name = "do_block";
} else {
std::string raw = extractAfterKeyword(stmt, "FUNCTION ");
fn->name = trim(untilDelimiter(raw, "("));
}
module->addChild("functions", fn);
continue;
}
}
}
static void parseCreateTable(const std::string& stmt, Module* module) {
std::string tableName = extractAfterKeyword(stmt, "TABLE ");
tableName = trim(untilDelimiter(tableName, "("));
auto* table = new TableDeclaration();
table->id = IdGenerator::next("table");
table->name = tableName;
auto dotPos = tableName.find('.');
if (dotPos != std::string::npos) {
table->schema = tableName.substr(0, dotPos);
}
auto lp = stmt.find('(');
auto rp = stmt.rfind(')');
if (lp != std::string::npos && rp != std::string::npos && rp > lp) {
std::string cols = stmt.substr(lp + 1, rp - lp - 1);
std::stringstream ss(cols);
std::string item;
while (std::getline(ss, item, ',')) {
std::string colLine = trim(item);
if (colLine.empty()) continue;
if (startsWith(toLower(colLine), "constraint ") ||
startsWith(toLower(colLine), "primary key") ||
startsWith(toLower(colLine), "foreign key")) continue;
std::vector<std::string> parts = splitWhitespace(colLine);
if (parts.size() < 2) continue;
auto* col = new ColumnDefinition();
col->id = IdGenerator::next("col");
col->name = parts[0];
col->dataType = parts[1];
col->nullable = (toLower(colLine).find("not null") == std::string::npos);
table->addChild("columns", col);
}
}
module->addChild("statements", table);
}
static void parseCreateIndex(const std::string& stmt, Module* module) {
std::string lower = toLower(stmt);
bool unique = startsWith(lower, "create unique index ");
std::string idxName = unique ? extractAfterKeyword(stmt, "INDEX ") : extractAfterKeyword(stmt, "INDEX ");
std::string tableName = extractAfterKeyword(stmt, "ON ");
auto* idx = new IndexDefinition();
idx->id = IdGenerator::next("idx");
idx->name = trim(untilDelimiter(idxName, " ON"));
idx->tableName = trim(untilDelimiter(tableName, "("));
idx->unique = unique;
module->addChild("statements", idx);
}
static void parseSelect(const std::string& stmt, Module* module) {
std::string lower = toLower(stmt);
auto* q = new SelectQuery();
q->id = IdGenerator::next("sel");
q->distinct = (lower.find("select distinct") == 0);
auto fromPos = lower.find(" from ");
if (fromPos != std::string::npos) {
std::string afterFrom = trim(stmt.substr(fromPos + 6));
std::string fromName = firstToken(afterFrom);
auto* from = new TableDeclaration();
from->id = IdGenerator::next("tbl");
from->name = fromName;
q->setChild("from", from);
}
size_t search = 0;
while (true) {
auto joinPos = lower.find(" join ", search);
if (joinPos == std::string::npos) break;
auto tableStart = joinPos + 6;
std::string joinRest = stmt.substr(tableStart);
std::string tableName = firstToken(joinRest);
auto* j = new JoinClause();
j->id = IdGenerator::next("join");
j->joinType = detectJoinType(lower, joinPos);
j->tableName = tableName;
q->addChild("joins", j);
search = tableStart + 1;
}
auto wherePos = lower.find(" where ");
if (wherePos != std::string::npos) {
auto* w = new WhereClause();
w->id = IdGenerator::next("where");
w->expression = trim(stmt.substr(wherePos + 7));
q->setChild("where", w);
}
module->addChild("statements", q);
}
static void parseInsert(const std::string& stmt, Module* module) {
auto* ins = new InsertStatement();
ins->id = IdGenerator::next("ins");
ins->tableName = extractAfterKeyword(stmt, "INTO ");
ins->tableName = trim(untilDelimiter(ins->tableName, "("));
module->addChild("statements", ins);
}
static void parseUpdate(const std::string& stmt, Module* module) {
auto* up = new UpdateStatement();
up->id = IdGenerator::next("upd");
up->tableName = trim(untilDelimiter(extractAfterKeyword(stmt, "UPDATE "), " SET"));
std::string lower = toLower(stmt);
auto wherePos = lower.find(" where ");
if (wherePos != std::string::npos) {
auto* w = new WhereClause();
w->id = IdGenerator::next("where");
w->expression = trim(stmt.substr(wherePos + 7));
up->setChild("where", w);
}
module->addChild("statements", up);
}
static void parseDelete(const std::string& stmt, Module* module) {
auto* del = new DeleteStatement();
del->id = IdGenerator::next("del");
del->tableName = trim(untilDelimiter(extractAfterKeyword(stmt, "FROM "), " WHERE"));
std::string lower = toLower(stmt);
auto wherePos = lower.find(" where ");
if (wherePos != std::string::npos) {
auto* w = new WhereClause();
w->id = IdGenerator::next("where");
w->expression = trim(stmt.substr(wherePos + 7));
del->setChild("where", w);
}
module->addChild("statements", del);
}
static std::string detectJoinType(const std::string& lower, size_t joinPos) {
std::string prefix = lower.substr((joinPos > 8 ? joinPos - 8 : 0), 8);
if (prefix.find("left") != std::string::npos) return "LEFT";
if (prefix.find("right") != std::string::npos) return "RIGHT";
if (prefix.find("full") != std::string::npos) return "FULL";
return "INNER";
}
static std::vector<std::string> splitStatements(const std::string& source) {
std::vector<std::string> out;
std::string cur;
int dollarDepth = 0;
for (size_t i = 0; i < source.size(); ++i) {
if (i + 1 < source.size() && source[i] == '$' && source[i + 1] == '$') {
dollarDepth = 1 - dollarDepth;
cur += "$$";
++i;
continue;
}
if (source[i] == ';' && dollarDepth == 0) {
out.push_back(cur);
cur.clear();
continue;
}
cur.push_back(source[i]);
}
if (!trim(cur).empty()) out.push_back(cur);
return out;
}
static std::string extractAfterKeyword(const std::string& s, const std::string& keywordUpper) {
std::string lower = toLower(s);
std::string kw = toLower(keywordUpper);
auto pos = lower.find(kw);
if (pos == std::string::npos) return "";
pos += kw.size();
return trim(s.substr(pos));
}
static std::string firstToken(const std::string& s) {
std::string t = trim(s);
auto end = t.find_first_of(" \t\r\n");
if (end == std::string::npos) return t;
return t.substr(0, end);
}
static std::string untilDelimiter(const std::string& s, const std::string& delimiterUpper) {
std::string lower = toLower(s);
std::string d = toLower(delimiterUpper);
auto pos = lower.find(d);
if (pos == std::string::npos) return s;
return s.substr(0, pos);
}
static std::vector<std::string> splitWhitespace(const std::string& s) {
std::vector<std::string> out;
std::stringstream ss(s);
std::string token;
while (ss >> token) out.push_back(token);
return out;
}
static bool startsWith(const std::string& s, const std::string& prefix) {
return s.rfind(prefix, 0) == 0;
}
static std::string toLower(const std::string& s) {
std::string out = s;
std::transform(out.begin(), out.end(), out.begin(),
[](unsigned char c){ return static_cast<char>(std::tolower(c)); });
return out;
}
static std::string trim(const std::string& s) {
auto start = s.find_first_not_of(" \t\r\n");
if (start == std::string::npos) return "";
auto end = s.find_last_not_of(" \t\r\n");
return s.substr(start, end - start + 1);
}
};

185
editor/tests/step411_test.cpp Normal file
View File

@@ -0,0 +1,185 @@
// Step 411: PostgreSQL Parser Tests (12 tests)
#include "ast/PostgreSQLParser.h"
#include "Pipeline.h"
#include "ast/SqlNodes.h"
#include "ast/Function.h"
#include <iostream>
#include <string>
#include <vector>
static int passed = 0, failed = 0;
#define TEST(name) { std::cout << " " << #name << "... "; }
#define PASS() { std::cout << "PASS\n"; ++passed; }
#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; }
#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {}
template <typename T>
T* firstStatementOf(const Module* mod, const std::string& conceptName) {
for (auto* s : mod->getChildren("statements")) {
if (s->conceptType == conceptName) return static_cast<T*>(s);
}
return nullptr;
}
void test_parse_create_table() {
TEST(parse_create_table);
auto mod = PostgreSQLParser::parsePostgreSQL(
"CREATE TABLE users (id SERIAL PRIMARY KEY, name TEXT NOT NULL);\n");
CHECK(mod != nullptr, "module is null");
auto* t = firstStatementOf<TableDeclaration>(mod.get(), "TableDeclaration");
CHECK(t != nullptr, "expected table declaration");
CHECK(t->name == "users", "expected table users");
CHECK(t->getChildren("columns").size() >= 2, "expected parsed columns");
PASS();
}
void test_parse_select_query() {
TEST(parse_select_query);
auto mod = PostgreSQLParser::parsePostgreSQL("SELECT id, name FROM users;\n");
auto* q = firstStatementOf<SelectQuery>(mod.get(), "SelectQuery");
CHECK(q != nullptr, "expected select query");
CHECK(q->getChild("from") != nullptr, "expected from child");
PASS();
}
void test_parse_insert_statement() {
TEST(parse_insert_statement);
auto mod = PostgreSQLParser::parsePostgreSQL(
"INSERT INTO users (id, name) VALUES (1, 'a');\n");
auto* ins = firstStatementOf<InsertStatement>(mod.get(), "InsertStatement");
CHECK(ins != nullptr, "expected insert");
CHECK(ins->tableName == "users", "expected insert table users");
PASS();
}
void test_parse_update_statement() {
TEST(parse_update_statement);
auto mod = PostgreSQLParser::parsePostgreSQL(
"UPDATE users SET name = 'b' WHERE id = 1;\n");
auto* up = firstStatementOf<UpdateStatement>(mod.get(), "UpdateStatement");
CHECK(up != nullptr, "expected update");
CHECK(up->tableName == "users", "expected update table users");
CHECK(up->getChild("where") != nullptr, "expected where clause");
PASS();
}
void test_parse_delete_statement() {
TEST(parse_delete_statement);
auto mod = PostgreSQLParser::parsePostgreSQL("DELETE FROM users WHERE id = 2;\n");
auto* del = firstStatementOf<DeleteStatement>(mod.get(), "DeleteStatement");
CHECK(del != nullptr, "expected delete");
CHECK(del->tableName == "users", "expected delete table users");
CHECK(del->getChild("where") != nullptr, "expected where");
PASS();
}
void test_parse_join_and_where() {
TEST(parse_join_and_where);
auto mod = PostgreSQLParser::parsePostgreSQL(
"SELECT u.id FROM users u LEFT JOIN orders o ON u.id = o.user_id WHERE u.active = true;\n");
auto* q = firstStatementOf<SelectQuery>(mod.get(), "SelectQuery");
CHECK(q != nullptr, "expected select");
CHECK(!q->getChildren("joins").empty(), "expected join clause");
CHECK(q->getChild("where") != nullptr, "expected where clause");
PASS();
}
void test_parse_postgres_types_serial_jsonb_array() {
TEST(parse_postgres_types_serial_jsonb_array);
auto mod = PostgreSQLParser::parsePostgreSQL(
"CREATE TABLE events (id SERIAL, payload JSONB, tags TEXT[]);\n");
auto* t = firstStatementOf<TableDeclaration>(mod.get(), "TableDeclaration");
CHECK(t != nullptr, "expected table");
auto cols = t->getChildren("columns");
CHECK(cols.size() == 3, "expected 3 columns");
CHECK(static_cast<ColumnDefinition*>(cols[0])->dataType == "SERIAL", "expected SERIAL");
CHECK(static_cast<ColumnDefinition*>(cols[1])->dataType == "JSONB", "expected JSONB");
CHECK(static_cast<ColumnDefinition*>(cols[2])->dataType == "TEXT[]", "expected TEXT[]");
PASS();
}
void test_parse_schema_qualified_names() {
TEST(parse_schema_qualified_names);
auto mod = PostgreSQLParser::parsePostgreSQL(
"CREATE TABLE public.accounts (id BIGINT, email TEXT);\n");
auto* t = firstStatementOf<TableDeclaration>(mod.get(), "TableDeclaration");
CHECK(t != nullptr, "expected table");
CHECK(t->name == "public.accounts", "expected schema-qualified name");
CHECK(t->schema == "public", "expected schema public");
PASS();
}
void test_parse_create_index() {
TEST(parse_create_index);
auto mod = PostgreSQLParser::parsePostgreSQL(
"CREATE UNIQUE INDEX idx_users_email ON users (email);\n");
auto* idx = firstStatementOf<IndexDefinition>(mod.get(), "IndexDefinition");
CHECK(idx != nullptr, "expected index");
CHECK(idx->name == "idx_users_email", "expected index name");
CHECK(idx->tableName == "users", "expected table name");
CHECK(idx->unique, "expected unique index");
PASS();
}
void test_parse_do_block_and_plpgsql_function_markers() {
TEST(parse_do_block_and_plpgsql_function_markers);
auto mod = PostgreSQLParser::parsePostgreSQL(
"DO $$ BEGIN RAISE NOTICE 'x'; END $$;\n"
"CREATE FUNCTION add_one(x INTEGER) RETURNS INTEGER AS $$ BEGIN RETURN x + 1; END; $$ LANGUAGE plpgsql;\n");
auto fns = mod->getChildren("functions");
CHECK(fns.size() >= 2, "expected function markers for DO and CREATE FUNCTION");
bool hasDo = false, hasFn = false;
for (auto* f : fns) {
auto* fn = static_cast<Function*>(f);
if (fn->name == "do_block") hasDo = true;
if (fn->name == "add_one") hasFn = true;
}
CHECK(hasDo && hasFn, "expected both markers");
PASS();
}
void test_parse_with_diagnostics_entrypoint() {
TEST(parse_with_diagnostics_entrypoint);
auto result = PostgreSQLParser::parsePostgreSQLWithDiagnostics(
"SELECT DISTINCT id FROM users;\n");
CHECK(result.module != nullptr, "module null");
auto* q = firstStatementOf<SelectQuery>(result.module.get(), "SelectQuery");
CHECK(q != nullptr, "expected select");
CHECK(q->distinct, "expected distinct flag");
PASS();
}
void test_pipeline_parse_routing_aliases() {
TEST(pipeline_parse_routing_aliases);
Pipeline p;
std::vector<ParseDiagnostic> d1, d2;
auto m1 = p.parse("SELECT id FROM users;\n", "postgresql", d1);
auto m2 = p.parse("SELECT id FROM users;\n", "postgres", d2);
CHECK(m1 != nullptr && m2 != nullptr, "expected parse success");
CHECK(m1->targetLanguage == "postgresql", "expected postgresql target");
CHECK(m2->targetLanguage == "postgresql", "expected alias target");
PASS();
}
int main() {
std::cout << "Step 411: PostgreSQL Parser Tests\n";
test_parse_create_table(); // 1
test_parse_select_query(); // 2
test_parse_insert_statement(); // 3
test_parse_update_statement(); // 4
test_parse_delete_statement(); // 5
test_parse_join_and_where(); // 6
test_parse_postgres_types_serial_jsonb_array(); // 7
test_parse_schema_qualified_names(); // 8
test_parse_create_index(); // 9
test_parse_do_block_and_plpgsql_function_markers(); // 10
test_parse_with_diagnostics_entrypoint(); // 11
test_pipeline_parse_routing_aliases(); // 12
std::cout << "\nResults: " << passed << "/" << (passed + failed)
<< " passed\n";
return failed == 0 ? 0 : 1;
}

49
progress.md
View File

@@ -3915,6 +3915,55 @@ pipeline tooling.
- `editor/src/MCPServer.h` (`1679` > `600`)
- `editor/src/HeadlessAgentRPCHandler.h` (`2623` > `600`)
### Step 411: PostgreSQL Parser
**Status:** PASS (12/12 tests)
Added a standalone PostgreSQL parser with SQL-statement extraction and mapping
to the new SQL AST nodes. Covers DDL/DML basics plus PostgreSQL-specific type
and procedural patterns.
**Files created:**
- `editor/src/ast/PostgreSQLParser.h` — PostgreSQL parse support:
- `parsePostgreSQL(...)` and `parsePostgreSQLWithDiagnostics(...)`
- statement splitting with `$$ ... $$` block awareness
- `CREATE TABLE` -> `TableDeclaration` + `ColumnDefinition`
- `SELECT` -> `SelectQuery` with `from`/`joins`/`where` and `distinct`
- `INSERT`/`UPDATE`/`DELETE` mappings
- `CREATE INDEX`/`CREATE UNIQUE INDEX` -> `IndexDefinition`
- `DO $$` and `CREATE FUNCTION` markers -> `Function` nodes
- `editor/tests/step411_test.cpp` — 12 tests covering:
1. CREATE TABLE parse
2. SELECT parse
3. INSERT parse
4. UPDATE parse
5. DELETE parse
6. JOIN + WHERE parse
7. PostgreSQL types (`SERIAL`, `JSONB`, `TEXT[]`)
8. schema-qualified table names
9. CREATE UNIQUE INDEX parse
10. DO block + PL/pgSQL function marker parse
11. diagnostics entry point
12. pipeline parse routing aliases (`postgresql`, `postgres`)
**Files modified:**
- `editor/src/ast/Parser.h` — include `ast/PostgreSQLParser.h`
- `editor/src/Pipeline.h` — parse routing for `postgresql`, `postgres`
- `editor/CMakeLists.txt` — `step411_test` target
**Verification run:**
- `step411_test` — PASS (12/12) new step coverage
- `step410_test` — PASS (12/12) regression coverage
- `step409_test` — PASS (8/8) regression coverage
**Architecture gate check:**
- `editor/src/ast/PostgreSQLParser.h` within header-size limit (`288` <= `600`)
- `editor/tests/step411_test.cpp` within test-file size guidance (`185` lines)
- `editor/src/Pipeline.h` within header-size limit (`251` <= `600`)
- Legacy oversized headers persist:
- `editor/src/ast/Serialization.h` (`1427` > `600`)
- `editor/src/MCPServer.h` (`1679` > `600`)
- `editor/src/HeadlessAgentRPCHandler.h` (`2623` > `600`)
# Roadmap Planning — Sprints 12-25+
## Status: Planning Complete (Sprints 12-19 detailed, 20-25 in roadmap.md)