diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 9706218..1c72d4a 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -13,7 +13,16 @@ "Bash(python3:*)", "Bash(python:*)", "WebFetch(domain:github.com)", - "WebFetch(domain:java.hotexamples.com)" + "WebFetch(domain:java.hotexamples.com)", + "Bash(dir:*)", + "Bash(cmake:*)", + "Bash(\"E:\\\\Whetstone_DSL\\\\editor\\\\build\\\\Debug\\\\step1_test.exe\")", + "Bash(/e/Whetstone_DSL/editor/build/step1_test.exe)", + "Bash(/e/Whetstone_DSL/editor/build/step2_test.exe)", + "Bash(/e/Whetstone_DSL/editor/build/step3_test.exe)", + "Bash(build/step3_test.exe)", + "Bash(build/step4_test.exe)", + "Bash(/e/Whetstone_DSL/editor/build/step5_test.exe)" ] } } diff --git a/Phase1Example.sandbox/models/ConditionalExample.mps b/Phase1Example.sandbox/models/ConditionalExample.mps new file mode 100644 index 0000000..1a8de2a --- /dev/null +++ b/Phase1Example.sandbox/models/ConditionalExample.mps @@ -0,0 +1,101 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Phase1Example.sandbox/models/Phase1Test.mps b/Phase1Example.sandbox/models/Phase1Test.mps index edd8a2d..732a9e1 100644 --- a/Phase1Example.sandbox/models/Phase1Test.mps +++ b/Phase1Example.sandbox/models/Phase1Test.mps @@ -12,7 +12,7 @@ - + @@ -35,12 +35,12 @@ + - - + @@ -52,7 +52,7 @@ - + @@ -76,24 +76,23 @@ - - + - + - + - + - + @@ -119,10 +118,10 @@ - + - + diff --git a/Phase1Example.sandbox/models/SimpleFunctionExample.mps b/Phase1Example.sandbox/models/SimpleFunctionExample.mps new file mode 100644 index 0000000..5350755 --- /dev/null +++ b/Phase1Example.sandbox/models/SimpleFunctionExample.mps @@ -0,0 +1,98 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/BuildCacheExample.md b/docs/BuildCacheExample.md new file mode 100644 index 0000000..9b452f3 --- /dev/null +++ b/docs/BuildCacheExample.md @@ -0,0 +1,16 @@ +# Building a Cache - Whetstone AST (Canonical Form) + +``` +Function: getOrCreate + @deref(???) + Parameter: cache -> Map + Parameter: key -> string + Body: + IfStatement: + condition: key in cache + then: Return cache[key] + else: + Assignment: widget = Widget.create(key) + Assignment: cache[key] = widget + Return: widget +``` \ No newline at end of file diff --git a/docs/ProcessRecordsExample.md b/docs/ProcessRecordsExample.md new file mode 100644 index 0000000..f69b7c4 --- /dev/null +++ b/docs/ProcessRecordsExample.md @@ -0,0 +1,14 @@ +# Processing a List of Records - Whetstone AST (Canonical Form) + +``` +Function: processRecords + @deref(???) // Strategy applied here + Parameter: records -> List + Body: + ForLoop: + iterator: record in records + body: + Call: record.validate() + Call: record.transform() + Return: records +``` \ No newline at end of file diff --git a/docs/SPRINT_1_REQUIREMENTS.md b/docs/SPRINT_1_REQUIREMENTS.md index bb7ac15..efd64f8 100644 --- a/docs/SPRINT_1_REQUIREMENTS.md +++ b/docs/SPRINT_1_REQUIREMENTS.md @@ -477,31 +477,31 @@ languages/ Build SemAnno incrementally, testing each phase before moving forward: -1. **Week 1-2: Core AST Structure** +1. **Core AST Structure** - Add core concepts to SemAnno.structure.mps (Module, Function, Variable, Statement, Expression, Type nodes) - Create basic editors in SemAnno.editor.mps for each node type - Manual AST creation works in MPS - Test: Create a simple function with statements and expressions -2. **Week 3-4: Python Projection & Generator** +2. **Python Projection & Generator** - Extend SemAnno.editor.mps with Python-syntax projections - Implement Python generator in SemAnno.textGen.mps - Manual round-trip: type Python-like code → see generated .py file - Test: Generate valid, runnable Python code -3. **Week 5-6: C++ Projection & Generator** +3. **C++ Projection & Generator** - Extend SemAnno.editor.mps with C++ syntax projections - Implement C++ generator in SemAnno.textGen.mps - Add deref strategy translation to generator - Test: Generate valid, compilable C++ code for each deref strategy -4. **Week 7-8: Tree-sitter Import** +4. **Tree-sitter Import** - Add import behavior to SemAnno.behavior.mps - Integrate tree-sitter-python for Python parsing - Integrate tree-sitter-cpp for C++ parsing - Test: Parse Python/C++ files → populate SemAnno AST -5. **Week 9-10: Warning System & Annotations** +5. **Warning System & Annotations** - Add OptimizationLock and DerefStrategy annotations to SemAnno.structure.mps - Implement warning logic in SemAnno.behavior.mps - Add warning UI to SemAnno.editor.mps diff --git a/docs/SPRINT_2_PLAN.md b/docs/SPRINT_2_PLAN.md new file mode 100644 index 0000000..e3c0388 --- /dev/null +++ b/docs/SPRINT_2_PLAN.md @@ -0,0 +1,288 @@ +# Sprint 2 Plan: Whetstone Editor Stack + +38 steps. Build one thing, test it, move on. If something breaks, the problem is in that step. + +For detailed architecture and design rationale, see: +- `SPRINT_2_VISION.md` — Architecture diagrams, structured editing UI, agent API design +- `annotations/C++ Implementation Roadmap.md` — Full technical specs for each component (§1–§7) + +--- + +## Phase 2a: AST in C++ + +Port the 33 SemAnno concepts from MPS XML into C++ data structures. + +### Step 1: Base node + one concept +- `ASTNode` base class with ID, concept type tag, parent pointer +- `Module` concept only — properties (name, targetLanguage), empty child vectors +- **Test:** create a Module in C++, verify fields work + +### Step 2: Child links +- Add child link support to `ASTNode` (single-valued and multi-valued) +- Add `Function` and `Variable` concepts as children of Module +- **Test:** build Module → Function tree in code, walk it, verify parent/child pointers + +### Step 3: Remaining concepts +- Add all Statement, Expression, Literal, and Type concepts +- Each one gets its own properties and child link definitions +- **Test:** build the Calculator example as a C++ object graph + +### Step 4: Annotation concepts +- Add `DerefStrategy`, `OptimizationLock`, `LangSpecific` +- Attach to Module/Function/Variable via annotations link +- **Test:** build SimpleFunctionExample with @deref(batched), verify annotation reads back + +### Step 5: JSON serialization (save) +- Serialize the C++ AST graph to JSON +- One function: `toJson(ASTNode*) → nlohmann::json` +- **Test:** build Calculator graph → serialize → inspect JSON output by hand + +### Step 6: JSON deserialization (load) +- Parse JSON back into C++ AST graph +- One function: `fromJson(nlohmann::json) → ASTNode*` +- **Test:** save Calculator → load it back → save again → compare JSON (byte-identical) + +> **CHECKPOINT:** AST round-trips through JSON. Save → load → save produces identical output. Stop here until this passes. + +### Step 7: Schema validation +- `ASTSchema` class: for each concept, list legal child types per role + cardinalities +- Validation function: `isLegalChild(parentConcept, role, childConcept) → bool` +- **Test:** legal placements accepted, illegal placements rejected (e.g., Module as child of IntegerLiteral) + +--- + +## Phase 2b: First Generator + +Python generator — port of the working MPS textGen rules into C++ visitor methods. + +### Step 8: Generator base + Module/Function +- `ProjectionGenerator` base with visitor dispatch +- `PythonGenerator`: generate Module header and Function signatures only +- **Test:** Calculator AST → Python output has correct `def add(x, y):` signature + +### Step 9: Statements and expressions +- Add Assignment, Return, BinaryOperation, VariableReference to Python generator +- **Test:** Calculator AST → full Python output matches MPS textGen output + +> **CHECKPOINT:** Python generator matches MPS. Calculator output identical to what MPS produces. Stop here until this passes. + +### Step 10: Remaining Python concepts +- Literals, control flow (If/While/For), types, FunctionCall, IndexAccess, MemberAccess +- **Test:** ConditionalExample AST → correct Python with if/else + +### Step 11: Annotation output +- DerefStrategy → `# @deref(strategy)` comment in Python +- OptimizationLock → `# @lock(...)` comment +- **Test:** SimpleFunctionExample with annotations → comments appear in output + +--- + +## Phase 2c: Dear ImGui Shell + +Pure presentation layer — no business logic. + +### Step 12: Window + empty layout +- Dear ImGui + SDL2 scaffold: opens a window with docking enabled +- Empty panes: file tree (left), editor area (center), panel (bottom) +- **Test:** app launches, window renders, panes are resizable + +### Step 13: Text viewport +- Load a hardcoded Python string into the editor pane +- Render with monospace font, line numbers in gutter +- **Test:** text displays correctly, scrolls + +### Step 14: AST → text viewport +- Connect: load Calculator JSON → build AST → run Python generator → display in viewport +- **Test:** change the JSON file, restart, see different output + +> **CHECKPOINT:** ImGui shows live AST. Change JSON, restart, see different output. Stop here until this passes. + +### Step 15: Projection toggle +- Toolbar with [Python] [AST] buttons +- Python: show generated code. AST: show JSON tree dump +- **Test:** toggle between views, both render correctly + +--- + +## Phase 2d: Orchestrator (Standalone Process) + +The brain — separate C++ process that owns the AST and coordinates everything. + +### Step 16: Orchestrator process + AST ownership +- Separate C++ process that loads AST from JSON and holds it in memory +- Exposes nothing yet — just loads, holds, saves on exit +- **Test:** start orchestrator with Calculator.json, kill it, verify JSON saved back + +### Step 17: JSON-RPC server +- Orchestrator listens on a Unix socket (or named pipe on Windows) +- Responds to `ping` and `getAST` methods +- **Test:** send JSON-RPC from a Python script, get response + +### Step 18: AST mutation via RPC +- Add `setProperty` method: change a node's property value via RPC +- Validate against schema before applying +- **Test:** change Calculator function name via RPC, query AST back, see new name + +### Step 19: Insert and delete via RPC +- `insertNode(parentId, role, conceptType, properties)` — creates node, validates, inserts +- `deleteNode(nodeId)` — removes node and subtree +- **Test:** insert a new Parameter via RPC, delete it, verify AST integrity + +### Step 20: Undo/redo +- Operation journal: each mutation recorded as a reversible op +- `undo` and `redo` RPC methods +- **Test:** insert node → undo → node gone → redo → node back + +### Step 21: Connect ImGui to orchestrator +- ImGui shell connects to orchestrator via JSON-RPC instead of loading JSON directly +- Requests AST on startup, displays projection +- **Test:** launch orchestrator, launch ImGui shell, see Calculator output + +> **CHECKPOINT:** ImGui talks to orchestrator. Two processes running, viewport shows AST content. Stop here until this passes. + +--- + +## Phase 2e: Structured Editing + +Build code by choosing from valid options — no typing syntax. + +### Step 22: Construct chooser (read-only) +- ImGui panel queries orchestrator: `getValidConstructs(parentId, role)` → list of legal types +- Display as button grid +- **Test:** click on Function body → see [Assignment] [Return] [If] [While] [For] etc. + +### Step 23: Insert from chooser +- Click a construct button → orchestrator creates node with default properties +- Viewport refreshes with new projection +- **Test:** add a Return statement to a function, see it in Python output + +### Step 24: Property editing +- Click a node → property panel shows editable fields (name, type, operator, etc.) +- Edit field → `setProperty` RPC → viewport refreshes +- **Test:** rename a function, see output update + +### Step 25: Scope-aware dropdowns +- Orchestrator implements `getInScopeSymbols(nodeId)` → variables, functions, types visible at that position +- VariableReference name field shows dropdown of in-scope variables +- **Test:** inside Calculator.add body, dropdown shows [x, y, result] + +> **CHECKPOINT:** Structured editing works. Build a function entirely through the UI. Stop here until this passes. + +--- + +## Phase 2f: Emacs Integration + +Headless Emacs servers for file ops, search, git, completion. + +### Step 26: Spawn one headless Emacs +- Orchestrator spawns `emacs --daemon=whetstone-file --load whetstone-bridge.el` +- Sends a test command via `emacsclient`, gets response +- **Test:** orchestrator starts, Emacs server responds to ping + +### Step 27: File operations +- `openFile(path)` → routes to Emacs file-server → returns contents +- `saveFile(path, contents)` → writes via Emacs +- **Test:** open a file through orchestrator, verify contents match + +### Step 28: Search +- `search(query)` → routes to Emacs search-server → returns file:line results +- Results displayed in ImGui panel +- **Test:** search for "calculate_sum", get result pointing to SimpleFunctionExample + +### Step 29: Server pool +- Multiple Emacs daemons: file-server, search-server, git-server +- Orchestrator routes by method name +- Health monitoring: restart crashed servers +- **Test:** kill a server, verify orchestrator restarts it and retries the request + +> **CHECKPOINT:** Emacs servers respond. File open, search, and git status all return results. Stop here until this passes. + +--- + +## Phase 2g: Elisp Projection + +Generate Elisp from AST, and parse Elisp into AST. + +### Step 30: Elisp generator — functions and variables +- Module → `(provide 'name)`, Function → `(defun name (params) body)`, Variable → `(defvar name value)` +- **Test:** Calculator AST → valid Elisp that loads in Emacs without errors + +### Step 31: Elisp generator — statements and expressions +- Assignment → `(setq target value)`, Return → value (last expression), If → `(if cond then else)` +- BinaryOperation → `(op left right)`, Literals → values +- **Test:** ConditionalExample AST → Elisp with `(if (= status_code 200) "OK" "Error")` + +### Step 32: Elisp parser (tree-sitter) +- Integrate tree-sitter-elisp, parse a `.el` file into CST +- Map `defun` → Function, `defvar` → Variable, `if` → IfStatement +- **Test:** parse a small hand-written `.el` file → inspect resulting AST + +### Step 33: Elisp round-trip +- Parse `.el` → AST → generate Elisp → compare with original +- Handle `@LangSpecific` for idioms that don't map cleanly +- **Test:** write a 3-function `.el` file, round-trip it, verify semantic equivalence + +> **CHECKPOINT:** Elisp round-trips. Parse .el → AST → generate .el → semantically equivalent. Stop here until this passes. + +--- + +## Phase 2h: C++ Generator + Agent API + +The final projection and the external agent interface. + +### Step 34: C++ generator — basic output +- Module → `#include` guards, Function → typed signatures, variables → declarations +- No memory strategy yet — just syntactically valid C++ +- **Test:** Calculator AST → compilable C++ with g++ + +### Step 35: C++ generator — memory strategies +- `@dealloc(manual)` → raw pointers, `@dealloc(gc)` → shared_ptr, `@dealloc(ownership)` → unique_ptr +- **Test:** same AST with different @dealloc values → different C++ output, all compile + +> **CHECKPOINT:** C++ compiles from AST. Generated C++ compiles with g++ for all 3 example models. Stop here until this passes. + +### Step 36: Agent API +- Orchestrator accepts JSON-RPC from external clients (same protocol as ImGui) +- `getValidConstructs`, `insertNode`, `setProperty`, `getInScopeSymbols` +- **Test:** Python script acts as agent, builds a Function node by node via RPC + +### Step 37: Agent batch mode +- `insertSubtree(parentId, role, jsonSubtree)` — submit entire subtree at once +- Orchestrator validates recursively, applies atomically (all or nothing) +- **Test:** agent submits full ConditionalExample as JSON, orchestrator builds the AST + +> **CHECKPOINT:** Agent builds AST remotely. Python script creates full ConditionalExample via RPC. Stop here until this passes. + +### Step 38: OptimizationLock warnings +- When mutating a locked node, orchestrator returns warning (not rejection) +- ImGui shows lock icon + banner +- Shadow system: original annotation preserved, modification tracked via `@provenance` +- **Test:** lock a function, modify it, verify warning fires and original preserved + +--- + +## Summary + +| Phase | Steps | What You Have When Done | +|-------|-------|------------------------| +| 2a: AST in C++ | 1–7 | 33 concepts as C++ objects, JSON round-trip, schema validation | +| 2b: First Generator | 8–11 | Python output matching MPS for all examples | +| 2c: Dear ImGui Shell | 12–15 | Window showing live AST projection with toggle | +| 2d: Orchestrator | 16–21 | Separate process, RPC mutations, undo/redo, ImGui connected | +| 2e: Structured Editing | 22–25 | Build code through UI, scope-aware dropdowns | +| 2f: Emacs Integration | 26–29 | File ops, search, git via headless Emacs servers | +| 2g: Elisp Projection | 30–33 | Generate and parse Elisp, round-trip verified | +| 2h: C++ + Agent API | 34–38 | C++ generator with memory strategies, agent RPC interface | + +## Dependencies + +| Dependency | Version | Purpose | +|------------|---------|---------| +| Dear ImGui | 1.90+ | GUI rendering | +| SDL2 or GLFW | latest | Window/input backend for ImGui | +| Emacs | 29+ | Headless servers | +| nlohmann/json | 3.11+ | JSON serialization, JSON-RPC | +| tree-sitter | 0.22+ | Elisp/Python/C++ parsing | +| tree-sitter-elisp | latest | Elisp grammar | +| C++ compiler | C++20 | Entire C++ stack | +| GoogleTest | latest | C++ unit tests | diff --git a/docs/annotations/.space/def.json b/docs/annotations/.space/def.json new file mode 100644 index 0000000..9ff9c39 --- /dev/null +++ b/docs/annotations/.space/def.json @@ -0,0 +1 @@ +{"_joins":[],"_contexts":[],"_links":[],"_sort":{"field":"rank","asc":false,"group":false,"recursive":false},"_template":"","_templateName":"","defaultSticker":"","defaultColor":"","readMode":false,"fullWidth":false} \ No newline at end of file diff --git a/docs/annotations/6 optimization and intent.md b/docs/annotations/6 optimization and intent.md new file mode 100644 index 0000000..d7ef204 --- /dev/null +++ b/docs/annotations/6 optimization and intent.md @@ -0,0 +1,70 @@ +# Subject 6: Optimization & Intent Hints + +This document defines the annotations for capturing programmer intent regarding performance, layout, and compiler behavior. These hints allow the projection engine to respect the original developer's optimizations even when moving between radically different execution environments. + +## 1. Function & Execution Hints + +These annotations signal how a specific block of logic should be treated by the target compiler or runtime. + +| | | | +|---|---|---| +|**Annotation**|**Semantic Purpose**|**Example Conflict**| +|**`@Inline(Always|Never|Hint)`**| +|**`@Pure`**|Signals no side effects (referential transparency).|Essential for Haskell-to-imperative transpilation.| +|**`@TailCall`**|Marks a call as eligible for TCO.|Preserves recursion safety when projecting to languages without native TCO.| +|**`@Cold / @Hot`**|Profile-guided hints for branch prediction.|C++ `[[likely]]` / `[[unlikely]]` vs. profile-blind targets.| + +## 2. Loop & Data Parallelism Hints + +For modernizing legacy Fortran or C++, capturing the "shape" of iterative logic is key to generating efficient modern code. + +### A. Loop Transformation (`@Loop`) + +- **`@Loop(Unroll, N)`**: Intent to expand the loop body $N$ times. + +- **`@Loop(Vectorize)`**: Assertion that iterations are independent (safe for SIMD). + +- **`@Loop(Fuse)`**: Hint to combine adjacent loops for better cache locality. + + +### B. Memory Locality (`@Data`) + +- **`@Data(Prefetch)`**: Intent to load data into cache before use. + +- **`@Data(Restrict)`**: Assertion that a pointer is not aliased (C `restrict`). + + +## 3. The "Legacy Performance" Preservation + +In 40 trillion lines of C++, optimizations are often "baked in" to the syntax (e.g., bit-shifting instead of multiplication). + +- **Annotation:** `@Intent(Identity)` + +- **Definition:** Identifies a pattern that is an optimization of a simpler mathematical identity. + +- **Projection Logic:** + + - **Raising:** If the engine sees a bit-shift `x << 1` annotated as `@Intent(Multiply)`, it can project it as `x * 2` in a high-level language, allowing the target compiler to choose the best implementation. + + - **Lowering:** Injects the specific low-level trick into C/Assembly to match the original performance profile. + + +## 4. Hardware Mapping & Layout + +| | | | +|---|---|---| +|**Annotation**|**Semantic Purpose**|**Target Action**| +|**`@Align(N)`**|Force memory alignment to $N$ bytes.|Project to `alignas(N)` in C++ or `__attribute__((aligned(N)))`.| +|**`@Pack`**|Remove all padding between struct members.|Maps to `#pragma pack` in C/C++.| +|**`@ConstExpr`**|Enforce compile-time evaluation.|Project to `constexpr` (C++) or `static final` (Java) if possible.| + +## 5. Security & Safety Overrides + +While modernizing, we often need to trade "blind speed" for safety. + +- **`@BoundsCheck(Disable)`**: Used in legacy code for performance; flagged for review in safe-target projection. + +- **`@Overflow(Wrap | Saturation | Panic)`**: Defines how to handle integer wrap-around, ensuring the target language mimics the source's arithmetic behavior. + + +**Projection Conflict:** If C++ code relies on `unsigned` wrap-around behavior and is projected to a language that panics on overflow, the engine must inject explicit modular arithmetic to remain "lossless." \ No newline at end of file diff --git a/docs/annotations/8 strategy choice and policy.md b/docs/annotations/8 strategy choice and policy.md new file mode 100644 index 0000000..59e94e4 --- /dev/null +++ b/docs/annotations/8 strategy choice and policy.md @@ -0,0 +1,69 @@ +# Subject 8: Strategic Choice & Policy + +This document defines the annotations for "Decision Support." Instead of dictating a modernization strategy, the universal AST tracks multiple valid transformation paths, allowing the "pro" to select the optimal projection based on specific project constraints (e.g., Real-time requirements vs. Memory safety). + +## 1. Divergent Projection Strategies + +When a legacy pattern is identified, the engine generates a "Strategy Menu" rather than a single output. This menu is surfaced in the Whetstone IDE as a set of interactive choices. + +| | | | | +|---|---|---|---| +|**Legacy Node**|**Strategy A: Conservative (Preservation)**|**Strategy B: Progressive (Safe)**|**Strategy C: Aggressive (Modern)**| +|**Raw Pointer**|Raw Pointer (C-style)|Smart Pointer (C++11/14)|Owned/Borrowed (Rust)| +|**Manual Loop**|Preserve exact structure|Auto-vectorize (SIMD)|Functional (map/filter)| +|**Global State**|Preserve as Global|Wrap in Singleton/Namespace|Dependency Injection| +|**Goto/Jumps**|Direct Jump mapping|Structured Loop/Match|State Machine / Continuation| + +## 2. Policy Annotations (`@Policy`) + +These annotations allow the engineer to set "Guardrails" for the projection engine. Policies can be applied globally, per-module, or per-node. + +- **`@Policy(Strictness: High)`**: Rejects any projection that results in unsafe memory access or undefined behavior. + +- **`@Policy(Perf: Critical)`**: Prioritizes zero-cost abstractions over safety wrappers (e.g., preferring raw pointers over `shared_ptr`). + +- **`@Policy(Binary: Stable)`**: Ensures that the projected code maintains exact ABI compatibility with existing binary libraries. + +- **`@Policy(Style: Idiomatic)`**: Instructs the engine to favor the target language's "best practices" even if it requires a larger semantic leap from the source. + + +## 3. Conflict & Ambiguity Resolution + +In complex legacy code, the "intent" is often ambiguous. We use these annotations to flag areas requiring human intervention in the Whetstone UI. + +| | | | +|---|---|---| +|**Annotation**|**Semantic Purpose**|**Human Action Required**| +|**`@Ambiguity(Intent)`**|Patterns that could be either an optimization or a bug.|Choose: "Preserve Hack" or "Refactor to Identity".| +|**`@Candidate(Type)`**|A `void*` that could map to several possible types.|Select the correct semantic type from an inferred list.| +|**`@Tradeoff(Reason)`**|Highlights where safety and speed are in direct opposition.|Authorize the specific safety-violation or accept the latency.| + +## 4. The "Pro-Assistant" Workflow (The Whetstone Method) + +Whetstone functions as an expert advisor rather than a black-box transpiler. The workflow follows a "Sharpening" cycle: + +1. **Survey:** Ingest legacy code (C++/Fortran) and tag patterns with `@Policy(TBD)`. + +2. **Telemetry:** The engine runs data-flow analysis to show the "Cost of Safety" (e.g., "Applying `@Owner(Single)` here will add 4% latency"). + +3. **Selection:** The engineer reviews the "Strategy Menu" and commits to a choice. This creates a `@Decision` node. + +4. **Audit:** The `@Original` metadata (Subject 5) is displayed side-by-side with the projection, allowing the pro to verify that their intent hasn't been mangled. + + +## 5. Non-Deterministic Projections + +- **`@Choice(ID, Options[])`**: A meta-annotation that stores a list of possible valid AST transformations. In Whetstone, this is rendered as a "Lens" that can toggle the view between different strategies. + +- **`@Decision(ID, Selection, Author)`**: Records which option was chosen, why it was chosen, and who made the call. This turns the codebase into a living "Modernization Log." + + +## 6. Tooling & IDE Interaction (Whetstone Intent) + +The Whetstone IDE uses these annotations to provide a specialized UI: + +- **"Modernization Lenses":** Toggling the editor view between the legacy "Source Intent" and the modern "Target Projection." + +- **"Safety Heatmaps":** Visualizing nodes tagged with `@Policy(Strictness: Low)` or `@Ambiguity`. + +- **"Homoiconic Refactoring":** Allowing the user to manipulate the AST metadata directly through a Lisp-based command layer. \ No newline at end of file diff --git a/docs/annotations/C++ Implementation Roadmap.md b/docs/annotations/C++ Implementation Roadmap.md new file mode 100644 index 0000000..e135935 --- /dev/null +++ b/docs/annotations/C++ Implementation Roadmap.md @@ -0,0 +1,506 @@ +# C++ Implementation Roadmap: Whetstone Editor Stack + +## Overview +This document outlines the implementation steps to build the Whetstone standalone editor — a C++ application combining Dear ImGui, headless Emacs servers, and an orchestrator node that operates on a native C++ AST. Sprint 1 (MPS) proved out the AST schema and dual projection. Sprint 2 ports that schema into a self-contained runtime and builds the editor around it. + +The three deliverables are: +1. **C++ AST Runtime** — The SemAnno graph as native C++ data structures +2. **Dear ImGui Editor Shell** — Rendering, structured editing UI, construct chooser +3. **Orchestrator Node** — Server coordination, AST validation, agent API, Elisp↔C++ projection + +--- + +## 1. C++ AST Runtime + +The 33 SemAnno concepts currently live in MPS XML. They need to become a C++ object graph that the editor manipulates at runtime. + +### 1a. Node Base Class and Concept Hierarchy +- Base `ASTNode` with: unique ID, concept type tag, parent pointer, annotation list +- Concept classes mirroring the MPS structure hierarchy: + - `Module`, `Function`, `Parameter`, `Variable` (top-level containers) + - `Statement` (abstract) → `Block`, `Assignment`, `Return`, `IfStatement`, `WhileLoop`, `ForLoop` + - `Expression` (abstract) → `BinaryOperation`, `UnaryOperation`, `FunctionCall`, `VariableReference`, `IndexAccess`, `MemberAccess`, `ListLiteral` + - Literals: `IntegerLiteral`, `FloatLiteral`, `StringLiteral`, `BooleanLiteral` + - Types: `PrimitiveType`, `ListType`, `SetType`, `MapType`, `TupleType`, `ArrayType`, `OptionalType`, `CustomType` + - Annotations: `DerefStrategy`, `OptimizationLock`, `LangSpecific` +- Child links as `std::vector>` for 0..n, `std::unique_ptr` for 1 +- Properties as typed fields (string, int, enum) + +### 1b. Schema-Driven Validation +- Each concept declares its legal children (by type, cardinality, role name) +- `ASTSchema` class loaded at startup — mirrors the SemAnno structure definitions +- Validation: given a parent concept + role, return the set of legal child concept types +- This is what powers the structured editing construct chooser + +### 1c. AST Serialization +- **Primary format: JSON** — human-readable, debuggable, matches the Agent API contract +- Schema: mirrors MPS XML structure but in JSON (concept, id, properties, children keyed by role) +- Round-trip: load JSON → C++ graph → save JSON with no information loss +- Future option: SQLite-backed graph for large projects (deferred — JSON is sufficient for MVP) + +### 1d. Annotation System +- Annotations attach to any node via the `annotations` child link +- `DerefStrategy`: strategy enum (manual, gc, ownership, content-addressed) + derefLocation, derefTime +- `OptimizationLock`: lockedBy, lockReason, lockLevel (warning/soft/hard), affectedStrategies, timestamp +- `LangSpecific`: language, idiomType, rawSyntax, semanticHint, position +- Rename consideration: `@deref` → `@dealloc` for accuracy (update all references) +- Annotations are metadata — they affect projection output, not the AST structure itself + +--- + +## 2. Projection Generators (C++ and Elisp) + +Sprint 1's MPS textGen rules proved the projection model. Sprint 2 needs equivalent generators as C++ code operating on the C++ AST. + +### 2a. Generator Architecture +- `ProjectionGenerator` base class with `generate(ASTNode*, OutputStream&)` interface +- Per-concept visitor pattern: each concept type has a generate method +- Generator holds state: current indentation level, target language context +- Output is plain text (source code string) + +### 2b. Python Generator (port from MPS) +- Direct port of the working MPS textGen rules into C++ visitor methods +- Module → `def`-based output with Python syntax +- Memory annotations ignored (Python is GC — `@dealloc` has no effect on output) +- `LangSpecific(python)` annotations emit their `rawSyntax` inline + +### 2c. C++ Generator +- Module → `#include`-guarded output with C++ syntax +- Functions → typed signatures with `auto` where appropriate +- Memory strategy application: + - `@dealloc(manual)` → raw pointers, explicit `new`/`delete` + - `@dealloc(gc)` → `std::shared_ptr` / `std::weak_ptr` + - `@dealloc(ownership)` → `std::unique_ptr`, move semantics + - `@dealloc(content-addressed)` → `const` types, content-hash identity +- Type mapping: `list[T]` → `std::vector`, `map[K,V]` → `std::unordered_map`, `optional[T]` → `std::optional` +- `LangSpecific(cpp)` annotations emit their `rawSyntax` inline +- Optimization hints: `@Inline` → `[[gnu::always_inline]]`, `@Pure` → `[[nodiscard]]` + `const`, `@Loop(Unroll)` → `#pragma unroll` + +### 2d. Elisp Generator +- Module → `(provide 'module-name)` with `defun`/`defvar` output +- Functions → `(defun name (params) body)` +- Variables → `(defvar name value)` +- Control flow: `IfStatement` → `(if cond then else)`, `WhileLoop` → `(while cond body)` +- Type annotations as comments (Elisp is dynamically typed) +- `LangSpecific(elisp)` annotations emit original forms: `(interactive ...)`, `(with-temp-buffer ...)`, `(advice-add ...)` + +### 2e. Elisp Parser (tree-sitter) +- Integrate `tree-sitter-elisp` grammar +- CST → AST mapping: `defun` → Function, `let`/`let*` → Variable + Block, `if`/`cond`/`when` → IfStatement +- Elisp-specific constructs that don't map cleanly get `@LangSpecific(elisp, ...)` annotations +- Key Elisp idioms to handle: + - `defvar`/`defcustom`/`defconst` → Variable with annotation metadata + - `interactive` forms → `@LangSpecific(elisp, "special_form", ...)` + - `advice-add`/`advice-remove` → `@LangSpecific(elisp, "advice", ...)` + - `condition-case`/`unwind-protect` → error handling mapped to IfStatement + annotation + - Dynamic binding (`let` vs `lexical-let`) → `@Binding(Dynamic)` / `@Binding(Static)` annotations + - Macros → expand first, annotate with `@Meta(Expanded)` + `@Original(source)` + +--- + +## 3. Dear ImGui Editor Shell + +The rendering surface. Pure presentation — no business logic. + +### 3a. Application Scaffold +- Dear ImGui + SDL2/GLFW backend (cross-platform) +- Docking layout: file tree, editor tabs, terminal panel, structured edit panel, command palette +- Font rendering with ligature support for code display +- Theme system (dark/light, configurable) + +### 3b. Text Viewport +- Display projected source code (read from generator output) +- Syntax highlighting via token classification from the AST (not regex) +- Cursor and selection rendering +- Gutter: line numbers, annotation indicators, lock icons for `OptimizationLock` + +### 3c. Structured Editing Panel +- **Construct Chooser**: grid of legal construct types for current cursor position (queried from AST schema) +- **Field Editors**: constrained inputs for each concept's properties + - Name fields: identifier validation + - Type fields: dropdown of available types (PrimitiveType kinds + user-defined) + - Expression fields: recursive construct chooser + - Enum fields: dropdown of legal values (e.g., TargetLanguage, deref strategies) +- **Scope-Aware Dropdowns**: only show in-scope variables, functions, types +- Keyboard shortcuts: single-key construct selection (f=Function, v=Variable, i=If, etc.) + +### 3d. AST Tree View +- Collapsible tree showing the live AST structure +- Click node → highlight in text viewport +- Right-click → context menu (edit, delete, move, add annotation) +- Drag-and-drop reordering of statement lists + +### 3e. Dual Projection Toggle +- Toolbar buttons: [Python] [C++] [Elisp] [AST] +- Switching regenerates the text viewport from the same AST via different generators +- Annotations panel shows/hides based on relevance to current projection + +--- + +## 4. Orchestrator / Management Node + +The brain. C++ process that coordinates everything. + +### 4a. Core Responsibilities +- Owns the in-memory AST graph (single source of truth) +- Routes messages between ImGui shell, Emacs servers, and agent clients +- Validates all AST mutations against the schema +- Manages undo/redo stack (AST operation journal) +- Persists AST to disk (JSON serialization from §1c) + +### 4b. AST Mutation API +- All changes go through the orchestrator — no direct graph manipulation +- Operations: `insertNode(parentId, role, conceptType, properties)`, `deleteNode(nodeId)`, `moveNode(nodeId, newParentId, role)`, `setProperty(nodeId, propertyName, value)` +- Each operation validated against schema before applying +- Each operation recorded for undo and provenance tracking + +### 4c. IPC Protocol (JSON-RPC over Unix Sockets) +- **ImGui → Orchestrator**: user input events, construct choices, property edits +- **Orchestrator → ImGui**: AST updates, projection text, validation errors, scope options +- **Orchestrator → Emacs**: file I/O, search queries, completion requests, git operations +- **Emacs → Orchestrator**: results, file contents, candidate lists +- Message format: JSON-RPC 2.0 (`{ "jsonrpc": "2.0", "method": "...", "params": {...}, "id": N }`) + +### 4d. Emacs Server Pool Management +- Spawn headless Emacs daemons on startup: `emacs --daemon=serverN --load whetstone-bridge.el` +- Specialized servers (file-server, search-server, complete-server, git-server, org-server) +- Health monitoring: heartbeat, restart on crash +- Request routing: match method name to responsible server +- Response caching: file contents, completion results (invalidated on change) + +### 4e. Agent API +- External JSON interface for AI agents to submit construct choices +- Same validation as human UI — invalid choices rejected with `valid_options` list +- Workflow: + 1. Agent queries: `getValidConstructs(parentId, role)` → list of legal concept types + 2. Agent submits: `insertNode(parentId, role, "Function", { name: "process_data", ... })` + 3. Orchestrator validates → applies → returns updated AST subtree +- Batch mode: agent submits an entire subtree as JSON, orchestrator validates recursively +- Trust levels (configurable): auto-accept, review-required, sandbox-only + +### 4f. Scope Resolution Engine +- Walks the AST to determine what's in scope at any given node position +- Used by: structured editing dropdowns, agent API validation, completion +- Tracks: variables, functions, parameters, types, imported modules +- Respects: block scoping, function boundaries, module boundaries +- Closure capture analysis: which outer-scope bindings are referenced + +### 4g. Warning System (OptimizationLock) +- When a node has `OptimizationLock` and the current user/agent tier < lock level: + - Structured editing panel shows lock icon and warning banner + - Mutation still allowed (warning only, not blocking) + - Original optimization preserved as shadow (annotation chain via `@provenance`) + - Undo restores the locked state + +--- + +## 5. Emacs Integration Layer + +### 5a. whetstone-bridge.el +- Elisp package loaded by each headless Emacs server +- Exposes server capabilities as JSON-RPC endpoints +- Translates between Emacs primitives and Whetstone protocol: + - `find-file` → buffer contents as JSON response + - `consult-ripgrep` → search results as JSON array + - `magit-status` → git status as structured data + - `completion-at-point` → candidate list with metadata + +### 5b. Buffer↔AST Synchronization +- Emacs buffers display projected text (generated from AST) +- Edits in Emacs buffer → parse diff → translate to AST mutations → send to orchestrator +- Orchestrator regenerates projection → pushes updated text back to Emacs buffer +- This enables using Emacs keybindings for text navigation while the AST remains authoritative + +### 5c. Org-mode Integration +- Documentation lives alongside code as AST annotations +- Org-server renders `@doc` annotations as org-mode formatted text +- Edit documentation in Emacs → annotation updated in AST + +--- + +## 6. Elisp↔C++ Transpilation (Dogfooding) + +The editor itself is the first real Whetstone project. Write editor logic in Elisp, generate C++ for performance-critical paths. + +### 6a. Target: whetstone-bridge.el +- Parse `whetstone-bridge.el` into Whetstone AST via tree-sitter-elisp +- Generate equivalent C++ for the Management Node's hot paths +- Verify semantic equivalence: same JSON-RPC responses for same inputs + +### 6b. Elisp→C++ Mapping Challenges +- **Dynamic typing** → `std::variant` or tagged union with `@Candidate(Type)` annotations +- **Symbol tables** → `std::unordered_map` with `@Binding(Dynamic)` annotation +- **Garbage collection** → `std::shared_ptr` (default) or `std::unique_ptr` where ownership is clear +- **Homoiconicity** → AST nodes ARE the code-as-data representation (Whetstone is inherently homoiconic) +- **Buffer model** → C++ string with gap buffer or rope, annotated with `@LangSpecific(elisp, "buffer", ...)` +- **Event loop** → `@Exec(Event)` annotation, map to C++ event loop (SDL2 or libuv) + +### 6c. Incremental Approach +- Start with pure functions (no side effects, no dynamic binding) — easiest to transpile +- Add stateful functions with explicit `@Capture` and `@Binding` annotations +- Tackle Emacs-specific idioms last (interactive forms, advice, buffer-local variables) +- Each function gets a confidence score: high = auto-transpile, low = human review + +--- + +## 7. Testing and Validation + +### 7a. AST Runtime Tests +- Round-trip: JSON → C++ graph → JSON (byte-identical) +- Schema validation: reject illegal child placements, accept legal ones +- Mutation: insert/delete/move operations maintain graph invariants + +### 7b. Generator Tests +- Per-concept: each of the 33 concepts generates valid output in each target language +- Integration: example modules (Calculator, SimpleFunctionExample, ConditionalExample) generate compilable code +- Cross-language: Python output and C++ output are semantically equivalent for the same AST + +### 7c. Editor Integration Tests +- Structured editing: construct chooser offers exactly the legal options +- Agent API: JSON submissions create correct AST structures +- Emacs IPC: file operations, search, completion respond within latency budget + +### 7d. Transpilation Tests +- Elisp → AST → C++ → compile → run → compare output with Elisp evaluation +- Start with pure arithmetic/string functions, build up to stateful operations +- Annotation preservation: `@LangSpecific` survives round-trip + +--- + +## Implementation Order + +Each step is small enough to build, test, and verify before moving on. If a step breaks, the problem is in that step — not in 4 things you changed at once. + +### Phase 2a: AST in C++ + +**Step 1: Base node + one concept** +- `ASTNode` base class with ID, concept tag, parent pointer +- `Module` concept only — properties (name, targetLanguage), empty child vectors +- Test: create a Module in C++, verify fields work + +**Step 2: Child links** +- Add child link support to `ASTNode` (single-valued and multi-valued) +- Add `Function` and `Variable` concepts as children of Module +- Test: build Module → Function tree in code, walk it, verify parent/child pointers + +**Step 3: Remaining concepts** +- Add all Statement, Expression, Literal, and Type concepts +- Each one gets its own properties and child link definitions +- Test: build the Calculator example as a C++ object graph + +**Step 4: Annotation concepts** +- Add `DerefStrategy`, `OptimizationLock`, `LangSpecific` +- Attach to Module/Function/Variable via annotations link +- Test: build SimpleFunctionExample with @deref(batched), verify annotation reads back + +**Step 5: JSON serialization (save)** +- Serialize the C++ AST graph to JSON +- One function: `toJson(ASTNode*) → nlohmann::json` +- Test: build Calculator graph → serialize → inspect JSON output by hand + +**Step 6: JSON deserialization (load)** +- Parse JSON back into C++ AST graph +- One function: `fromJson(nlohmann::json) → ASTNode*` +- Test: save Calculator → load it back → save again → compare JSON (byte-identical) + +**Step 7: Schema validation** +- `ASTSchema` class: for each concept, list legal child types per role + cardinalities +- Validation function: `isLegalChild(parentConcept, role, childConcept) → bool` +- Test: legal placements accepted, illegal placements rejected (e.g., Module as child of IntegerLiteral) + +### Phase 2b: First Generator + +**Step 8: Generator base + Module/Function** +- `ProjectionGenerator` base with visitor dispatch +- `PythonGenerator`: generate Module header and Function signatures only +- Test: Calculator AST → Python output has correct `def add(x, y):` signature + +**Step 9: Statements and expressions** +- Add Assignment, Return, BinaryOperation, VariableReference to Python generator +- Test: Calculator AST → full Python output matches MPS textGen output + +**Step 10: Remaining Python concepts** +- Literals, control flow (If/While/For), types, FunctionCall, IndexAccess, MemberAccess +- Test: ConditionalExample AST → correct Python with if/else + +**Step 11: Annotation output** +- DerefStrategy → `# @deref(strategy)` comment in Python +- OptimizationLock → `# @lock(...)` comment +- Test: SimpleFunctionExample with annotations → comments appear in output + +### Phase 2c: Dear ImGui Shell + +**Step 12: Window + empty layout** +- Dear ImGui + SDL2 scaffold: opens a window with docking enabled +- Empty panes: file tree (left), editor area (center), panel (bottom) +- Test: app launches, window renders, panes are resizable + +**Step 13: Text viewport** +- Load a hardcoded Python string into the editor pane +- Render with monospace font, line numbers in gutter +- Test: text displays correctly, scrolls + +**Step 14: AST → text viewport** +- Connect: load Calculator JSON → build AST → run Python generator → display in viewport +- Test: change the JSON file, restart, see different output + +**Step 15: Projection toggle** +- Toolbar with [Python] [AST] buttons +- Python: show generated code. AST: show JSON tree dump +- Test: toggle between views, both render correctly + +### Phase 2d: Orchestrator (Standalone Process) + +**Step 16: Orchestrator process + AST ownership** +- Separate C++ process that loads AST from JSON and holds it in memory +- Exposes nothing yet — just loads, holds, saves on exit +- Test: start orchestrator with Calculator.json, kill it, verify JSON saved back + +**Step 17: JSON-RPC server** +- Orchestrator listens on a Unix socket (or named pipe on Windows) +- Responds to `ping` and `getAST` methods +- Test: send JSON-RPC from a Python script, get response + +**Step 18: AST mutation via RPC** +- Add `setProperty` method: change a node's property value via RPC +- Validate against schema before applying +- Test: change Calculator function name via RPC, query AST back, see new name + +**Step 19: Insert and delete via RPC** +- `insertNode(parentId, role, conceptType, properties)` — creates node, validates, inserts +- `deleteNode(nodeId)` — removes node and subtree +- Test: insert a new Parameter via RPC, delete it, verify AST integrity + +**Step 20: Undo/redo** +- Operation journal: each mutation recorded as a reversible op +- `undo` and `redo` RPC methods +- Test: insert node → undo → node gone → redo → node back + +**Step 21: Connect ImGui to orchestrator** +- ImGui shell connects to orchestrator via JSON-RPC instead of loading JSON directly +- Requests AST on startup, displays projection +- Test: launch orchestrator, launch ImGui shell, see Calculator output + +### Phase 2e: Structured Editing + +**Step 22: Construct chooser (read-only)** +- ImGui panel queries orchestrator: `getValidConstructs(parentId, role)` → list of legal types +- Display as button grid +- Test: click on Function body → see [Assignment] [Return] [If] [While] [For] etc. + +**Step 23: Insert from chooser** +- Click a construct button → orchestrator creates node with default properties +- Viewport refreshes with new projection +- Test: add a Return statement to a function, see it in Python output + +**Step 24: Property editing** +- Click a node → property panel shows editable fields (name, type, operator, etc.) +- Edit field → `setProperty` RPC → viewport refreshes +- Test: rename a function, see output update + +**Step 25: Scope-aware dropdowns** +- Orchestrator implements `getInScopeSymbols(nodeId)` → variables, functions, types visible at that position +- VariableReference name field shows dropdown of in-scope variables +- Test: inside Calculator.add body, dropdown shows [x, y, result] + +### Phase 2f: Emacs Integration + +**Step 26: Spawn one headless Emacs** +- Orchestrator spawns `emacs --daemon=whetstone-file --load whetstone-bridge.el` +- Sends a test command via `emacsclient`, gets response +- Test: orchestrator starts, Emacs server responds to ping + +**Step 27: File operations** +- `openFile(path)` → routes to Emacs file-server → returns contents +- `saveFile(path, contents)` → writes via Emacs +- Test: open a file through orchestrator, verify contents match + +**Step 28: Search** +- `search(query)` → routes to Emacs search-server → returns file:line results +- Results displayed in ImGui panel +- Test: search for "calculate_sum", get result pointing to SimpleFunctionExample + +**Step 29: Server pool** +- Multiple Emacs daemons: file-server, search-server, git-server +- Orchestrator routes by method name +- Health monitoring: restart crashed servers +- Test: kill a server, verify orchestrator restarts it and retries the request + +### Phase 2g: Elisp Projection + +**Step 30: Elisp generator — functions and variables** +- Module → `(provide 'name)`, Function → `(defun name (params) body)`, Variable → `(defvar name value)` +- Test: Calculator AST → valid Elisp that loads in Emacs without errors + +**Step 31: Elisp generator — statements and expressions** +- Assignment → `(setq target value)`, Return → value (last expression), If → `(if cond then else)` +- BinaryOperation → `(op left right)`, Literals → values +- Test: ConditionalExample AST → Elisp with `(if (= status_code 200) "OK" "Error")` + +**Step 32: Elisp parser (tree-sitter)** +- Integrate tree-sitter-elisp, parse a `.el` file into CST +- Map `defun` → Function, `defvar` → Variable, `if` → IfStatement +- Test: parse a small hand-written `.el` file → inspect resulting AST + +**Step 33: Elisp round-trip** +- Parse `.el` → AST → generate Elisp → compare with original +- Handle `@LangSpecific` for idioms that don't map cleanly +- Test: write a 3-function `.el` file, round-trip it, verify semantic equivalence + +### Phase 2h: C++ Generator + Agent API + +**Step 34: C++ generator — basic output** +- Module → `#include` guards, Function → typed signatures, variables → declarations +- No memory strategy yet — just syntactically valid C++ +- Test: Calculator AST → compilable C++ with g++ + +**Step 35: C++ generator — memory strategies** +- `@dealloc(manual)` → raw pointers, `@dealloc(gc)` → shared_ptr, `@dealloc(ownership)` → unique_ptr +- Test: same AST with different @dealloc values → different C++ output, all compile + +**Step 36: Agent API** +- Orchestrator accepts JSON-RPC from external clients (same protocol as ImGui) +- `getValidConstructs`, `insertNode`, `setProperty`, `getInScopeSymbols` +- Test: Python script acts as agent, builds a Function node by node via RPC + +**Step 37: Agent batch mode** +- `insertSubtree(parentId, role, jsonSubtree)` — submit entire subtree at once +- Orchestrator validates recursively, applies atomically (all or nothing) +- Test: agent submits full ConditionalExample as JSON, orchestrator builds the AST + +**Step 38: OptimizationLock warnings** +- When mutating a locked node, orchestrator returns warning (not rejection) +- ImGui shows lock icon + banner +- Shadow system: original annotation preserved, modification tracked via `@provenance` +- Test: lock a function, modify it, verify warning fires and original preserved + +--- + +## Milestone Checkpoints + +Each checkpoint is a "stop and verify" gate. Don't proceed until it passes. + +| After Step | Checkpoint | How to Verify | +|------------|-----------|---------------| +| 6 | AST round-trips through JSON | Save → load → save produces identical JSON | +| 9 | Python generator matches MPS | Calculator output identical to MPS textGen | +| 14 | ImGui shows live AST | Change JSON, restart, see different output | +| 21 | ImGui talks to orchestrator | Two processes running, viewport shows AST content | +| 25 | Structured editing works | Build a function entirely through the UI | +| 29 | Emacs servers respond | File open, search, and git status all return results | +| 33 | Elisp round-trips | Parse .el → AST → generate .el → semantically equivalent | +| 35 | C++ compiles from AST | Generated C++ compiles with g++ for all 3 example models | +| 37 | Agent builds AST remotely | Python script creates full ConditionalExample via RPC | + +## Dependencies + +| Dependency | Version | Purpose | +|------------|---------|---------| +| Dear ImGui | 1.90+ | GUI rendering | +| SDL2 or GLFW | latest | Window/input backend for ImGui | +| Emacs | 29+ | Headless servers | +| nlohmann/json | 3.11+ | JSON serialization, JSON-RPC | +| tree-sitter | 0.22+ | Elisp/Python/C++ parsing | +| tree-sitter-elisp | latest | Elisp grammar | +| C++ compiler | C++20 | Entire C++ stack | +| GoogleTest | latest | C++ unit tests | diff --git a/docs/annotations/DSL overview.md b/docs/annotations/DSL overview.md new file mode 100644 index 0000000..3b08170 --- /dev/null +++ b/docs/annotations/DSL overview.md @@ -0,0 +1,80 @@ +# Universal AST Annotation Taxonomy for Lossless Transpilation + +This document outlines the semantic metadata required to create a language-agnostic Abstract Syntax Tree (AST). These annotations bridge the "Semantic Gap" between high-level abstractions (Python, Java) and low-level imperatives (C++, Rust). + +## Subject 1: Memory Management & Lifecycle (Priority Sprint) + +_This section is reserved for the initial implementation phase. It will cover Allocation, Deallocation, Ownership, and Lifetime semantics._ + +## Subject 2: Type System & Variance + +To map types across languages, we must go beyond "Int" or "String" and annotate the underlying constraints. + +- **Memory Layout:** Padding, alignment requirements, and bit-width. + +- **Mutability:** Deep vs. shallow immutability (e.g., `const` in C++ vs. `readonly` in TS). + +- **Nullability & Optionals:** Defining how "nothingness" is handled (Pointers, Option types, or Sentinel values). + +- **Type Variance:** Covariance and contravariance constraints for generics and collections. + +- **Nominal vs. Structural Identity:** Does the type match by name or by shape? + + +## Subject 3: Execution Model & Concurrency + +Capturing how code flows and interacts with the processor or runtime. + +- **Evaluation Strategy:** Eager vs. Lazy evaluation nodes. + +- **Concurrency Primitives:** Shared memory vs. Message passing (Actors). + +- **Atomicity:** Annotating blocks that must be thread-safe or non-interruptible. + +- **Async/Await Semantics:** Capturing the state machine logic of asynchronous tasks. + +- **Exception Handling:** Distinguishing between recoverable errors and fatal panics. + + +## Subject 4: Scope & Namespace Resolution + +Handling how identifiers are looked up and isolated. + +- **Closure Captures:** Explicitly marking which variables are moved, copied, or referenced by a closure. + +- **Shadowing Rules:** Defining behavior when identifiers overlap across scopes. + +- **Visibility & Access:** Beyond `public/private`; defining "friendship" and module-level internal access. + +- **Symbol Binding:** Static vs. Dynamic linking requirements. + + +## Subject 5: Target-Specific "Shims" (The Escape Hatch) + +Handling features that cannot be expressed purely through universal semantics. + +- **Inline Assembly/Intrinsics:** Preserving raw hardware instructions. + +- **FFI Boundaries:** Annotating nodes that interact with foreign binary interfaces. + +- **Platform-Specific Conditional Logic:** Semantic tags for OS-specific or Hardware-specific implementations. + + +## Subject 6: Optimization & Intent Hints + +Metadata that guides the projection engine to produce "idiomatic" target code. + +- **Inlining Hints:** Programmer intent regarding function expansion. + +- **Vectorization Potential:** Marking loops that are safe for SIMD instructions. + +- **Purity & Side Effects:** Annotating functions as "Pure" to allow for aggressive caching/reordering during projection. + + +## Subject 7: Meta-Programming & Macros + +How the AST handles code that generates more code. + +- **Expansion State:** Tracking if a node is the result of a macro expansion. + +- **Reflection Requirements:** Marking objects that need metadata available at runtime. \ No newline at end of file diff --git a/docs/annotations/Exec model and Concurrency.md b/docs/annotations/Exec model and Concurrency.md new file mode 100644 index 0000000..e5b811e --- /dev/null +++ b/docs/annotations/Exec model and Concurrency.md @@ -0,0 +1,68 @@ +# Subject 3: Execution Model & Concurrency + +This document defines the annotations for the universal AST to capture how code executes, interacts with hardware threads, and manages asynchronous state. This is essential for converting legacy synchronous code (like old C++ or Fortran) into modern non-blocking paradigms. + +## 1. Concurrency Primitives & Memory Models + +| | | | +|---|---|---| +|**Annotation**|**Semantic Purpose**|**Example Conflict**| +|**`@Atomic(Consistency)`**|Defines memory visibility (SeqCst, Relaxed, Acquire/Release).|C++ `std::atomic` vs. Java `volatile`.| +|**`@Sync(Monitor|Spin|Semaphore)`**| +|**`@ThreadModel(Green|OS|Fiber)`**| +|**`@MemoryBarrier`**|Explicit hardware-level memory fencing.|Essential for legacy C/Assembly transpilation to prevent reordering.| + +## 2. Asynchronous State & Control Flow + +Mapping between "Blocking" and "Non-blocking" logic requires tracking the state machine of the execution. + +### A. Async/Await Projection (`@Exec(Async)`) + +- **Source:** JS, Python, Rust, C#. + +- **Definition:** Functions that yield execution and resume via a task runner. + +- **Projection Logic:** + + - **Into C (No runtime):** Must be projected into a manual `struct`-based state machine with function pointers. + + - **Into Java (Project Loom):** Maps directly to Virtual Threads. + + +### B. Reactive/Event-Driven (`@Exec(Event)`) + +- **Source:** Node.js, Erlang, Prolog (backtracking). + +- **Definition:** Execution triggered by signals or goal-matching. + +- **Projection Logic:** In imperative languages, this requires injecting a central event loop or a "Trampoline" function to prevent stack overflow. + + +## 3. The "Legacy Sync-to-Async" Upgrade (The Karpathy Goal) + +For modernizing 40 trillion lines of C++, the AST identifies "Blocking IO" and annotates it for transformation. + +- **Annotation:** `@Blocking(IO | Compute)` + +- **Transformation:** If the target is a modern safe language (Rust/Go), the projection engine replaces `@Blocking(IO)` calls with their non-blocking equivalents, wrapping the call-site in an `async` context. + + +## 4. Parallelism & Vectorization + +| | | | +|---|---|---| +|**Annotation**|**Semantic Purpose**|**Target Action**| +|**`@Parallel(Data)`**|Safe for SIMD/Vectorization.|Inject `pragma omp parallel` (C++) or use `Intrinsics`.| +|**`@Parallel(Task)`**|Safe for multi-core distribution.|Project to `ForkJoinPool` (Java) or `Ray` (Python).| +|**`@Pure`**|No side effects; execution order is irrelevant.|Allows aggressive reordering and memoization.| + +## 5. Error Handling & Interrupts + +- **`@Trap(Signal)`**: Handling hardware-level interrupts (C/Fortran). + +- **`@Exception(Checked | Unchecked)`**: Software-level error propagation. + +- **`@Panic(Abort | Unwind)`**: Defining what happens during an unrecoverable state. + + +**Projection Conflict:** When projecting C++ (Exceptions) to Go (Return Values), the engine must wrap every `@Exception` call-site in an `if err != nil` block, "exploding" the single line of C++ into the multi-line Go pattern. \ No newline at end of file diff --git a/docs/annotations/Memory strategy.md b/docs/annotations/Memory strategy.md new file mode 100644 index 0000000..c867735 --- /dev/null +++ b/docs/annotations/Memory strategy.md @@ -0,0 +1,63 @@ +# Subject 1: Memory Strategy Mapping (Top 20 Languages) + +This document maps the memory management strategies of the top 20 languages and defines the annotation-based "projection" strategy used when the target language does not natively support the source paradigm. + +## 1. Universal Memory Paradigms + +| | | | +|---|---|---| +|**Paradigm**|**Description**|**Key Languages**| +|**Manual (MM)**|Explicit allocation/deallocation.|C, C++, Assembly, Zig| +|**Garbage Collection (GC)**|Automatic runtime reclamation.|Java, Python, JS, Go, C#, Ruby, PHP| +|**ARC (Automatic Ref Counting)**|Deterministic reference tracking.|Swift, Objective-C| +|**Ownership/Borrowing (OB)**|Compile-time lifetime tracking.|Rust| + +## 2. Language-Specific Strategy & Projection Mapping + +| | | | | | +|---|---|---|---|---| +|**Language**|**Primary Strategy**|**Annotation for Non-Supporting Target**|**Projection Action (into Low-Level/C)**|**Projection Action (into High-Level/Java)**| +|**1. Python**|GC (Ref Counting + Cycle)|`@Reclaim(Tracing)`|Inject ref-count logic; use `free` on zero.|Treat as native object; let JVM handle.| +|**2. Java**|GC (Generational)|`@Reclaim(Tracing)`|Inject tracking hooks; require runtime GC lib.|Native mapping.| +|**3. JavaScript**|GC (Mark-and-Sweep)|`@Reclaim(Tracing)`|Transform closures into heap-allocated structs.|Map to native classes/objects.| +|**4. C++**|RAII / Manual|`@Lifetime(RAII)`|Inject `destructor` calls at scope end.|Wrap in `AutoCloseable` or `finalize`.| +|**5. C**|Manual|`@Deallocate(Explicit)`|Direct `free()` mapping.|Requires manual `.close()` or `cleaner`.| +|**6. Rust**|Ownership/Borrowing|`@Owner(Single)`|Zero-cost; inject `free` at end-of-life.|Emulate with single-reference patterns.| +|**7. TypeScript**|GC (via JS)|`@Reclaim(Tracing)`|Same as JavaScript.|Same as JavaScript.| +|**8. C#**|GC (Generational)|`@Reclaim(Tracing)`|Inject `IDisposable` pattern logic.|Native mapping.| +|**9. Go**|GC (Concurrent)|`@Reclaim(Escape)`|Perform escape analysis; inject heap vs stack.|Map to native objects.| +|**10. Swift**|ARC|`@Owner(Shared_ARC)`|Inject `increment`/`decrement` calls.|Map to native GC objects.| +|**11. PHP**|GC (Ref Counting)|`@Reclaim(Cycle)`|Inject cycle-detection logic.|Map to native objects.| +|**12. Ruby**|GC (Mark-and-Sweep)|`@Reclaim(Tracing)`|Inject tracking for all object refs.|Map to native objects.| +|**13. Kotlin**|GC (via JVM/Native)|`@Reclaim(Tracing)`|Same as Java (or ARC if Kotlin/Native).|Native mapping.| +|**14. Go**|GC|`@Reclaim(Tracing)`|See #9.|Native mapping.| +|**15. Lua**|GC (Incremental)|`@Reclaim(Tracing)`|Inject minimal state-machine GC.|Map to native objects.| +|**16. Fortran**|Static / Manual|`@Allocate(Static)`|Fixed memory buffers.|Map to static arrays.| +|**17. Assembly**|Register / Raw|`@Allocate(Register)`|Direct register mapping or stack spill.|Use local variables (JVM locals).| +|**18. Dart**|GC (Generational)|`@Reclaim(Tracing)`|Inject hook for "Isolates" memory.|Native mapping.| +|**19. Objective-C**|ARC / Manual (MRR)|`@Owner(Shared_ARC)`|See #10.|Map to native objects.| +|**20. Zig**|Manual (Explicit)|`@Allocate(Allocator)`|Pass allocator pointers to all functions.|Wrap in resource manager.| + +## 3. The "Lossless" Projection Logic + +When the transpiler encounters a node with a memory annotation that the target language cannot express, it applies these transformations: + +### A. High-Level $\to$ Low-Level (e.g., Python to C) + +- **Rule:** If `@Reclaim(Tracing)` is found, the engine must implement a **Reference Counting Shim**. + +- **Action:** Every variable assignment is wrapped in an `INC_REF` function, and every scope exit or reassignment triggers `DEC_REF`. If count == 0, `free()` is injected. + + +### B. Low-Level $\to$ High-Level (e.g., C to Java) + +- **Rule:** If `@Deallocate(Explicit)` is found, the engine must ensure the resource is not leaked by the high-level runtime's lack of "free". + +- **Action:** The node is projected into a `try-with-resources` block or the object is wrapped in a `Cleaner` API to ensure the underlying buffer is released even if the user forgets. + + +### C. Strict $\to$ Permissive (e.g., Rust to Python) + +- **Rule:** If `@Owner(Single)` is found, the engine can "relax" the constraint. + +- **Action:** Python doesn't care about single ownership; the annotation is preserved in the AST metadata for analysis but doesn't affect the generated Python code beyond standard object creation. \ No newline at end of file diff --git a/docs/annotations/Meta-Programming & Homoiconicity.md b/docs/annotations/Meta-Programming & Homoiconicity.md new file mode 100644 index 0000000..204ea03 --- /dev/null +++ b/docs/annotations/Meta-Programming & Homoiconicity.md @@ -0,0 +1,77 @@ +# Subject 7: Meta-Programming & Homoiconicity + +This document defines the annotations for code-generating-code, compile-time evaluation, and runtime introspection. In the Whetstone ecosystem, we treat the AST as a homoiconic medium where the boundary between code and data is a matter of projection policy rather than a hardcoded transformation. + +## 1. Homoiconicity & Data-Code Duality + +For languages like Lisp, the AST must distinguish between code intended for execution and code intended to be treated as a data structure. + +| | | | +|---|---|---| +|**Annotation**|**Semantic Purpose**|**Example Conflict**| +|**`@Meta(Quoted)`**|Identifies a node as data, not for immediate execution.|Lisp `'expr` vs. standard variable reference.| +|**`@Meta(Unquoted)`**|Signals an "escape" back to execution within a quoted block.|Lisp `,expr` (comma) or `~expr`.| +|**`@Symbol(Gensym)`**|A unique, non-colliding identifier generated at compile-time.|Preventing "Macro Hygiene" issues during expansion.| +|**`@Evaluate(Phase)`**|Defines when a node should be run (Compile-time vs. Runtime).|Zig `comptime` vs. standard execution.| + +## 2. Macro Systems & Expansion Options + +The AST tracks the relationship between a macro "call" and its "expansion" as a set of non-destructive choices. + +### A. Expansion Strategy (`@Choice`) + +- **Strategy A: Preservation (Opaque):** Keep the macro call intact (if the target supports it, like Rust). + +- **Strategy B: Static Inlining:** Inline the expanded code directly but retain the `@Expansion(Source)` metadata for auditing. + +- **Strategy C: Dynamic Wrapper:** Wrap the expansion in a dedicated function or object to maintain a clear boundary in the target language. + + +## 3. Reflection & Introspection Policies + +Instead of hardcoding a lookup table, Whetstone offers policies for how types "see" themselves at runtime. + +### A. Introspection Policy (`@Policy`) + +- **`@Policy(Reflect: Strip)`**: Remove all reflection metadata for maximum performance/minified size. + +- **`@Policy(Reflect: Static_Shims)`**: Generate specific getter/setter functions for requested members only (avoids full runtime overhead). + +- **`@Policy(Reflect: Full_Runtime)`**: Embed a comprehensive type-registry (e.g., Java-style Reflection or C# TypeInfo). + + +## 4. Templates & Generics Meta-Programming + +Whetstone treats C++ templates and generic specializations as meta-programming candidates. + +- **Annotation:** `@Template(Specialization)` + +- **Modernization Options:** + + - **Option 1: Trait-based (Rust Style):** Project as a bounded generic with shared interfaces. + + - **Option 2: Monomorphization (C++ Style):** Generate distinct copies of the code for each type used. + + - **Option 3: Type Erasure (Java Style):** Use a single implementation with casts or Object-references. + + +## 5. Synthetic Generation & The Legacy Shim + +When modernizing "string-building" code (e.g., C code using `printf` to generate script fragments): + +- **Annotation:** `@Synthetic(Generator)` + +- **Whetstone Intent:** The editor highlights these blocks as "Structural Risks." The user is offered a choice to either preserve the raw string generation or refactor the generator into an AST-native Macro that produces valid nodes rather than raw text. + + +## 6. Logic Execution & Search (Prolog Style) + +- **`@Goal(Query)`**: Defines a node as a logical search. + +- **`@Unification`**: Annotates how variables are bound during a search. + +- **Projection Choices:** * **Candidate 1: Backtracking Engine:** Inject a recursive search algorithm. + + - **Candidate 2: Constraint Solver:** Map the logic to an external SMT/SAT solver library. + + - **Candidate 3: Iterative Search:** Flatten the goal into a series of nested loops (if the search space is finite and predictable). \ No newline at end of file diff --git a/docs/annotations/Scope & Namespace resolution.md b/docs/annotations/Scope & Namespace resolution.md new file mode 100644 index 0000000..0dff369 --- /dev/null +++ b/docs/annotations/Scope & Namespace resolution.md @@ -0,0 +1,60 @@ +# Subject 4: Scope & Namespace Resolution + +This document defines the annotations required to map identifier lookup, isolation, and visibility across disparate language environments. To ensure a lossless transition, the AST must track not just the name of a symbol, but the exact "binding site" and "reachability" rules of that symbol. + +## 1. Symbol Binding & Lookup Semantics + +| | | | +|---|---|---| +|**Annotation**|**Semantic Purpose**|**Example Conflict**| +|**`@Binding(Static)`**|Resolved at compile time based on text.|C, Java, Rust.| +|**`@Binding(Dynamic)`**|Resolved at runtime based on call stack.|Lisp (Emacs Lisp), Perl (local), early TeX.| +|**`@Lookup(Lexical)`**|Scope is defined by where code is written.|Most modern languages (JS, Python, C++).| +|**`@Lookup(Hoisted)`**|Declarations are moved to the top of scope.|JavaScript `var`, Python function definitions.| + +## 2. Closure Captures & Environment Mapping + +When a function "remembers" its environment, we must annotate how it accesses those external variables to prevent memory leaks or data races. + +### A. Capture Strategies (`@Capture`) + +- **`@Capture(Value)`**: A copy of the variable at the time of creation (C++ `[x]`). + +- **`@Capture(Ref)`**: A reference to the live variable (JS, Python, C++ `[&x]`). + +- **`@Capture(Move)`**: Transfer of ownership into the closure (Rust `move ||`). + + +**Projection Logic:** * **Into C:** The engine must generate a "Context Struct" containing all captured values/pointers and pass it as a hidden first argument to the projected function. + +## 3. Visibility & Access Control + +| | | | | +|---|---|---|---| +|**Annotation**|**Level**|**Target Action (into Java)**|**Target Action (into Python)**| +|**`@Visibility(Private)`**|Strictly local to the owner.|`private` keyword.|`__name` (double underscore) mangling.| +|**`@Visibility(Internal)`**|Visible within the same module/binary.|`internal` / `package-private`.|Standard naming; logic enforced via linter.| +|**`@Visibility(Friend)`**|Visible to specific other entities.|C++ `friend` class/func.|Requires interface projection or reflection.| + +## 4. Namespace & Module Resolution + +Handling how names are qualified and imported. + +- **`@Namespace(Qual)`**: The symbol requires a prefix (e.g., `std::vector`). + +- **`@Namespace(Flat)`**: The symbol exists in a global or current flat scope. + +- **`@Conflict(Shadow)`**: Annotates where a local variable "hides" an outer one. + + +**Projection Logic:** + +If projecting from a **Qualified** language (C++) to a **Flat** one (Global-heavy JS), the engine must "mangle" names (e.g., `std_vector`) to avoid collisions. + +## 5. The "Legacy Symbol" Problem + +For modernizing C++ or Fortran, we often encounter "Global State" or "Header Bloat." + +- **Annotation:** `@Scope(Global_Leaked)` + +- **Transformation:** Identify globals that are only used in one logical unit and "demote" them to `@Scope(Local)` or wrap them in a `@Singleton` class during projection to a safe language like Rust. \ No newline at end of file diff --git a/docs/annotations/Shims and Escape Hatches.md b/docs/annotations/Shims and Escape Hatches.md new file mode 100644 index 0000000..aa44946 --- /dev/null +++ b/docs/annotations/Shims and Escape Hatches.md @@ -0,0 +1,69 @@ +# Subject 5: Target-Specific Shims & Escape Hatches + +This document defines the annotations required to preserve platform-specific or language-specific features that cannot be mapped to universal semantics. These "Shims" ensure that even when a feature is "inexpressible" in the target language, the intent is preserved through emulation or raw embedding. + +## 1. Hardware & Platform Interop + +When code interacts directly with the environment (OS, CPU, or specific hardware), we use these annotations to flag non-portable nodes. + +| | | | +|---|---|---| +|**Annotation**|**Semantic Purpose**|**Example Conflict**| +|**`@Intrinsic`**|Maps to a specific CPU instruction (SIMD, AES, etc.).|Intel AVX instructions in C++ vs. high-level Go.| +|**`@Raw(LangID)`**|Literal code string in a specific language.|Inline Assembly in C or `__asm__`.| +|**`@CallingConv`**|Specific binary interface (stdcall, cdecl, fastcall).|Essential for FFI (Foreign Function Interface) stability.| +|**`@Link(Name)`**|Linkage requirement for external binary symbols.|Referencing a `.lib` or `.so` file.| + +## 2. Emulation of "Inexpressible" Concepts + +If the source language has a feature that the target lacks (e.g., C++ Multiple Inheritance into Java), we use these shims. + +### A. Feature Emulation (`@Shim(Strategy)`) + +- **`@Shim(VTable)`**: Manual virtual table generation for languages without polymorphism. + +- **`@Shim(Trampoline)`**: Used for deep recursion in languages without Tail-Call Optimization (TCO). + +- **`@Shim(Union_Tag)`**: Injecting a discriminator into raw C unions to make them "Safe Sum Types" in Rust/Swift. + + +### B. Pointer Arithmetic Shim (`@Pointer(Arithmetic)`) + +- **Source:** C, C++, Assembly. + +- **Target Action (into Java):** The engine must project this into a `ByteBuffer` or `Unsafe` offset calculation. + +- **Target Action (into Rust):** Maps to an `unsafe { ptr.offset(n) }` block. + + +## 3. The "Legacy Escape Hatch" + +Modernizing 40 trillion lines of code requires handling "The Unknown." + +- **Annotation:** `@Opaque(Reason)` + +- **Definition:** Identifies code that the transpiler cannot safely interpret but must carry forward. + +- **Projection Logic:** + + - The engine wraps the code in a "Foreign Block" (e.g., `extern "C"` in C++ or `Native` methods in Java). + + - It triggers a "Review Required" flag in the MPS editor, signaling that a human or specialized AI needs to provide a semantic mapping. + + +## 4. Conditional Compilation (The `#ifdef` Problem) + +| | | | +|---|---|---| +|**Annotation**|**Semantic Purpose**|**Target Action**| +|**`@Target(Platform)`**|Code only valid for specific OS/Arch.|Maps to `#[cfg]` in Rust or `#if` in C#.| +|**`@Feature(Flag)`**|Code enabled by build-time flags.|Maps to build tags or conditional constants.| + +## 5. Metadata Preservation (The "Lossless" Guarantee) + +- **`@Original(SourceCode)`**: Stores the original raw string of the ingested legacy code. + +- **`@Mapping(History)`**: Tracks every transformation the node has undergone (e.g., "From C pointer to Rust Reference"). + + +**Projection Conflict:** If a legacy C function uses a `void*` for generic data, and we are projecting to a strictly typed language, the `@Shim(Cast)` annotation tracks the intended type discovered through data-flow analysis, allowing the engine to "Force" a type-safe cast in the target. \ No newline at end of file diff --git a/docs/annotations/Type System and Variance.md b/docs/annotations/Type System and Variance.md new file mode 100644 index 0000000..d5aca34 --- /dev/null +++ b/docs/annotations/Type System and Variance.md @@ -0,0 +1,63 @@ +# Subject 2: Type System & Variance + +This document defines the annotations required to map disparate type systems into a universal AST. To achieve lossless transpilation, we must capture the "underlying truth" of a type—its size, its flexibility, and its identity—rather than just its high-level name. + +## 1. Core Type Annotations + +| | | | +|---|---|---| +|**Annotation**|**Semantic Purpose**|**Example Conflict**| +|**`@BitWidth(N)`**|Precise size of the primitive.|C `int` (platform dependent) vs. Java `int` (32-bit).| +|**`@Endian(Big|Little)`**|Byte order for multi-byte types.| +|**`@Layout(Packed|Aligned)`**|Memory padding constraints.| +|**`@Nullability(Strict|Nullable)`**|Whether the type can represent "None".| +|**`@Variance(Co|Contra|In)`**| + +## 2. Identity & Structural Mapping + +Languages differ on what makes two types "the same." We use these annotations to tell the projection engine how to handle comparisons and assignments. + +### A. Nominal Identity (`@Identity(Nominal)`) + +- **Source:** Java, C++, Swift. + +- **Definition:** Types are different even if they look the same (e.g., `class User` vs `class Admin` with identical fields). + +- **Projection Logic:** In TypeScript (structural), the engine must inject a "brand" or "tag" to prevent them from being interchangeable. + + +### B. Structural Identity (`@Identity(Structural)`) + +- **Source:** TypeScript, Go, OCaml. + +- **Definition:** If it walks like a duck and quacks like a duck, it is a duck. + +- **Projection Logic:** In Java (nominal), the engine might need to generate an `Interface` that both classes implement to satisfy the compiler. + + +## 3. Immutability & Constancy + +| | | | | +|---|---|---|---| +|**Annotation**|**Level**|**Target Action (into C++)**|**Target Action (into JS)**| +|**`@Mut(Shallow)`**|Top-level is immutable.|`Type * const ptr`|`Object.freeze()`| +|**`@Mut(Deep)`**|Everything reachable is immutable.|`const Type * const ptr`|Recursive `Object.freeze` or conversion to `ReadonlyArray`.| +|**`@Mut(Interior)`**|Immutable container, mutable data.|`std::cell` or `mutable` keyword.|Standard variable with access controls.| + +## 4. Variance & Generics Projection + +Handling how `List` relates to `List` is a classic transpilation failure point. + +- **Lowering (High** $\to$ **Low):** When projecting Java's wildcards (`List`) into C++ templates, the engine must generate specific template constraints or use `std::variant` to preserve the flexible logic. + +- **Raising (Low** $\to$ **High):** When projecting C++'s template specialization into C#, the engine may need to use `Reflection` or generate multiple distinct classes if the generics don't map 1:1. + + +## 5. Type Erasure vs. Reification + +- **`@TypeState(Erased)`**: Metadata exists only at compile time (Java/TS). + +- **`@TypeState(Reified)`**: Metadata exists at runtime (C#/C++). + + +**Projection Conflict:** If source is `@TypeState(Reified)` (like C# `typeof(T)`) and target is `@TypeState(Erased)` (like Java), the projection engine **must** inject an extra `Class` parameter into constructors/methods to carry that type information forward. \ No newline at end of file diff --git a/docs/annotations/annotations.md b/docs/annotations/annotations.md new file mode 100644 index 0000000..e69de29 diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 4ab4120..a99b0fe 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -33,3 +33,6 @@ target_link_libraries(step5_test PRIVATE nlohmann_json::nlohmann_json) add_executable(step6_test tests/step6_test.cpp) target_include_directories(step6_test PRIVATE src) target_link_libraries(step6_test PRIVATE nlohmann_json::nlohmann_json) + +add_executable(step7_test tests/step7_test.cpp) +target_include_directories(step7_test PRIVATE src) diff --git a/editor/src/ast/Schema.h b/editor/src/ast/Schema.h new file mode 100644 index 0000000..9f0fe0b --- /dev/null +++ b/editor/src/ast/Schema.h @@ -0,0 +1,162 @@ +#pragma once +#include +#include +#include +#include +#include + +struct ChildRule { + std::vector allowedConcepts; + bool isSingleValued = false; // false = multi-valued (list), true = single-valued (pointer) +}; + +class ASTSchema { +public: + ASTSchema() { + initializeSchema(); + } + + // Check if a child concept is allowed under a parent concept in a specific role + bool isLegalChild(const std::string& parentConcept, const std::string& role, const std::string& childConcept) const { + auto parentIt = schema_.find(parentConcept); + if (parentIt == schema_.end()) { + return false; // Parent concept not in schema + } + + auto roleIt = parentIt->second.find(role); + if (roleIt == parentIt->second.end()) { + return false; // Role not defined for this parent + } + + const ChildRule& rule = roleIt->second; + + // Check if the child concept is in the allowed list + for (const auto& allowedConcept : rule.allowedConcepts) { + if (allowedConcept == childConcept) { + return true; + } + } + + return false; + } + + // Get the cardinality of a role (single vs multi-valued) + bool isSingleValued(const std::string& parentConcept, const std::string& role) const { + auto parentIt = schema_.find(parentConcept); + if (parentIt == schema_.end()) { + return false; // Parent concept not in schema + } + + auto roleIt = parentIt->second.find(role); + if (roleIt == parentIt->second.end()) { + return false; // Role not defined for this parent + } + + return roleIt->second.isSingleValued; + } + + // Get allowed child concepts for a role + std::vector getAllowedConcepts(const std::string& parentConcept, const std::string& role) const { + std::vector result; + auto parentIt = schema_.find(parentConcept); + if (parentIt == schema_.end()) { + return result; // Empty if parent not found + } + + auto roleIt = parentIt->second.find(role); + if (roleIt == parentIt->second.end()) { + return result; // Empty if role not found + } + + return roleIt->second.allowedConcepts; + } + +private: + std::map> schema_; + + void initializeSchema() { + // Module concept rules + addRule("Module", "functions", {"Function"}, false); // multi-valued + addRule("Module", "variables", {"Variable"}, false); // multi-valued + addRule("Module", "targetLanguage", {}, true); // property, not used for AST nodes + addRule("Module", "annotations", {"DerefStrategy", "OptimizationLock", "LangSpecific"}, false); // multi-valued + + // Function concept rules + addRule("Function", "parameters", {"Parameter"}, false); // multi-valued + addRule("Function", "returnType", {"PrimitiveType", "ListType", "SetType", "MapType", "TupleType", "ArrayType", "OptionalType", "CustomType"}, true); // single-valued + addRule("Function", "body", {"Block", "Assignment", "IfStatement", "WhileLoop", "ForLoop", "Return", "ExpressionStatement"}, false); // multi-valued + addRule("Function", "annotations", {"DerefStrategy", "OptimizationLock", "LangSpecific"}, false); // multi-valued + + // Parameter concept rules + addRule("Parameter", "type", {"PrimitiveType", "ListType", "SetType", "MapType", "TupleType", "ArrayType", "OptionalType", "CustomType"}, true); // single-valued + addRule("Parameter", "defaultValue", {"IntegerLiteral", "FloatLiteral", "StringLiteral", "BooleanLiteral", "NullLiteral", "ListLiteral", "BinaryOperation", "UnaryOperation", "FunctionCall", "VariableReference"}, true); // single-valued + + // Variable concept rules + addRule("Variable", "type", {"PrimitiveType", "ListType", "SetType", "MapType", "TupleType", "ArrayType", "OptionalType", "CustomType"}, true); // single-valued + addRule("Variable", "initializer", {"IntegerLiteral", "FloatLiteral", "StringLiteral", "BooleanLiteral", "NullLiteral", "ListLiteral", "BinaryOperation", "UnaryOperation", "FunctionCall", "VariableReference"}, true); // single-valued + addRule("Variable", "annotations", {"DerefStrategy", "OptimizationLock", "LangSpecific"}, false); // multi-valued + + // Block concept rules + addRule("Block", "statements", {"Assignment", "IfStatement", "WhileLoop", "ForLoop", "Return", "ExpressionStatement"}, false); // multi-valued + + // Assignment concept rules + addRule("Assignment", "target", {"VariableReference"}, true); // single-valued + addRule("Assignment", "value", {"IntegerLiteral", "FloatLiteral", "StringLiteral", "BooleanLiteral", "NullLiteral", "ListLiteral", "BinaryOperation", "UnaryOperation", "FunctionCall", "VariableReference", "IndexAccess", "MemberAccess"}, true); // single-valued + + // IfStatement concept rules + addRule("IfStatement", "condition", {"BinaryOperation", "UnaryOperation", "FunctionCall", "VariableReference", "IntegerLiteral", "FloatLiteral", "StringLiteral", "BooleanLiteral", "NullLiteral"}, true); // single-valued + addRule("IfStatement", "thenBranch", {"Block", "Assignment", "IfStatement", "WhileLoop", "ForLoop", "Return", "ExpressionStatement"}, false); // multi-valued + addRule("IfStatement", "elseBranch", {"Block", "Assignment", "IfStatement", "WhileLoop", "ForLoop", "Return", "ExpressionStatement"}, false); // multi-valued + + // WhileLoop concept rules + addRule("WhileLoop", "condition", {"BinaryOperation", "UnaryOperation", "FunctionCall", "VariableReference", "IntegerLiteral", "FloatLiteral", "StringLiteral", "BooleanLiteral", "NullLiteral"}, true); // single-valued + addRule("WhileLoop", "body", {"Block", "Assignment", "IfStatement", "WhileLoop", "ForLoop", "Return", "ExpressionStatement"}, false); // multi-valued + + // ForLoop concept rules + addRule("ForLoop", "iteratorName", {}, true); // property, not used for AST nodes + addRule("ForLoop", "iterable", {"VariableReference", "FunctionCall", "BinaryOperation", "UnaryOperation", "ListLiteral"}, true); // single-valued + addRule("ForLoop", "body", {"Block", "Assignment", "IfStatement", "WhileLoop", "ForLoop", "Return", "ExpressionStatement"}, false); // multi-valued + + // Return concept rules + addRule("Return", "value", {"IntegerLiteral", "FloatLiteral", "StringLiteral", "BooleanLiteral", "NullLiteral", "ListLiteral", "BinaryOperation", "UnaryOperation", "FunctionCall", "VariableReference", "IndexAccess", "MemberAccess"}, true); // single-valued + + // ExpressionStatement concept rules + addRule("ExpressionStatement", "expression", {"BinaryOperation", "UnaryOperation", "FunctionCall", "VariableReference", "IntegerLiteral", "FloatLiteral", "StringLiteral", "BooleanLiteral", "NullLiteral", "ListLiteral", "IndexAccess", "MemberAccess"}, true); // single-valued + + // BinaryOperation concept rules + addRule("BinaryOperation", "left", {"IntegerLiteral", "FloatLiteral", "StringLiteral", "BooleanLiteral", "NullLiteral", "ListLiteral", "BinaryOperation", "UnaryOperation", "FunctionCall", "VariableReference", "IndexAccess", "MemberAccess"}, true); // single-valued + addRule("BinaryOperation", "right", {"IntegerLiteral", "FloatLiteral", "StringLiteral", "BooleanLiteral", "NullLiteral", "ListLiteral", "BinaryOperation", "UnaryOperation", "FunctionCall", "VariableReference", "IndexAccess", "MemberAccess"}, true); // single-valued + + // UnaryOperation concept rules + addRule("UnaryOperation", "operand", {"IntegerLiteral", "FloatLiteral", "StringLiteral", "BooleanLiteral", "NullLiteral", "ListLiteral", "BinaryOperation", "UnaryOperation", "FunctionCall", "VariableReference", "IndexAccess", "MemberAccess"}, true); // single-valued + + // FunctionCall concept rules + addRule("FunctionCall", "functionName", {}, true); // property, not used for AST nodes + addRule("FunctionCall", "arguments", {"IntegerLiteral", "FloatLiteral", "StringLiteral", "BooleanLiteral", "NullLiteral", "ListLiteral", "BinaryOperation", "UnaryOperation", "FunctionCall", "VariableReference", "IndexAccess", "MemberAccess"}, false); // multi-valued + + // VariableReference concept rules + addRule("VariableReference", "variableName", {}, true); // property, not used for AST nodes + + // ListLiteral concept rules + addRule("ListLiteral", "elements", {"IntegerLiteral", "FloatLiteral", "StringLiteral", "BooleanLiteral", "NullLiteral", "ListLiteral", "BinaryOperation", "UnaryOperation", "FunctionCall", "VariableReference", "IndexAccess", "MemberAccess"}, false); // multi-valued + + // IndexAccess concept rules + addRule("IndexAccess", "target", {"VariableReference", "FunctionCall", "IndexAccess", "MemberAccess"}, true); // single-valued + addRule("IndexAccess", "index", {"IntegerLiteral", "VariableReference", "BinaryOperation", "FunctionCall"}, true); // single-valued + + // MemberAccess concept rules + addRule("MemberAccess", "target", {"VariableReference", "FunctionCall", "IndexAccess", "MemberAccess"}, true); // single-valued + addRule("MemberAccess", "memberName", {}, true); // property, not used for AST nodes + + // Type concepts - no child rules needed as they're leaf nodes for types + + // Annotation concepts - no child rules needed as they're leaf nodes for annotations + } + + void addRule(const std::string& parentConcept, const std::string& role, const std::vector& allowedConcepts, bool isSingleValued) { + ChildRule rule; + rule.allowedConcepts = allowedConcepts; + rule.isSingleValued = isSingleValued; + schema_[parentConcept][role] = rule; + } +}; \ No newline at end of file diff --git a/editor/tests/step7_test.cpp b/editor/tests/step7_test.cpp new file mode 100644 index 0000000..7ab73c0 --- /dev/null +++ b/editor/tests/step7_test.cpp @@ -0,0 +1,68 @@ +// Step 7: Schema validation. Verify legal placements accepted, illegal rejected. + +#include "../src/ast/ASTNode.h" +#include "../src/ast/Module.h" +#include "../src/ast/Function.h" +#include "../src/ast/Variable.h" +#include "../src/ast/Statement.h" +#include "../src/ast/Expression.h" +#include "../src/ast/Type.h" +#include "../src/ast/Annotation.h" +#include "../src/ast/Schema.h" +#include +#include + +int main() { + ASTSchema schema; + + // Test 1: Legal placements should be accepted + assert(schema.isLegalChild("Module", "functions", "Function")); + assert(schema.isLegalChild("Module", "variables", "Variable")); + assert(schema.isLegalChild("Function", "parameters", "Parameter")); + assert(schema.isLegalChild("Function", "returnType", "PrimitiveType")); + assert(schema.isLegalChild("Function", "body", "Return")); + assert(schema.isLegalChild("Function", "body", "Assignment")); + assert(schema.isLegalChild("Assignment", "target", "VariableReference")); + assert(schema.isLegalChild("Assignment", "value", "BinaryOperation")); + assert(schema.isLegalChild("BinaryOperation", "left", "VariableReference")); + assert(schema.isLegalChild("BinaryOperation", "right", "IntegerLiteral")); + + // Test 2: Illegal placements should be rejected + assert(!schema.isLegalChild("Module", "functions", "Variable")); // Variable not allowed in functions role + assert(!schema.isLegalChild("Module", "variables", "Function")); // Function not allowed in variables role + assert(!schema.isLegalChild("Function", "parameters", "Variable")); // Variable not allowed in parameters role + assert(!schema.isLegalChild("Function", "returnType", "Function")); // Function not allowed as return type + assert(!schema.isLegalChild("Assignment", "target", "IntegerLiteral")); // Literal not allowed as target + assert(!schema.isLegalChild("Assignment", "value", "Function")); // Function not allowed as value (directly) + assert(!schema.isLegalChild("IntegerLiteral", "value", "Variable")); // IntegerLiteral has no "value" role + assert(!schema.isLegalChild("NonExistentConcept", "someRole", "SomeChild")); // Non-existent parent + + // Test 3: Verify cardinality (single vs multi-valued roles) + assert(schema.isSingleValued("Function", "returnType")); // Single-valued + assert(!schema.isSingleValued("Function", "parameters")); // Multi-valued + assert(schema.isSingleValued("Assignment", "target")); // Single-valued + assert(schema.isSingleValued("Assignment", "value")); // Single-valued + assert(!schema.isSingleValued("Function", "body")); // Multi-valued + + // Test 4: Get allowed concepts + auto funcParamsAllowed = schema.getAllowedConcepts("Function", "parameters"); + assert(funcParamsAllowed.size() == 1); + assert(funcParamsAllowed[0] == "Parameter"); + + auto funcBodyAllowed = schema.getAllowedConcepts("Function", "body"); + assert(funcBodyAllowed.size() >= 3); // Should have multiple allowed concepts + bool hasAssignment = false, hasReturn = false, hasIf = false; + for (const auto& allowedConcept : funcBodyAllowed) { + if (allowedConcept == "Assignment") hasAssignment = true; + if (allowedConcept == "Return") hasReturn = true; + if (allowedConcept == "IfStatement") hasIf = true; + } + assert(hasAssignment && hasReturn && hasIf); + + // Test 5: Edge cases + assert(schema.getAllowedConcepts("NonExistentConcept", "someRole").empty()); + assert(schema.getAllowedConcepts("Function", "nonExistentRole").empty()); + + std::cout << "Step 7: PASS — Schema validation rejects illegal placements, accepts legal ones" << std::endl; + return 0; +} \ No newline at end of file diff --git a/languages/SemAnno/models/SemAnno.structure.mps b/languages/SemAnno/models/SemAnno.structure.mps index 3468f30..8226234 100644 --- a/languages/SemAnno/models/SemAnno.structure.mps +++ b/languages/SemAnno/models/SemAnno.structure.mps @@ -182,7 +182,7 @@ - + @@ -247,7 +247,7 @@ - + @@ -313,7 +313,7 @@ - + @@ -335,14 +335,14 @@ - + - + @@ -361,7 +361,7 @@ - + @@ -378,14 +378,14 @@ - + - + @@ -467,16 +467,16 @@ - + - + - + @@ -488,7 +488,7 @@ - + @@ -498,7 +498,7 @@ - + @@ -543,7 +543,7 @@ - + @@ -555,14 +555,14 @@ - + - + @@ -581,7 +581,7 @@ - + @@ -638,7 +638,7 @@ - + @@ -693,7 +693,7 @@ - + @@ -705,11 +705,11 @@ - + - + @@ -724,7 +724,7 @@ - + @@ -732,37 +732,37 @@ - + - + - + - + - + - + - + @@ -771,7 +771,7 @@ - +