From 568fa38f46e4325c66f7cb2b1be8df5e995104db Mon Sep 17 00:00:00 2001 From: Bill Date: Mon, 9 Feb 2026 18:02:57 -0700 Subject: [PATCH] Step 143: index Emacs Elisp functions --- PROGRESS.md | 1 + editor/CMakeLists.txt | 4 + editor/src/EditorState.h | 30 ++++++ editor/src/EmacsFunctionDiscovery.h | 139 ++++++++++++++++++++++++++++ editor/src/EmacsIntegration.h | 39 ++++++++ editor/src/EmacsPackageBrowser.h | 46 +++++---- editor/src/PrimitivesRegistry.h | 2 + editor/src/main.cpp | 7 +- editor/tests/step143_test.cpp | 50 ++++++++++ sprint5_plan.md | 2 +- 10 files changed, 301 insertions(+), 19 deletions(-) create mode 100644 editor/src/EmacsFunctionDiscovery.h create mode 100644 editor/tests/step143_test.cpp diff --git a/PROGRESS.md b/PROGRESS.md index 2579593..8a1895d 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -490,3 +490,4 @@ Sprint 5 in progress. Step 141 (Emacs daemon with user config) done. Next: Step | 2026-02-09 | Codex | Added FEATURE_REQUESTS.md to track security vulnerability awareness and semantic library annotations for future sprints. | | 2026-02-09 | Codex | Added LLM tooling + MCP bridge feature request to FEATURE_REQUESTS.md. | | 2026-02-10 | Codex | Step 142: Emacs package browser panel with loaded/available status, load actions, and package queries via emacsclient. 7/7 tests pass. | +| 2026-02-10 | Codex | Step 143: Elisp function discovery via apropos/describe-function, Emacs function indexing into ExternalModule + LibraryIndex, and elisp primitive updates. 7/7 tests pass. | diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index d36baf5..ca97d94 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -786,6 +786,10 @@ add_executable(step142_test tests/step142_test.cpp) target_include_directories(step142_test PRIVATE src) target_link_libraries(step142_test PRIVATE nlohmann_json::nlohmann_json imgui::imgui) +add_executable(step143_test tests/step143_test.cpp) +target_include_directories(step143_test PRIVATE src) +target_link_libraries(step143_test PRIVATE nlohmann_json::nlohmann_json imgui::imgui) + find_package(SDL2 CONFIG REQUIRED) find_package(OpenGL REQUIRED) find_package(glad CONFIG REQUIRED) diff --git a/editor/src/EditorState.h b/editor/src/EditorState.h index 112b393..01d82be 100644 --- a/editor/src/EditorState.h +++ b/editor/src/EditorState.h @@ -53,6 +53,7 @@ #include "IncrementalOptimizer.h" #include "EmacsIntegration.h" #include "EmacsPackageBrowser.h" +#include "EmacsFunctionDiscovery.h" #include "ast/Serialization.h" #include "ast/Generator.h" #include "ast/Annotation.h" @@ -182,6 +183,8 @@ struct EditorState { CompositionPanelState compositionPanel; bool showEmacsPackagesPanel = false; EmacsPackageBrowserState emacsPackages; + EmacsFunctionIndex emacsFunctionIndex; + bool emacsFunctionIndexDirty = true; struct LibraryIndexRequest { std::string name; std::string version; @@ -453,6 +456,9 @@ struct EditorState { primitives.setRoot(activeAST()); primitives.setLanguage(active()->language); recordUndoSnapshot(); + if (language == "elisp") { + emacsFunctionIndexDirty = true; + } if (path.rfind("(untitled", 0) != 0) watcher.watch(path); if (lsp && path.rfind("(untitled", 0) != 0) { lsp->didOpen(toFileUri(path), language, content, activeBuffer->lspVersion); @@ -545,6 +551,9 @@ struct EditorState { symbolsPending = true; symbolsLastChange = ImGui::GetTime(); if (lsp) lsp->clearDocumentSymbols(); + if (active() && active()->language == "elisp") { + emacsFunctionIndexDirty = true; + } } std::filesystem::path configDir() const { @@ -970,10 +979,28 @@ struct EditorState { } } + void updateEmacsFunctionIndex() { + if (!active() || active()->language != "elisp") return; + std::string error; + auto packages = queryEmacsPackageList(emacs, false, outputLog, error); + if (!error.empty()) return; + refreshEmacsFunctionIndex(emacsFunctionIndex, emacs, packages, outputLog); + addEmacsSymbolsToLibraryIndex(libraryIndex, emacsFunctionIndex); + emacsFunctionIndexDirty = false; + rebuildExternalModulesFromIndex(); + outputLog += "[emacs] Indexed functions for " + + std::to_string(emacsFunctionIndex.functionsByPackage.size()) + + " packages.\n"; + } + void rebuildExternalModulesFromIndex() { Module* ast = activeAST(); if (!ast) return; rebuildExternalModules(ast, dependencyPanel.deps, libraryIndex); + if (active() && active()->language == "elisp") { + int signatureId = 0; + appendEmacsExternalModules(ast, emacsFunctionIndex, signatureId); + } if (active()) active()->orchestratorDirty = true; } @@ -1337,6 +1364,9 @@ struct EditorState { } active()->orchestratorDirty = true; active()->highlightsDirty = true; + if (lang == "elisp") { + emacsFunctionIndexDirty = true; + } } std::string buildRunCommand(const std::string& path, diff --git a/editor/src/EmacsFunctionDiscovery.h b/editor/src/EmacsFunctionDiscovery.h new file mode 100644 index 0000000..1854252 --- /dev/null +++ b/editor/src/EmacsFunctionDiscovery.h @@ -0,0 +1,139 @@ +#pragma once +#include "EmacsIntegration.h" +#include "EmacsPackageBrowser.h" +#include "LibraryIndexer.h" +#include "ast/Module.h" +#include "ast/ExternalModule.h" +#include "ast/TypeSignature.h" +#include +#include +#include +#include + +struct EmacsFunctionDoc { + std::string name; + std::string signature; + std::string doc; +}; + +struct EmacsFunctionIndex { + std::unordered_map> functionsByPackage; +}; + +static std::vector parseLines(const std::string& text) { + std::vector out; + size_t start = 0; + while (start < text.size()) { + size_t end = text.find('\n', start); + if (end == std::string::npos) end = text.size(); + std::string line = trimCopy(text.substr(start, end - start)); + if (!line.empty()) out.push_back(std::move(line)); + start = end + 1; + } + return out; +} + +static std::vector queryEmacsFunctions(EmacsConnection& emacs, + const std::string& prefix, + std::string& logOut) { + std::string cmd = ElispCommandBuilder::aproposFunctions(prefix); + std::string resp = emacs.sendCommand(cmd); + if (resp.empty() && !emacs.getLastError().empty()) { + logOut += "[emacs] " + emacs.getLastError() + "\n"; + return {}; + } + return parseLines(resp); +} + +static EmacsFunctionDoc queryEmacsFunctionDoc(EmacsConnection& emacs, + const std::string& name, + std::string& logOut) { + EmacsFunctionDoc info; + info.name = name; + std::string cmd = ElispCommandBuilder::describeFunction(name); + std::string resp = emacs.sendCommand(cmd); + if (resp.empty() && !emacs.getLastError().empty()) { + logOut += "[emacs] " + emacs.getLastError() + "\n"; + return info; + } + auto pos = resp.find('\n'); + if (pos == std::string::npos) { + info.signature = trimCopy(resp); + } else { + info.signature = trimCopy(resp.substr(0, pos)); + info.doc = trimCopy(resp.substr(pos + 1)); + } + return info; +} + +static std::vector buildPackageFunctionPrefixes(const std::string& pkgName) { + std::vector prefixes; + if (pkgName.empty()) return prefixes; + prefixes.push_back(pkgName + "-"); + prefixes.push_back(pkgName); + auto dash = pkgName.find('-'); + if (dash != std::string::npos) { + prefixes.push_back(pkgName.substr(0, dash) + "-"); + } + return prefixes; +} + +static void refreshEmacsFunctionIndex(EmacsFunctionIndex& index, + EmacsConnection& emacs, + const std::vector& packages, + std::string& logOut) { + index.functionsByPackage.clear(); + for (const auto& pkg : packages) { + if (pkg.status != "loaded") continue; + std::vector functions; + for (const auto& prefix : buildPackageFunctionPrefixes(pkg.name)) { + functions = queryEmacsFunctions(emacs, "^" + prefix, logOut); + if (!functions.empty()) break; + } + if (functions.empty()) continue; + std::vector docs; + docs.reserve(functions.size()); + for (const auto& fn : functions) { + docs.push_back(queryEmacsFunctionDoc(emacs, fn, logOut)); + } + index.functionsByPackage[pkg.name] = std::move(docs); + } +} + +static void appendEmacsExternalModules(Module* module, + const EmacsFunctionIndex& index, + int& signatureId) { + if (!module) return; + for (const auto& kv : index.functionsByPackage) { + const std::string& pkgName = kv.first; + auto* ext = new ExternalModule("emacs_" + pkgName, pkgName, "elisp"); + for (const auto& fn : kv.second) { + auto* sig = new TypeSignature("emacs_sig_" + std::to_string(signatureId++), fn.name); + ext->addChild("signatures", sig); + } + module->addChild("externalModules", ext); + } +} + +static void addEmacsSymbolsToLibraryIndex(LibraryIndexData& libIndex, + const EmacsFunctionIndex& index) { + for (const auto& kv : index.functionsByPackage) { + const std::string& pkgName = kv.first; + std::vector symbols; + symbols.reserve(kv.second.size()); + for (const auto& fn : kv.second) { + LSPClient::WorkspaceSymbol sym; + sym.name = fn.name; + sym.containerName = pkgName; + if (!fn.signature.empty()) { + sym.detail = fn.signature; + if (!fn.doc.empty()) sym.detail += "\n" + fn.doc; + } else { + sym.detail = fn.doc; + } + sym.kind = 12; // Function + symbols.push_back(std::move(sym)); + } + if (!symbols.empty()) libIndex.symbolsByLibrary[pkgName] = std::move(symbols); + } +} diff --git a/editor/src/EmacsIntegration.h b/editor/src/EmacsIntegration.h index a5e5641..64675bd 100644 --- a/editor/src/EmacsIntegration.h +++ b/editor/src/EmacsIntegration.h @@ -65,6 +65,23 @@ public: result += ")"; return result; } + + // Build an Elisp expression to list functions matching a prefix/regex + static std::string aproposFunctions(const std::string& prefixRegex) { + std::string pattern = prefixRegex; + return "(mapconcat #'symbol-name (apropos-internal \"" + + escapeString(pattern) + "\" 'fboundp) \"\\n\")"; + } + + // Build an Elisp expression to describe a function (signature + doc) + static std::string describeFunction(const std::string& name) { + std::string sym = escapeString(name); + return "(let* ((sym '" + sym + ") " + "(args (if (fboundp 'help-function-arglist) " + " (help-function-arglist sym t) nil)) " + "(doc (or (documentation sym t) \"\"))) " + "(concat (if args (prin1-to-string args) \"\") \"\\n\" doc))"; + } }; // --------------------------------------------------------------------------- @@ -279,6 +296,27 @@ public: if (elispCommand.find("(require '") != std::string::npos) { return "t"; } + if (elispCommand.find("apropos-internal") != std::string::npos) { + if (elispCommand.find("use-package") != std::string::npos) { + return "use-package\nuse-package-alias"; + } + if (elispCommand.find("projectile") != std::string::npos) { + return "projectile-find-file\nprojectile-switch-project"; + } + if (elispCommand.find("cl-") != std::string::npos) { + return "cl-loop\ncl-mapcar"; + } + return ""; + } + if (elispCommand.find("help-function-arglist") != std::string::npos) { + if (elispCommand.find("use-package") != std::string::npos) { + return "(name &rest args)\nConfigure package via use-package."; + } + if (elispCommand.find("projectile") != std::string::npos) { + return "(project)\nProjectile project switching."; + } + return "()\nEmacs function."; + } // Unknown command — simulate error lastError_ = "void-function"; @@ -301,3 +339,4 @@ private: std::string lastSentCommand_; std::string lastInitPath_; }; + diff --git a/editor/src/EmacsPackageBrowser.h b/editor/src/EmacsPackageBrowser.h index 1ef5b84..0d8773e 100644 --- a/editor/src/EmacsPackageBrowser.h +++ b/editor/src/EmacsPackageBrowser.h @@ -121,26 +121,34 @@ static std::vector mergePackageLists(const std::vector queryEmacsPackageList(EmacsConnection& emacs, + bool includeAvailable, + std::string& logOut, + std::string& outError) { + outError.clear(); + std::string loadedText = emacs.sendCommand(buildPackageListCommand("package-alist")); + if (loadedText.empty() && !emacs.getLastError().empty()) { + outError = emacs.getLastError(); + logOut += "[emacs] " + outError + "\n"; + } + auto loaded = parsePackageListText(loadedText); + std::vector available; + if (includeAvailable) { + std::string availText = emacs.sendCommand(buildPackageListCommand("package-archive-contents")); + if (availText.empty() && !emacs.getLastError().empty()) { + outError = emacs.getLastError(); + logOut += "[emacs] " + outError + "\n"; + } + available = parsePackageListText(availText); + } + return mergePackageLists(loaded, available, includeAvailable); +} + static void refreshEmacsPackages(EmacsPackageBrowserState& state, EmacsConnection& emacs, std::string& logOut) { state.lastError.clear(); - std::string loadedText = emacs.sendCommand(buildPackageListCommand("package-alist")); - if (loadedText.empty() && !emacs.getLastError().empty()) { - state.lastError = emacs.getLastError(); - logOut += "[emacs] " + state.lastError + "\n"; - } - auto loaded = parsePackageListText(loadedText); - std::vector available; - if (state.showAvailable) { - std::string availText = emacs.sendCommand(buildPackageListCommand("package-archive-contents")); - if (availText.empty() && !emacs.getLastError().empty()) { - state.lastError = emacs.getLastError(); - logOut += "[emacs] " + state.lastError + "\n"; - } - available = parsePackageListText(availText); - } - state.packages = mergePackageLists(loaded, available, state.showAvailable); + state.packages = queryEmacsPackageList(emacs, state.showAvailable, logOut, state.lastError); state.needsRefresh = false; } @@ -165,9 +173,10 @@ static bool emacsPackageMatchesFilter(const EmacsPackageEntry& entry, const std: return hay.find(needle) != std::string::npos; } -static void renderEmacsPackageBrowser(EmacsPackageBrowserState& state, +static bool renderEmacsPackageBrowser(EmacsPackageBrowserState& state, EmacsConnection& emacs, std::string& logOut) { + bool updated = false; if (ImGui::Button("Refresh")) { state.needsRefresh = true; } @@ -180,6 +189,7 @@ static void renderEmacsPackageBrowser(EmacsPackageBrowserState& state, if (state.needsRefresh) { refreshEmacsPackages(state, emacs, logOut); + updated = true; } ImGui::Separator(); @@ -238,10 +248,12 @@ static void renderEmacsPackageBrowser(EmacsPackageBrowserState& state, logOut += "[emacs] Package not installed: " + loadTarget + "\n"; } state.needsRefresh = true; + updated = true; } } if (!loadTarget.empty()) { ImGui::SameLine(); ImGui::TextDisabled("%s", loadTarget.c_str()); } + return updated; } diff --git a/editor/src/PrimitivesRegistry.h b/editor/src/PrimitivesRegistry.h index a70c769..73d5b5f 100644 --- a/editor/src/PrimitivesRegistry.h +++ b/editor/src/PrimitivesRegistry.h @@ -122,6 +122,8 @@ private: add("nil", "constant"); } else if (lang == "elisp") { add("message", "function"); + add("require", "function"); + add("use-package", "function"); add("t", "constant"); add("nil", "constant"); } diff --git a/editor/src/main.cpp b/editor/src/main.cpp index f8ede42..a7a41fd 100644 --- a/editor/src/main.cpp +++ b/editor/src/main.cpp @@ -193,6 +193,9 @@ int main(int, char**) { } state.pollLspMessages(); state.processLibraryIndexResponses(); + if (state.emacsFunctionIndexDirty) { + state.updateEmacsFunctionIndex(); + } // Start frame ImGui_ImplOpenGL3_NewFrame(); @@ -614,7 +617,9 @@ int main(int, char**) { if (state.showEmacsPackagesPanel) { ImGui::Begin("Emacs Packages", &state.showEmacsPackagesPanel); ImGui::PushFont(uiFont); - renderEmacsPackageBrowser(state.emacsPackages, state.emacs, state.outputLog); + if (renderEmacsPackageBrowser(state.emacsPackages, state.emacs, state.outputLog)) { + state.emacsFunctionIndexDirty = true; + } ImGui::PopFont(); ImGui::End(); } diff --git a/editor/tests/step143_test.cpp b/editor/tests/step143_test.cpp new file mode 100644 index 0000000..8141507 --- /dev/null +++ b/editor/tests/step143_test.cpp @@ -0,0 +1,50 @@ +// Step 143 TDD Test: Elisp function discovery and indexing +#include "EmacsFunctionDiscovery.h" +#include + +static void expect(bool cond, const std::string& name, int& passed, int& failed) { + if (cond) { + std::cout << "Test " << (passed + failed + 1) << " PASS: " << name << "\n"; + ++passed; + } else { + std::cout << "Test " << (passed + failed + 1) << " FAIL: " << name << "\n"; + ++failed; + } +} + +int main() { + int passed = 0; + int failed = 0; + + MockEmacsConnection mock; + std::string log; + std::string error; + + auto packages = queryEmacsPackageList(mock, false, log, error); + expect(!packages.empty(), "loaded packages fetched", passed, failed); + + auto funcs = queryEmacsFunctions(mock, "^use-package", log); + expect(!funcs.empty(), "apropos returns functions", passed, failed); + + auto doc = queryEmacsFunctionDoc(mock, "use-package", log); + expect(!doc.signature.empty(), "describe-function signature", passed, failed); + expect(!doc.doc.empty(), "describe-function docstring", passed, failed); + + EmacsFunctionIndex index; + refreshEmacsFunctionIndex(index, mock, packages, log); + expect(index.functionsByPackage.count("use-package") > 0, "index contains use-package", passed, failed); + + Module module("mod1", "test", "elisp"); + int sigId = 0; + appendEmacsExternalModules(&module, index, sigId); + auto extMods = module.getChildren("externalModules"); + expect(!extMods.empty(), "external modules appended", passed, failed); + + LibraryIndexData libIndex; + addEmacsSymbolsToLibraryIndex(libIndex, index); + expect(libIndex.symbolsByLibrary.count("use-package") > 0, "library index updated", passed, failed); + + std::cout << "\n=== Step 143 Results: " << passed << " passed, " + << failed << " failed ===\n"; + return failed == 0 ? 0 : 1; +} diff --git a/sprint5_plan.md b/sprint5_plan.md index ef9ee00..92b87e2 100644 --- a/sprint5_plan.md +++ b/sprint5_plan.md @@ -186,7 +186,7 @@ functionality through the Whetstone UI. to the daemon. Status: loaded/available/not-installed. *New:* `EmacsPackageBrowser` panel -- [ ] **Step 143: Elisp function discovery** +- [x] **Step 143: Elisp function discovery** Query the running Emacs daemon for available functions from loaded packages. Send `(apropos-internal "prefix")` → get function list. Send `(describe-function 'name)` → get docstring and signature.