bill/whetstone_DSL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Step 143: index Emacs Elisp functions

Browse Source
This commit is contained in:
Bill
2026-02-09 18:02:57 -07:00
parent beaec58073
commit 568fa38f46
10 changed files with 301 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
PROGRESS.md
View File

@@ -490,3 +490,4 @@ Sprint 5 in progress. Step 141 (Emacs daemon with user config) done. Next: Step
| 2026-02-09 | Codex | Added FEATURE_REQUESTS.md to track security vulnerability awareness and semantic library annotations for future sprints. |
| 2026-02-09 | Codex | Added LLM tooling + MCP bridge feature request to FEATURE_REQUESTS.md. |
| 2026-02-10 | Codex | Step 142: Emacs package browser panel with loaded/available status, load actions, and package queries via emacsclient. 7/7 tests pass. |
| 2026-02-10 | Codex | Step 143: Elisp function discovery via apropos/describe-function, Emacs function indexing into ExternalModule + LibraryIndex, and elisp primitive updates. 7/7 tests pass. |

4
editor/CMakeLists.txt
View File

@@ -786,6 +786,10 @@ add_executable(step142_test tests/step142_test.cpp)
target_include_directories(step142_test PRIVATE src)
target_link_libraries(step142_test PRIVATE nlohmann_json::nlohmann_json imgui::imgui)
add_executable(step143_test tests/step143_test.cpp)
target_include_directories(step143_test PRIVATE src)
target_link_libraries(step143_test PRIVATE nlohmann_json::nlohmann_json imgui::imgui)
find_package(SDL2 CONFIG REQUIRED)
find_package(OpenGL REQUIRED)
find_package(glad CONFIG REQUIRED)

30
editor/src/EditorState.h
View File

@@ -53,6 +53,7 @@
#include "IncrementalOptimizer.h"
#include "EmacsIntegration.h"
#include "EmacsPackageBrowser.h"
#include "EmacsFunctionDiscovery.h"
#include "ast/Serialization.h"
#include "ast/Generator.h"
#include "ast/Annotation.h"
@@ -182,6 +183,8 @@ struct EditorState {
CompositionPanelState compositionPanel;
bool showEmacsPackagesPanel = false;
EmacsPackageBrowserState emacsPackages;
EmacsFunctionIndex emacsFunctionIndex;
bool emacsFunctionIndexDirty = true;
struct LibraryIndexRequest {
std::string name;
std::string version;
@@ -453,6 +456,9 @@ struct EditorState {
primitives.setRoot(activeAST());
primitives.setLanguage(active()->language);
recordUndoSnapshot();
if (language == "elisp") {
emacsFunctionIndexDirty = true;
}
if (path.rfind("(untitled", 0) != 0) watcher.watch(path);
if (lsp && path.rfind("(untitled", 0) != 0) {
lsp->didOpen(toFileUri(path), language, content, activeBuffer->lspVersion);
@@ -545,6 +551,9 @@ struct EditorState {
symbolsPending = true;
symbolsLastChange = ImGui::GetTime();
if (lsp) lsp->clearDocumentSymbols();
if (active() && active()->language == "elisp") {
emacsFunctionIndexDirty = true;
}
}
std::filesystem::path configDir() const {
@@ -970,10 +979,28 @@ struct EditorState {
}
}
void updateEmacsFunctionIndex() {
if (!active() || active()->language != "elisp") return;
std::string error;
auto packages = queryEmacsPackageList(emacs, false, outputLog, error);
if (!error.empty()) return;
refreshEmacsFunctionIndex(emacsFunctionIndex, emacs, packages, outputLog);
addEmacsSymbolsToLibraryIndex(libraryIndex, emacsFunctionIndex);
emacsFunctionIndexDirty = false;
rebuildExternalModulesFromIndex();
outputLog += "[emacs] Indexed functions for " +
std::to_string(emacsFunctionIndex.functionsByPackage.size()) +
" packages.\n";
}
void rebuildExternalModulesFromIndex() {
Module* ast = activeAST();
if (!ast) return;
rebuildExternalModules(ast, dependencyPanel.deps, libraryIndex);
if (active() && active()->language == "elisp") {
int signatureId = 0;
appendEmacsExternalModules(ast, emacsFunctionIndex, signatureId);
}
if (active()) active()->orchestratorDirty = true;
}
@@ -1337,6 +1364,9 @@ struct EditorState {
}
active()->orchestratorDirty = true;
active()->highlightsDirty = true;
if (lang == "elisp") {
emacsFunctionIndexDirty = true;
}
}
std::string buildRunCommand(const std::string& path,

139
editor/src/EmacsFunctionDiscovery.h Normal file
View File

@@ -0,0 +1,139 @@
#pragma once
#include "EmacsIntegration.h"
#include "EmacsPackageBrowser.h"
#include "LibraryIndexer.h"
#include "ast/Module.h"
#include "ast/ExternalModule.h"
#include "ast/TypeSignature.h"
#include <string>
#include <vector>
#include <unordered_map>
#include <algorithm>
struct EmacsFunctionDoc {
std::string name;
std::string signature;
std::string doc;
};
struct EmacsFunctionIndex {
std::unordered_map<std::string, std::vector<EmacsFunctionDoc>> functionsByPackage;
};
static std::vector<std::string> parseLines(const std::string& text) {
std::vector<std::string> out;
size_t start = 0;
while (start < text.size()) {
size_t end = text.find('\n', start);
if (end == std::string::npos) end = text.size();
std::string line = trimCopy(text.substr(start, end - start));
if (!line.empty()) out.push_back(std::move(line));
start = end + 1;
}
return out;
}
static std::vector<std::string> queryEmacsFunctions(EmacsConnection& emacs,
const std::string& prefix,
std::string& logOut) {
std::string cmd = ElispCommandBuilder::aproposFunctions(prefix);
std::string resp = emacs.sendCommand(cmd);
if (resp.empty() && !emacs.getLastError().empty()) {
logOut += "[emacs] " + emacs.getLastError() + "\n";
return {};
}
return parseLines(resp);
}
static EmacsFunctionDoc queryEmacsFunctionDoc(EmacsConnection& emacs,
const std::string& name,
std::string& logOut) {
EmacsFunctionDoc info;
info.name = name;
std::string cmd = ElispCommandBuilder::describeFunction(name);
std::string resp = emacs.sendCommand(cmd);
if (resp.empty() && !emacs.getLastError().empty()) {
logOut += "[emacs] " + emacs.getLastError() + "\n";
return info;
}
auto pos = resp.find('\n');
if (pos == std::string::npos) {
info.signature = trimCopy(resp);
} else {
info.signature = trimCopy(resp.substr(0, pos));
info.doc = trimCopy(resp.substr(pos + 1));
}
return info;
}
static std::vector<std::string> buildPackageFunctionPrefixes(const std::string& pkgName) {
std::vector<std::string> prefixes;
if (pkgName.empty()) return prefixes;
prefixes.push_back(pkgName + "-");
prefixes.push_back(pkgName);
auto dash = pkgName.find('-');
if (dash != std::string::npos) {
prefixes.push_back(pkgName.substr(0, dash) + "-");
}
return prefixes;
}
static void refreshEmacsFunctionIndex(EmacsFunctionIndex& index,
EmacsConnection& emacs,
const std::vector<EmacsPackageEntry>& packages,
std::string& logOut) {
index.functionsByPackage.clear();
for (const auto& pkg : packages) {
if (pkg.status != "loaded") continue;
std::vector<std::string> functions;
for (const auto& prefix : buildPackageFunctionPrefixes(pkg.name)) {
functions = queryEmacsFunctions(emacs, "^" + prefix, logOut);
if (!functions.empty()) break;
}
if (functions.empty()) continue;
std::vector<EmacsFunctionDoc> docs;
docs.reserve(functions.size());
for (const auto& fn : functions) {
docs.push_back(queryEmacsFunctionDoc(emacs, fn, logOut));
}
index.functionsByPackage[pkg.name] = std::move(docs);
}
}
static void appendEmacsExternalModules(Module* module,
const EmacsFunctionIndex& index,
int& signatureId) {
if (!module) return;
for (const auto& kv : index.functionsByPackage) {
const std::string& pkgName = kv.first;
auto* ext = new ExternalModule("emacs_" + pkgName, pkgName, "elisp");
for (const auto& fn : kv.second) {
auto* sig = new TypeSignature("emacs_sig_" + std::to_string(signatureId++), fn.name);
ext->addChild("signatures", sig);
}
module->addChild("externalModules", ext);
}
}
static void addEmacsSymbolsToLibraryIndex(LibraryIndexData& libIndex,
const EmacsFunctionIndex& index) {
for (const auto& kv : index.functionsByPackage) {
const std::string& pkgName = kv.first;
std::vector<LSPClient::WorkspaceSymbol> symbols;
symbols.reserve(kv.second.size());
for (const auto& fn : kv.second) {
LSPClient::WorkspaceSymbol sym;
sym.name = fn.name;
sym.containerName = pkgName;
if (!fn.signature.empty()) {
sym.detail = fn.signature;
if (!fn.doc.empty()) sym.detail += "\n" + fn.doc;
} else {
sym.detail = fn.doc;
}
sym.kind = 12; // Function
symbols.push_back(std::move(sym));
}
if (!symbols.empty()) libIndex.symbolsByLibrary[pkgName] = std::move(symbols);
}
}

39
editor/src/EmacsIntegration.h
View File

@@ -65,6 +65,23 @@ public:
result += ")";
return result;
}
// Build an Elisp expression to list functions matching a prefix/regex
static std::string aproposFunctions(const std::string& prefixRegex) {
std::string pattern = prefixRegex;
return "(mapconcat #'symbol-name (apropos-internal \"" +
escapeString(pattern) + "\" 'fboundp) \"\\n\")";
}
// Build an Elisp expression to describe a function (signature + doc)
static std::string describeFunction(const std::string& name) {
std::string sym = escapeString(name);
return "(let* ((sym '" + sym + ") "
"(args (if (fboundp 'help-function-arglist) "
" (help-function-arglist sym t) nil)) "
"(doc (or (documentation sym t) \"\"))) "
"(concat (if args (prin1-to-string args) \"\") \"\\n\" doc))";
}
};
// ---------------------------------------------------------------------------
@@ -279,6 +296,27 @@ public:
if (elispCommand.find("(require '") != std::string::npos) {
return "t";
}
if (elispCommand.find("apropos-internal") != std::string::npos) {
if (elispCommand.find("use-package") != std::string::npos) {
return "use-package\nuse-package-alias";
}
if (elispCommand.find("projectile") != std::string::npos) {
return "projectile-find-file\nprojectile-switch-project";
}
if (elispCommand.find("cl-") != std::string::npos) {
return "cl-loop\ncl-mapcar";
}
return "";
}
if (elispCommand.find("help-function-arglist") != std::string::npos) {
if (elispCommand.find("use-package") != std::string::npos) {
return "(name &rest args)\nConfigure package via use-package.";
}
if (elispCommand.find("projectile") != std::string::npos) {
return "(project)\nProjectile project switching.";
}
return "()\nEmacs function.";
}
// Unknown command — simulate error
lastError_ = "void-function";
@@ -301,3 +339,4 @@ private:
std::string lastSentCommand_;
std::string lastInitPath_;
};

46
editor/src/EmacsPackageBrowser.h
View File

@@ -121,26 +121,34 @@ static std::vector<EmacsPackageEntry> mergePackageLists(const std::vector<EmacsP
return out;
}
static std::vector<EmacsPackageEntry> queryEmacsPackageList(EmacsConnection& emacs,
bool includeAvailable,
std::string& logOut,
std::string& outError) {
outError.clear();
std::string loadedText = emacs.sendCommand(buildPackageListCommand("package-alist"));
if (loadedText.empty() && !emacs.getLastError().empty()) {
outError = emacs.getLastError();
logOut += "[emacs] " + outError + "\n";
}
auto loaded = parsePackageListText(loadedText);
std::vector<EmacsPackageEntry> available;
if (includeAvailable) {
std::string availText = emacs.sendCommand(buildPackageListCommand("package-archive-contents"));
if (availText.empty() && !emacs.getLastError().empty()) {
outError = emacs.getLastError();
logOut += "[emacs] " + outError + "\n";
}
available = parsePackageListText(availText);
}
return mergePackageLists(loaded, available, includeAvailable);
}
static void refreshEmacsPackages(EmacsPackageBrowserState& state,
EmacsConnection& emacs,
std::string& logOut) {
state.lastError.clear();
std::string loadedText = emacs.sendCommand(buildPackageListCommand("package-alist"));
if (loadedText.empty() && !emacs.getLastError().empty()) {
state.lastError = emacs.getLastError();
logOut += "[emacs] " + state.lastError + "\n";
}
auto loaded = parsePackageListText(loadedText);
std::vector<EmacsPackageEntry> available;
if (state.showAvailable) {
std::string availText = emacs.sendCommand(buildPackageListCommand("package-archive-contents"));
if (availText.empty() && !emacs.getLastError().empty()) {
state.lastError = emacs.getLastError();
logOut += "[emacs] " + state.lastError + "\n";
}
available = parsePackageListText(availText);
}
state.packages = mergePackageLists(loaded, available, state.showAvailable);
state.packages = queryEmacsPackageList(emacs, state.showAvailable, logOut, state.lastError);
state.needsRefresh = false;
}
@@ -165,9 +173,10 @@ static bool emacsPackageMatchesFilter(const EmacsPackageEntry& entry, const std:
return hay.find(needle) != std::string::npos;
}
static void renderEmacsPackageBrowser(EmacsPackageBrowserState& state,
static bool renderEmacsPackageBrowser(EmacsPackageBrowserState& state,
EmacsConnection& emacs,
std::string& logOut) {
bool updated = false;
if (ImGui::Button("Refresh")) {
state.needsRefresh = true;
}
@@ -180,6 +189,7 @@ static void renderEmacsPackageBrowser(EmacsPackageBrowserState& state,
if (state.needsRefresh) {
refreshEmacsPackages(state, emacs, logOut);
updated = true;
}
ImGui::Separator();
@@ -238,10 +248,12 @@ static void renderEmacsPackageBrowser(EmacsPackageBrowserState& state,
logOut += "[emacs] Package not installed: " + loadTarget + "\n";
}
state.needsRefresh = true;
updated = true;
}
}
if (!loadTarget.empty()) {
ImGui::SameLine();
ImGui::TextDisabled("%s", loadTarget.c_str());
}
return updated;
}

2
editor/src/PrimitivesRegistry.h
View File

@@ -122,6 +122,8 @@ private:
add("nil", "constant");
} else if (lang == "elisp") {
add("message", "function");
add("require", "function");
add("use-package", "function");
add("t", "constant");
add("nil", "constant");
}

7
editor/src/main.cpp
View File

@@ -193,6 +193,9 @@ int main(int, char**) {
}
state.pollLspMessages();
state.processLibraryIndexResponses();
if (state.emacsFunctionIndexDirty) {
state.updateEmacsFunctionIndex();
}
// Start frame
ImGui_ImplOpenGL3_NewFrame();
@@ -614,7 +617,9 @@ int main(int, char**) {
if (state.showEmacsPackagesPanel) {
ImGui::Begin("Emacs Packages", &state.showEmacsPackagesPanel);
ImGui::PushFont(uiFont);
renderEmacsPackageBrowser(state.emacsPackages, state.emacs, state.outputLog);
if (renderEmacsPackageBrowser(state.emacsPackages, state.emacs, state.outputLog)) {
state.emacsFunctionIndexDirty = true;
}
ImGui::PopFont();
ImGui::End();
}

50
editor/tests/step143_test.cpp Normal file
View File

@@ -0,0 +1,50 @@
// Step 143 TDD Test: Elisp function discovery and indexing
#include "EmacsFunctionDiscovery.h"
#include <iostream>
static void expect(bool cond, const std::string& name, int& passed, int& failed) {
if (cond) {
std::cout << "Test " << (passed + failed + 1) << " PASS: " << name << "\n";
++passed;
} else {
std::cout << "Test " << (passed + failed + 1) << " FAIL: " << name << "\n";
++failed;
}
}
int main() {
int passed = 0;
int failed = 0;
MockEmacsConnection mock;
std::string log;
std::string error;
auto packages = queryEmacsPackageList(mock, false, log, error);
expect(!packages.empty(), "loaded packages fetched", passed, failed);
auto funcs = queryEmacsFunctions(mock, "^use-package", log);
expect(!funcs.empty(), "apropos returns functions", passed, failed);
auto doc = queryEmacsFunctionDoc(mock, "use-package", log);
expect(!doc.signature.empty(), "describe-function signature", passed, failed);
expect(!doc.doc.empty(), "describe-function docstring", passed, failed);
EmacsFunctionIndex index;
refreshEmacsFunctionIndex(index, mock, packages, log);
expect(index.functionsByPackage.count("use-package") > 0, "index contains use-package", passed, failed);
Module module("mod1", "test", "elisp");
int sigId = 0;
appendEmacsExternalModules(&module, index, sigId);
auto extMods = module.getChildren("externalModules");
expect(!extMods.empty(), "external modules appended", passed, failed);
LibraryIndexData libIndex;
addEmacsSymbolsToLibraryIndex(libIndex, index);
expect(libIndex.symbolsByLibrary.count("use-package") > 0, "library index updated", passed, failed);
std::cout << "\n=== Step 143 Results: " << passed << " passed, "
<< failed << " failed ===\n";
return failed == 0 ? 0 : 1;
}

2
sprint5_plan.md
View File

@@ -186,7 +186,7 @@ functionality through the Whetstone UI.
to the daemon. Status: loaded/available/not-installed.
*New:* `EmacsPackageBrowser` panel
- [ ] **Step 143: Elisp function discovery**
- [x] **Step 143: Elisp function discovery**
Query the running Emacs daemon for available functions from loaded packages.
Send `(apropos-internal "prefix")` → get function list.
Send `(describe-function 'name)` → get docstring and signature.