From 42ee684087fd3a2c90bcd367b69ab3e73c79e94d Mon Sep 17 00:00:00 2001 From: Bill Date: Mon, 9 Feb 2026 16:46:58 -0700 Subject: [PATCH] Step 130: LSP library API indexing --- PROGRESS.md | 3 +- editor/CMakeLists.txt | 4 ++ editor/src/DependencyPanel.h | 6 +++ editor/src/EditorState.h | 93 +++++++++++++++++++++++++++++++++ editor/src/LSPClient.h | 96 +++++++++++++++++++++++++++++++++++ editor/src/LibraryIndexer.h | 82 ++++++++++++++++++++++++++++++ editor/src/main.cpp | 6 +++ editor/tests/step130_test.cpp | 85 +++++++++++++++++++++++++++++++ sprint5_plan.md | 2 +- 9 files changed, 375 insertions(+), 2 deletions(-) create mode 100644 editor/src/LibraryIndexer.h create mode 100644 editor/tests/step130_test.cpp diff --git a/PROGRESS.md b/PROGRESS.md index 6b3a635..652964e 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -382,7 +382,7 @@ vcpkg's imgui 1.91.9 removed the `sdl2-binding` feature (only `sdl3-binding` exi ## What's Next -Sprint 5 in progress. Step 129 (dependency management UI) done. Next: Step 130 (library API indexing via LSP). +Sprint 5 in progress. Step 130 (library API indexing via LSP) done. Next: Step 131 (library API indexing via type stubs). --- @@ -475,3 +475,4 @@ Sprint 5 in progress. Step 129 (dependency management UI) done. Next: Step 130 ( | 2026-02-09 | Codex | Step 127: Package registry abstraction stubs for major ecosystems. 4/4 tests pass. | | 2026-02-09 | Codex | Step 128: Dependency file parsing with Import population. 6/6 tests pass. | | 2026-02-09 | Codex | Step 129: Dependency management UI panel with add/remove/update, source targeting, and writeback support for requirements.txt/package.json/Cargo.toml/go.mod/vcpkg.json. 8/8 tests pass. | +| 2026-02-09 | Codex | Step 130: LSP workspace symbol + completion indexing for dependencies with ExternalModule population and library index polling. 7/7 tests pass. | diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 44c9bc0..ba75471 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -732,6 +732,10 @@ add_executable(step129_test tests/step129_test.cpp) target_include_directories(step129_test PRIVATE src) target_link_libraries(step129_test PRIVATE nlohmann_json::nlohmann_json) +add_executable(step130_test tests/step130_test.cpp) +target_include_directories(step130_test PRIVATE src) +target_link_libraries(step130_test PRIVATE nlohmann_json::nlohmann_json) + find_package(SDL2 CONFIG REQUIRED) find_package(OpenGL REQUIRED) find_package(glad CONFIG REQUIRED) diff --git a/editor/src/DependencyPanel.h b/editor/src/DependencyPanel.h index 8a33e40..900bc02 100644 --- a/editor/src/DependencyPanel.h +++ b/editor/src/DependencyPanel.h @@ -29,6 +29,7 @@ struct DependencyPanelState { PackageEcosystem addEcosystem = PackageEcosystem::Python; PackageInfo lastLookup; std::string lastWorkspaceRoot; + bool needsIndex = false; }; static std::vector dependencyFileNames() { @@ -311,6 +312,7 @@ static void refreshDependencies(DependencyPanelState& state, state.selected = -1; state.editVersionIndex = -1; state.lastWorkspaceRoot = workspaceRoot; + state.needsIndex = true; if (workspaceRoot.empty()) return; for (const auto& file : state.sources) { auto parsed = DependencyParser::parseFile(file.path); @@ -408,6 +410,7 @@ static void renderDependencyPanel(DependencyPanelState& state, state.deps.push_back(dep); if (writeDependenciesForSource(dep.source, workspaceRoot, state.deps, logOut)) { refreshDependencies(state, workspaceRoot, logOut); + state.needsIndex = true; } } ImGui::CloseCurrentPopup(); @@ -449,6 +452,7 @@ static void renderDependencyPanel(DependencyPanelState& state, dep.version = state.editVersionBuf; if (writeDependenciesForSource(dep.source, workspaceRoot, state.deps, logOut)) { refreshDependencies(state, workspaceRoot, logOut); + state.needsIndex = true; } } ImGui::SameLine(); @@ -458,6 +462,7 @@ static void renderDependencyPanel(DependencyPanelState& state, state.selected = -1; if (writeDependenciesForSource(src, workspaceRoot, state.deps, logOut)) { refreshDependencies(state, workspaceRoot, logOut); + state.needsIndex = true; } } ImGui::SameLine(); @@ -468,6 +473,7 @@ static void renderDependencyPanel(DependencyPanelState& state, dep.version = info.versions.back(); if (writeDependenciesForSource(dep.source, workspaceRoot, state.deps, logOut)) { refreshDependencies(state, workspaceRoot, logOut); + state.needsIndex = true; } } else { logOut += "[deps] No version info for " + dep.name + "\n"; diff --git a/editor/src/EditorState.h b/editor/src/EditorState.h index cb9fd9a..488436a 100644 --- a/editor/src/EditorState.h +++ b/editor/src/EditorState.h @@ -44,6 +44,7 @@ #include "WebSocketServer.h" #include "BuildSystem.h" #include "DependencyPanel.h" +#include "LibraryIndexer.h" #include "IncrementalOptimizer.h" #include "ast/Serialization.h" #include "ast/Generator.h" @@ -166,6 +167,15 @@ struct EditorState { std::string lastBuildCommand; bool showDependencyPanel = true; DependencyPanelState dependencyPanel; + struct LibraryIndexRequest { + std::string name; + std::string version; + std::string source; + int symbolsRequestId = -1; + int completionRequestId = -1; + }; + std::vector libraryIndexRequests; + LibraryIndexData libraryIndex; // Custom editor widget state bool showWhitespace = false; @@ -775,6 +785,89 @@ struct EditorState { return code; } + void pollLspMessages() { + if (!lsp || !lspTransport || !lspTransport->isOpen()) return; + std::string msg; + while (lspTransport->receive(msg)) { + lsp->handleMessage(msg); + } + } + + void requestLibraryIndex() { + libraryIndex.symbolsByLibrary.clear(); + libraryIndex.completionsByLibrary.clear(); + libraryIndexRequests.clear(); + if (!isStructured() || !activeAST()) { + outputLog += "[deps] Library index skipped: no structured AST.\n"; + return; + } + if (!lsp || !lspTransport || !lspTransport->isOpen()) { + rebuildExternalModulesFromIndex(); + outputLog += "[deps] Library index uses cached symbols (LSP offline).\n"; + return; + } + + for (const auto& dep : dependencyPanel.deps) { + if (dep.name.empty()) continue; + LibraryIndexRequest req; + req.name = dep.name; + req.version = dep.version; + req.source = dep.source; + req.symbolsRequestId = lsp->requestWorkspaceSymbols(dep.name); + if (active() && active()->path.rfind("(untitled", 0) != 0) { + int line = std::max(0, active()->cursorLine - 1); + int col = std::max(0, active()->cursorCol - 1); + req.completionRequestId = lsp->requestLibraryCompletion( + toFileUri(active()->path), line, col, dep.name + "."); + } + libraryIndexRequests.push_back(req); + } + rebuildExternalModulesFromIndex(); + outputLog += "[deps] Library index requests queued: " + + std::to_string(libraryIndexRequests.size()) + "\n"; + } + + void processLibraryIndexResponses() { + if (!lsp) return; + bool updated = false; + for (auto& req : libraryIndexRequests) { + if (req.symbolsRequestId >= 0) { + std::vector symbols; + if (lsp->takeWorkspaceSymbols(req.symbolsRequestId, symbols)) { + libraryIndex.symbolsByLibrary[req.name] = std::move(symbols); + req.symbolsRequestId = -1; + updated = true; + } + } + if (req.completionRequestId >= 0) { + std::vector items; + if (lsp->takeLibraryCompletions(req.completionRequestId, items)) { + std::vector labels; + labels.reserve(items.size()); + for (const auto& item : items) { + if (!item.label.empty()) labels.push_back(item.label); + } + if (!labels.empty()) { + libraryIndex.completionsByLibrary[req.name] = std::move(labels); + } + req.completionRequestId = -1; + updated = true; + } + } + } + if (updated) { + rebuildExternalModulesFromIndex(); + outputLog += "[deps] Library index updated.\n"; + } + } + + void rebuildExternalModulesFromIndex() { + Module* ast = activeAST(); + if (!ast) return; + rebuildExternalModules(ast, dependencyPanel.deps, libraryIndex); + if (active()) active()->orchestratorDirty = true; + } + json buildDiagnosticsJson() const { json out; json lspArr = json::array(); diff --git a/editor/src/LSPClient.h b/editor/src/LSPClient.h index 8606c5f..8d0a938 100644 --- a/editor/src/LSPClient.h +++ b/editor/src/LSPClient.h @@ -7,6 +7,7 @@ #include #include #include +#include #include class LSPTransport { @@ -69,6 +70,13 @@ public: std::vector children; }; + struct WorkspaceSymbol { + std::string name; + std::string containerName; + std::string detail; + int kind = 0; + }; + void initialize(const std::string& rootUri, const std::string& clientName, int processId) { @@ -166,6 +174,28 @@ public: return id; } + int requestWorkspaceSymbols(const std::string& query) { + nlohmann::json params; + params["query"] = query; + int id = sendRequest("workspace/symbol", params); + workspaceSymbolRequestIds_.insert(id); + return id; + } + + int requestLibraryCompletion(const std::string& uri, + int line, + int character, + const std::string& prefixHint) { + nlohmann::json params; + params["textDocument"] = {{"uri", uri}}; + params["position"] = {{"line", line}, {"character", character}}; + params["context"] = {{"triggerKind", 2}, {"triggerCharacter", "."}}; + params["whetstonePrefix"] = prefixHint; // STUB: non-standard hint + int id = sendRequest("textDocument/completion", params); + libraryCompletionRequestIds_.insert(id); + return id; + } + void handleMessage(const std::string& msg) { nlohmann::json j; try { @@ -281,6 +311,43 @@ public: documentSymbols_ = std::move(parsed); return; } + + if (workspaceSymbolRequestIds_.count(id)) { + std::vector parsed; + if (result.is_array()) { + for (const auto& item : result) { + WorkspaceSymbol sym; + if (parseWorkspaceSymbol(item, sym)) parsed.push_back(std::move(sym)); + } + } + workspaceSymbolsByRequestId_[id] = std::move(parsed); + workspaceSymbolRequestIds_.erase(id); + return; + } + + if (libraryCompletionRequestIds_.count(id)) { + const nlohmann::json* items = nullptr; + if (result.is_array()) { + items = &result; + } else if (result.is_object() && result.contains("items")) { + items = &result["items"]; + } + if (items && items->is_array()) { + std::vector parsed; + for (const auto& item : *items) { + CompletionItem ci; + ci.label = item.value("label", ""); + ci.kind = item.value("kind", 0); + ci.detail = item.value("detail", ""); + ci.insertText = item.value("insertText", ci.label); + ci.filterText = item.value("filterText", ""); + parsed.push_back(std::move(ci)); + } + libraryCompletionsByRequestId_[id] = std::move(parsed); + } + libraryCompletionRequestIds_.erase(id); + return; + } } } @@ -338,6 +405,22 @@ public: documentSymbols_.clear(); } + bool takeWorkspaceSymbols(int requestId, std::vector& out) { + auto it = workspaceSymbolsByRequestId_.find(requestId); + if (it == workspaceSymbolsByRequestId_.end()) return false; + out = std::move(it->second); + workspaceSymbolsByRequestId_.erase(it); + return true; + } + + bool takeLibraryCompletions(int requestId, std::vector& out) { + auto it = libraryCompletionsByRequestId_.find(requestId); + if (it == libraryCompletionsByRequestId_.end()) return false; + out = std::move(it->second); + libraryCompletionsByRequestId_.erase(it); + return true; + } + void sendNotification(const std::string& method, const nlohmann::json& params) { nlohmann::json msg; msg["jsonrpc"] = "2.0"; @@ -377,6 +460,10 @@ private: SignatureHelp signatureHelp_; std::vector definitionLocations_; std::vector documentSymbols_; + std::unordered_set workspaceSymbolRequestIds_; + std::unordered_map> workspaceSymbolsByRequestId_; + std::unordered_set libraryCompletionRequestIds_; + std::unordered_map> libraryCompletionsByRequestId_; static Range parseRange(const nlohmann::json& range) { Range out; @@ -444,6 +531,15 @@ private: return !out.name.empty(); } + static bool parseWorkspaceSymbol(const nlohmann::json& item, WorkspaceSymbol& out) { + if (!item.contains("name")) return false; + out.name = item.value("name", ""); + out.detail = item.value("detail", ""); + out.kind = item.value("kind", 0); + out.containerName = item.value("containerName", ""); + return !out.name.empty(); + } + static std::string stringifyMarkup(const nlohmann::json& contents) { if (contents.is_string()) return contents.get(); if (contents.is_object()) { diff --git a/editor/src/LibraryIndexer.h b/editor/src/LibraryIndexer.h new file mode 100644 index 0000000..d5afa9f --- /dev/null +++ b/editor/src/LibraryIndexer.h @@ -0,0 +1,82 @@ +#pragma once +#include "DependencyParser.h" +#include "LSPClient.h" +#include "ast/ExternalModule.h" +#include "ast/TypeSignature.h" +#include "ast/Module.h" +#include +#include +#include + +struct LibraryIndexData { + std::unordered_map> symbolsByLibrary; + std::unordered_map> completionsByLibrary; +}; + +static inline std::string languageForDependencySource(const std::string& source) { + std::string base = source; + auto pos = base.find(':'); + if (pos != std::string::npos) base = base.substr(0, pos); + if (base == "requirements.txt" || base == "setup.py" || base == "pyproject.toml") return "python"; + if (base == "package.json") return "javascript"; + if (base == "Cargo.toml") return "rust"; + if (base == "go.mod") return "go"; + if (base == "pom.xml" || base == "build.gradle") return "java"; + return "cpp"; +} + +static inline void clearExternalModules(Module* module) { + if (!module) return; + std::vector existing = module->getChildren("externalModules"); + for (auto* child : existing) { + module->removeChild(child); + delete child; + } +} + +static inline bool symbolMatchesLibrary(const LSPClient::WorkspaceSymbol& sym, + const std::string& library) { + if (sym.containerName == library) return true; + std::string prefixDot = library + "."; + std::string prefixScope = library + "::"; + return sym.name.rfind(prefixDot, 0) == 0 || sym.name.rfind(prefixScope, 0) == 0; +} + +static inline void rebuildExternalModules(Module* module, + const std::vector& deps, + const LibraryIndexData& index) { + if (!module) return; + clearExternalModules(module); + std::unordered_map nameCounts; + int signatureId = 0; + + for (const auto& dep : deps) { + if (dep.name.empty()) continue; + int count = nameCounts[dep.name]++; + std::string id = "ext_" + dep.name + (count > 0 ? "_" + std::to_string(count) : ""); + auto* ext = new ExternalModule(id, dep.name, + languageForDependencySource(dep.source), + dep.version); + + std::unordered_set added; + auto it = index.symbolsByLibrary.find(dep.name); + if (it != index.symbolsByLibrary.end()) { + for (const auto& sym : it->second) { + if (!symbolMatchesLibrary(sym, dep.name)) continue; + if (!added.insert(sym.name).second) continue; + auto* sig = new TypeSignature("sig_" + std::to_string(signatureId++), sym.name); + ext->addChild("signatures", sig); + } + } + auto it2 = index.completionsByLibrary.find(dep.name); + if (it2 != index.completionsByLibrary.end()) { + for (const auto& name : it2->second) { + if (!added.insert(name).second) continue; + auto* sig = new TypeSignature("sig_" + std::to_string(signatureId++), name); + ext->addChild("signatures", sig); + } + } + + module->addChild("externalModules", ext); + } +} diff --git a/editor/src/main.cpp b/editor/src/main.cpp index 44149e5..f7e91d6 100644 --- a/editor/src/main.cpp +++ b/editor/src/main.cpp @@ -190,6 +190,8 @@ int main(int, char**) { state.lastAutoSave = now; } } + state.pollLspMessages(); + state.processLibraryIndexResponses(); // Start frame ImGui_ImplOpenGL3_NewFrame(); @@ -560,6 +562,10 @@ int main(int, char**) { renderDependencyPanel(state.dependencyPanel, state.workspaceRoot, state.outputLog); ImGui::PopFont(); ImGui::End(); + if (state.dependencyPanel.needsIndex) { + state.requestLibraryIndex(); + state.dependencyPanel.needsIndex = false; + } } // --------------------------------------------------------------- diff --git a/editor/tests/step130_test.cpp b/editor/tests/step130_test.cpp new file mode 100644 index 0000000..7830184 --- /dev/null +++ b/editor/tests/step130_test.cpp @@ -0,0 +1,85 @@ +// Step 130 TDD Test: Library API indexing via LSP +#include "LSPClient.h" +#include "LibraryIndexer.h" +#include +#include +#include + +struct FakeTransport : public LSPTransport { + std::vector sent; + void send(const std::string& msg) override { sent.push_back(msg); } + bool receive(std::string& out) override { (void)out; return false; } + bool isOpen() const override { return true; } + void close() override {} +}; + +static void expect(bool cond, const std::string& name, int& passed, int& failed) { + if (cond) { + std::cout << "Test " << (passed + failed + 1) << " PASS: " << name << "\n"; + ++passed; + } else { + std::cout << "Test " << (passed + failed + 1) << " FAIL: " << name << "\n"; + ++failed; + } +} + +int main() { + int passed = 0; + int failed = 0; + + auto transport = std::make_shared(); + LSPClient client(transport); + + int symReq = client.requestWorkspaceSymbols("numpy"); + { + nlohmann::json resp; + resp["jsonrpc"] = "2.0"; + resp["id"] = symReq; + resp["result"] = nlohmann::json::array({ + {{"name", "numpy.array"}, {"kind", 12}, {"containerName", "numpy"}} + }); + client.handleMessage(resp.dump()); + } + std::vector symbols; + expect(client.takeWorkspaceSymbols(symReq, symbols), "workspace symbols retrieved", passed, failed); + expect(symbols.size() == 1, "workspace symbols count", passed, failed); + if (!symbols.empty()) { + expect(symbols[0].name == "numpy.array", "workspace symbol name", passed, failed); + } + + int compReq = client.requestLibraryCompletion("file:///tmp.py", 0, 0, "numpy."); + { + nlohmann::json resp; + resp["jsonrpc"] = "2.0"; + resp["id"] = compReq; + resp["result"] = nlohmann::json::array({ + {{"label", "numpy.zeros"}, {"kind", 3}, {"detail", "func"}} + }); + client.handleMessage(resp.dump()); + } + std::vector completions; + expect(client.takeLibraryCompletions(compReq, completions), "library completion retrieved", passed, failed); + expect(!completions.empty() && completions[0].label == "numpy.zeros", + "library completion label", passed, failed); + + // Index rebuild into ExternalModule nodes + Module mod("m1", "Module", "python"); + std::vector deps; + deps.push_back({"numpy", "1.26.0", "requirements.txt"}); + deps.push_back({"serde", "1.0", "Cargo.toml"}); + + LibraryIndexData index; + index.symbolsByLibrary["numpy"] = symbols; + index.completionsByLibrary["numpy"] = {"numpy.zeros"}; + rebuildExternalModules(&mod, deps, index); + auto mods = mod.getChildren("externalModules"); + expect(mods.size() == 2, "external modules created", passed, failed); + if (!mods.empty()) { + auto* ext = static_cast(mods[0]); + auto sigs = ext->getChildren("signatures"); + expect(!sigs.empty(), "external module signatures", passed, failed); + } + + std::cout << "\n=== Step 130 Results: " << passed << " passed, " << failed << " failed ===\n"; + return failed == 0 ? 0 : 1; +} diff --git a/sprint5_plan.md b/sprint5_plan.md index b70da1a..f74becb 100644 --- a/sprint5_plan.md +++ b/sprint5_plan.md @@ -54,7 +54,7 @@ The foundation: import libraries, parse their APIs, index their symbols. Changes write back to the appropriate dependency file. *New:* `DependencyPanel` in `main.cpp` -- [ ] **Step 130: Library API indexing via LSP** +- [x] **Step 130: Library API indexing via LSP** When a package is imported, leverage the LSP server to extract its public API surface. Send `workspace/symbol` queries and `textDocument/completion` with library prefix to discover available functions, classes, constants.