bill/whetstone_DSL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Step 130: LSP library API indexing

Browse Source
This commit is contained in:
Bill
2026-02-09 16:46:58 -07:00
parent f88a129a7b
commit 42ee684087
9 changed files with 375 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
editor/CMakeLists.txt
View File

@@ -732,6 +732,10 @@ add_executable(step129_test tests/step129_test.cpp)
target_include_directories(step129_test PRIVATE src)
target_link_libraries(step129_test PRIVATE nlohmann_json::nlohmann_json)
add_executable(step130_test tests/step130_test.cpp)
target_include_directories(step130_test PRIVATE src)
target_link_libraries(step130_test PRIVATE nlohmann_json::nlohmann_json)
find_package(SDL2 CONFIG REQUIRED)
find_package(OpenGL REQUIRED)
find_package(glad CONFIG REQUIRED)

6
editor/src/DependencyPanel.h
View File

@@ -29,6 +29,7 @@ struct DependencyPanelState {
PackageEcosystem addEcosystem = PackageEcosystem::Python;
PackageInfo lastLookup;
std::string lastWorkspaceRoot;
bool needsIndex = false;
};
static std::vector<std::string> dependencyFileNames() {
@@ -311,6 +312,7 @@ static void refreshDependencies(DependencyPanelState& state,
state.selected = -1;
state.editVersionIndex = -1;
state.lastWorkspaceRoot = workspaceRoot;
state.needsIndex = true;
if (workspaceRoot.empty()) return;
for (const auto& file : state.sources) {
auto parsed = DependencyParser::parseFile(file.path);
@@ -408,6 +410,7 @@ static void renderDependencyPanel(DependencyPanelState& state,
state.deps.push_back(dep);
if (writeDependenciesForSource(dep.source, workspaceRoot, state.deps, logOut)) {
refreshDependencies(state, workspaceRoot, logOut);
state.needsIndex = true;
}
}
ImGui::CloseCurrentPopup();
@@ -449,6 +452,7 @@ static void renderDependencyPanel(DependencyPanelState& state,
dep.version = state.editVersionBuf;
if (writeDependenciesForSource(dep.source, workspaceRoot, state.deps, logOut)) {
refreshDependencies(state, workspaceRoot, logOut);
state.needsIndex = true;
}
}
ImGui::SameLine();
@@ -458,6 +462,7 @@ static void renderDependencyPanel(DependencyPanelState& state,
state.selected = -1;
if (writeDependenciesForSource(src, workspaceRoot, state.deps, logOut)) {
refreshDependencies(state, workspaceRoot, logOut);
state.needsIndex = true;
}
}
ImGui::SameLine();
@@ -468,6 +473,7 @@ static void renderDependencyPanel(DependencyPanelState& state,
dep.version = info.versions.back();
if (writeDependenciesForSource(dep.source, workspaceRoot, state.deps, logOut)) {
refreshDependencies(state, workspaceRoot, logOut);
state.needsIndex = true;
}
} else {
logOut += "[deps] No version info for " + dep.name + "\n";

93
editor/src/EditorState.h
View File

@@ -44,6 +44,7 @@
#include "WebSocketServer.h"
#include "BuildSystem.h"
#include "DependencyPanel.h"
#include "LibraryIndexer.h"
#include "IncrementalOptimizer.h"
#include "ast/Serialization.h"
#include "ast/Generator.h"
@@ -166,6 +167,15 @@ struct EditorState {
std::string lastBuildCommand;
bool showDependencyPanel = true;
DependencyPanelState dependencyPanel;
struct LibraryIndexRequest {
std::string name;
std::string version;
std::string source;
int symbolsRequestId = -1;
int completionRequestId = -1;
};
std::vector<LibraryIndexRequest> libraryIndexRequests;
LibraryIndexData libraryIndex;
// Custom editor widget state
bool showWhitespace = false;
@@ -775,6 +785,89 @@ struct EditorState {
return code;
}
void pollLspMessages() {
if (!lsp || !lspTransport || !lspTransport->isOpen()) return;
std::string msg;
while (lspTransport->receive(msg)) {
lsp->handleMessage(msg);
}
}
void requestLibraryIndex() {
libraryIndex.symbolsByLibrary.clear();
libraryIndex.completionsByLibrary.clear();
libraryIndexRequests.clear();
if (!isStructured() || !activeAST()) {
outputLog += "[deps] Library index skipped: no structured AST.\n";
return;
}
if (!lsp || !lspTransport || !lspTransport->isOpen()) {
rebuildExternalModulesFromIndex();
outputLog += "[deps] Library index uses cached symbols (LSP offline).\n";
return;
}
for (const auto& dep : dependencyPanel.deps) {
if (dep.name.empty()) continue;
LibraryIndexRequest req;
req.name = dep.name;
req.version = dep.version;
req.source = dep.source;
req.symbolsRequestId = lsp->requestWorkspaceSymbols(dep.name);
if (active() && active()->path.rfind("(untitled", 0) != 0) {
int line = std::max(0, active()->cursorLine - 1);
int col = std::max(0, active()->cursorCol - 1);
req.completionRequestId = lsp->requestLibraryCompletion(
toFileUri(active()->path), line, col, dep.name + ".");
}
libraryIndexRequests.push_back(req);
}
rebuildExternalModulesFromIndex();
outputLog += "[deps] Library index requests queued: " +
std::to_string(libraryIndexRequests.size()) + "\n";
}
void processLibraryIndexResponses() {
if (!lsp) return;
bool updated = false;
for (auto& req : libraryIndexRequests) {
if (req.symbolsRequestId >= 0) {
std::vector<LSPClient::WorkspaceSymbol> symbols;
if (lsp->takeWorkspaceSymbols(req.symbolsRequestId, symbols)) {
libraryIndex.symbolsByLibrary[req.name] = std::move(symbols);
req.symbolsRequestId = -1;
updated = true;
}
}
if (req.completionRequestId >= 0) {
std::vector<LSPClient::CompletionItem> items;
if (lsp->takeLibraryCompletions(req.completionRequestId, items)) {
std::vector<std::string> labels;
labels.reserve(items.size());
for (const auto& item : items) {
if (!item.label.empty()) labels.push_back(item.label);
}
if (!labels.empty()) {
libraryIndex.completionsByLibrary[req.name] = std::move(labels);
}
req.completionRequestId = -1;
updated = true;
}
}
}
if (updated) {
rebuildExternalModulesFromIndex();
outputLog += "[deps] Library index updated.\n";
}
}
void rebuildExternalModulesFromIndex() {
Module* ast = activeAST();
if (!ast) return;
rebuildExternalModules(ast, dependencyPanel.deps, libraryIndex);
if (active()) active()->orchestratorDirty = true;
}
json buildDiagnosticsJson() const {
json out;
json lspArr = json::array();

96
editor/src/LSPClient.h
View File

@@ -7,6 +7,7 @@
#include <memory>
#include <vector>
#include <unordered_map>
#include <unordered_set>
#include <nlohmann/json.hpp>
class LSPTransport {
@@ -69,6 +70,13 @@ public:
std::vector<DocumentSymbol> children;
};
struct WorkspaceSymbol {
std::string name;
std::string containerName;
std::string detail;
int kind = 0;
};
void initialize(const std::string& rootUri,
const std::string& clientName,
int processId) {
@@ -166,6 +174,28 @@ public:
return id;
}
int requestWorkspaceSymbols(const std::string& query) {
nlohmann::json params;
params["query"] = query;
int id = sendRequest("workspace/symbol", params);
workspaceSymbolRequestIds_.insert(id);
return id;
}
int requestLibraryCompletion(const std::string& uri,
int line,
int character,
const std::string& prefixHint) {
nlohmann::json params;
params["textDocument"] = {{"uri", uri}};
params["position"] = {{"line", line}, {"character", character}};
params["context"] = {{"triggerKind", 2}, {"triggerCharacter", "."}};
params["whetstonePrefix"] = prefixHint; // STUB: non-standard hint
int id = sendRequest("textDocument/completion", params);
libraryCompletionRequestIds_.insert(id);
return id;
}
void handleMessage(const std::string& msg) {
nlohmann::json j;
try {
@@ -281,6 +311,43 @@ public:
documentSymbols_ = std::move(parsed);
return;
}
if (workspaceSymbolRequestIds_.count(id)) {
std::vector<WorkspaceSymbol> parsed;
if (result.is_array()) {
for (const auto& item : result) {
WorkspaceSymbol sym;
if (parseWorkspaceSymbol(item, sym)) parsed.push_back(std::move(sym));
}
}
workspaceSymbolsByRequestId_[id] = std::move(parsed);
workspaceSymbolRequestIds_.erase(id);
return;
}
if (libraryCompletionRequestIds_.count(id)) {
const nlohmann::json* items = nullptr;
if (result.is_array()) {
items = &result;
} else if (result.is_object() && result.contains("items")) {
items = &result["items"];
}
if (items && items->is_array()) {
std::vector<CompletionItem> parsed;
for (const auto& item : *items) {
CompletionItem ci;
ci.label = item.value("label", "");
ci.kind = item.value("kind", 0);
ci.detail = item.value("detail", "");
ci.insertText = item.value("insertText", ci.label);
ci.filterText = item.value("filterText", "");
parsed.push_back(std::move(ci));
}
libraryCompletionsByRequestId_[id] = std::move(parsed);
}
libraryCompletionRequestIds_.erase(id);
return;
}
}
}
@@ -338,6 +405,22 @@ public:
documentSymbols_.clear();
}
bool takeWorkspaceSymbols(int requestId, std::vector<WorkspaceSymbol>& out) {
auto it = workspaceSymbolsByRequestId_.find(requestId);
if (it == workspaceSymbolsByRequestId_.end()) return false;
out = std::move(it->second);
workspaceSymbolsByRequestId_.erase(it);
return true;
}
bool takeLibraryCompletions(int requestId, std::vector<CompletionItem>& out) {
auto it = libraryCompletionsByRequestId_.find(requestId);
if (it == libraryCompletionsByRequestId_.end()) return false;
out = std::move(it->second);
libraryCompletionsByRequestId_.erase(it);
return true;
}
void sendNotification(const std::string& method, const nlohmann::json& params) {
nlohmann::json msg;
msg["jsonrpc"] = "2.0";
@@ -377,6 +460,10 @@ private:
SignatureHelp signatureHelp_;
std::vector<DefinitionLocation> definitionLocations_;
std::vector<DocumentSymbol> documentSymbols_;
std::unordered_set<int> workspaceSymbolRequestIds_;
std::unordered_map<int, std::vector<WorkspaceSymbol>> workspaceSymbolsByRequestId_;
std::unordered_set<int> libraryCompletionRequestIds_;
std::unordered_map<int, std::vector<CompletionItem>> libraryCompletionsByRequestId_;
static Range parseRange(const nlohmann::json& range) {
Range out;
@@ -444,6 +531,15 @@ private:
return !out.name.empty();
}
static bool parseWorkspaceSymbol(const nlohmann::json& item, WorkspaceSymbol& out) {
if (!item.contains("name")) return false;
out.name = item.value("name", "");
out.detail = item.value("detail", "");
out.kind = item.value("kind", 0);
out.containerName = item.value("containerName", "");
return !out.name.empty();
}
static std::string stringifyMarkup(const nlohmann::json& contents) {
if (contents.is_string()) return contents.get<std::string>();
if (contents.is_object()) {

82
editor/src/LibraryIndexer.h Normal file
View File

@@ -0,0 +1,82 @@
#pragma once
#include "DependencyParser.h"
#include "LSPClient.h"
#include "ast/ExternalModule.h"
#include "ast/TypeSignature.h"
#include "ast/Module.h"
#include <unordered_map>
#include <unordered_set>
#include <string>
struct LibraryIndexData {
std::unordered_map<std::string, std::vector<LSPClient::WorkspaceSymbol>> symbolsByLibrary;
std::unordered_map<std::string, std::vector<std::string>> completionsByLibrary;
};
static inline std::string languageForDependencySource(const std::string& source) {
std::string base = source;
auto pos = base.find(':');
if (pos != std::string::npos) base = base.substr(0, pos);
if (base == "requirements.txt" || base == "setup.py" || base == "pyproject.toml") return "python";
if (base == "package.json") return "javascript";
if (base == "Cargo.toml") return "rust";
if (base == "go.mod") return "go";
if (base == "pom.xml" || base == "build.gradle") return "java";
return "cpp";
}
static inline void clearExternalModules(Module* module) {
if (!module) return;
std::vector<ASTNode*> existing = module->getChildren("externalModules");
for (auto* child : existing) {
module->removeChild(child);
delete child;
}
}
static inline bool symbolMatchesLibrary(const LSPClient::WorkspaceSymbol& sym,
const std::string& library) {
if (sym.containerName == library) return true;
std::string prefixDot = library + ".";
std::string prefixScope = library + "::";
return sym.name.rfind(prefixDot, 0) == 0 || sym.name.rfind(prefixScope, 0) == 0;
}
static inline void rebuildExternalModules(Module* module,
const std::vector<DependencySpec>& deps,
const LibraryIndexData& index) {
if (!module) return;
clearExternalModules(module);
std::unordered_map<std::string, int> nameCounts;
int signatureId = 0;
for (const auto& dep : deps) {
if (dep.name.empty()) continue;
int count = nameCounts[dep.name]++;
std::string id = "ext_" + dep.name + (count > 0 ? "_" + std::to_string(count) : "");
auto* ext = new ExternalModule(id, dep.name,
languageForDependencySource(dep.source),
dep.version);
std::unordered_set<std::string> added;
auto it = index.symbolsByLibrary.find(dep.name);
if (it != index.symbolsByLibrary.end()) {
for (const auto& sym : it->second) {
if (!symbolMatchesLibrary(sym, dep.name)) continue;
if (!added.insert(sym.name).second) continue;
auto* sig = new TypeSignature("sig_" + std::to_string(signatureId++), sym.name);
ext->addChild("signatures", sig);
}
}
auto it2 = index.completionsByLibrary.find(dep.name);
if (it2 != index.completionsByLibrary.end()) {
for (const auto& name : it2->second) {
if (!added.insert(name).second) continue;
auto* sig = new TypeSignature("sig_" + std::to_string(signatureId++), name);
ext->addChild("signatures", sig);
}
}
module->addChild("externalModules", ext);
}
}

6
editor/src/main.cpp
View File

@@ -190,6 +190,8 @@ int main(int, char**) {
state.lastAutoSave = now;
}
}
state.pollLspMessages();
state.processLibraryIndexResponses();
// Start frame
ImGui_ImplOpenGL3_NewFrame();
@@ -560,6 +562,10 @@ int main(int, char**) {
renderDependencyPanel(state.dependencyPanel, state.workspaceRoot, state.outputLog);
ImGui::PopFont();
ImGui::End();
if (state.dependencyPanel.needsIndex) {
state.requestLibraryIndex();
state.dependencyPanel.needsIndex = false;
}
}
// ---------------------------------------------------------------

85
editor/tests/step130_test.cpp Normal file
View File

@@ -0,0 +1,85 @@
// Step 130 TDD Test: Library API indexing via LSP
#include "LSPClient.h"
#include "LibraryIndexer.h"
#include <iostream>
#include <sstream>
#include <filesystem>
struct FakeTransport : public LSPTransport {
std::vector<std::string> sent;
void send(const std::string& msg) override { sent.push_back(msg); }
bool receive(std::string& out) override { (void)out; return false; }
bool isOpen() const override { return true; }
void close() override {}
};
static void expect(bool cond, const std::string& name, int& passed, int& failed) {
if (cond) {
std::cout << "Test " << (passed + failed + 1) << " PASS: " << name << "\n";
++passed;
} else {
std::cout << "Test " << (passed + failed + 1) << " FAIL: " << name << "\n";
++failed;
}
}
int main() {
int passed = 0;
int failed = 0;
auto transport = std::make_shared<FakeTransport>();
LSPClient client(transport);
int symReq = client.requestWorkspaceSymbols("numpy");
{
nlohmann::json resp;
resp["jsonrpc"] = "2.0";
resp["id"] = symReq;
resp["result"] = nlohmann::json::array({
{{"name", "numpy.array"}, {"kind", 12}, {"containerName", "numpy"}}
});
client.handleMessage(resp.dump());
}
std::vector<LSPClient::WorkspaceSymbol> symbols;
expect(client.takeWorkspaceSymbols(symReq, symbols), "workspace symbols retrieved", passed, failed);
expect(symbols.size() == 1, "workspace symbols count", passed, failed);
if (!symbols.empty()) {
expect(symbols[0].name == "numpy.array", "workspace symbol name", passed, failed);
}
int compReq = client.requestLibraryCompletion("file:///tmp.py", 0, 0, "numpy.");
{
nlohmann::json resp;
resp["jsonrpc"] = "2.0";
resp["id"] = compReq;
resp["result"] = nlohmann::json::array({
{{"label", "numpy.zeros"}, {"kind", 3}, {"detail", "func"}}
});
client.handleMessage(resp.dump());
}
std::vector<LSPClient::CompletionItem> completions;
expect(client.takeLibraryCompletions(compReq, completions), "library completion retrieved", passed, failed);
expect(!completions.empty() && completions[0].label == "numpy.zeros",
"library completion label", passed, failed);
// Index rebuild into ExternalModule nodes
Module mod("m1", "Module", "python");
std::vector<DependencySpec> deps;
deps.push_back({"numpy", "1.26.0", "requirements.txt"});
deps.push_back({"serde", "1.0", "Cargo.toml"});
LibraryIndexData index;
index.symbolsByLibrary["numpy"] = symbols;
index.completionsByLibrary["numpy"] = {"numpy.zeros"};
rebuildExternalModules(&mod, deps, index);
auto mods = mod.getChildren("externalModules");
expect(mods.size() == 2, "external modules created", passed, failed);
if (!mods.empty()) {
auto* ext = static_cast<ExternalModule*>(mods[0]);
auto sigs = ext->getChildren("signatures");
expect(!sigs.empty(), "external module signatures", passed, failed);
}
std::cout << "\n=== Step 130 Results: " << passed << " passed, " << failed << " failed ===\n";
return failed == 0 ? 0 : 1;
}