bill/whetstone_DSL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Step 131: stub-based library indexing

Browse Source
This commit is contained in:
Bill
2026-02-09 17:00:55 -07:00
parent 42ee684087
commit ae95aeb303
7 changed files with 336 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
PROGRESS.md
View File

@@ -382,7 +382,7 @@ vcpkg's imgui 1.91.9 removed the `sdl2-binding` feature (only `sdl3-binding` exi
## What's Next
Sprint 5 in progress. Step 130 (library API indexing via LSP) done. Next: Step 131 (library API indexing via type stubs).
Sprint 5 in progress. Step 131 (library API indexing via type stubs) done. Next: Step 132 (library symbol browser).
---
@@ -476,3 +476,4 @@ Sprint 5 in progress. Step 130 (library API indexing via LSP) done. Next: Step 1
| 2026-02-09 | Codex | Step 128: Dependency file parsing with Import population. 6/6 tests pass. |
| 2026-02-09 | Codex | Step 129: Dependency management UI panel with add/remove/update, source targeting, and writeback support for requirements.txt/package.json/Cargo.toml/go.mod/vcpkg.json. 8/8 tests pass. |
| 2026-02-09 | Codex | Step 130: LSP workspace symbol + completion indexing for dependencies with ExternalModule population and library index polling. 7/7 tests pass. |
| 2026-02-09 | Codex | Step 131: Stub-based library indexing (pyi/d.ts/headers/lib.rs) with workspace scan fallback. 8/8 tests pass. |

4
editor/CMakeLists.txt
View File

@@ -736,6 +736,10 @@ add_executable(step130_test tests/step130_test.cpp)
target_include_directories(step130_test PRIVATE src)
target_link_libraries(step130_test PRIVATE nlohmann_json::nlohmann_json)
add_executable(step131_test tests/step131_test.cpp)
target_include_directories(step131_test PRIVATE src)
target_link_libraries(step131_test PRIVATE nlohmann_json::nlohmann_json)
find_package(SDL2 CONFIG REQUIRED)
find_package(OpenGL REQUIRED)
find_package(glad CONFIG REQUIRED)

3
editor/src/EditorState.h
View File

@@ -802,6 +802,7 @@ struct EditorState {
return;
}
if (!lsp || !lspTransport || !lspTransport->isOpen()) {
applyStubFallback(libraryIndex, dependencyPanel.deps, workspaceRoot);
rebuildExternalModulesFromIndex();
outputLog += "[deps] Library index uses cached symbols (LSP offline).\n";
return;
@@ -822,6 +823,7 @@ struct EditorState {
}
libraryIndexRequests.push_back(req);
}
applyStubFallback(libraryIndex, dependencyPanel.deps, workspaceRoot);
rebuildExternalModulesFromIndex();
outputLog += "[deps] Library index requests queued: " +
std::to_string(libraryIndexRequests.size()) + "\n";
@@ -856,6 +858,7 @@ struct EditorState {
}
}
if (updated) {
applyStubFallback(libraryIndex, dependencyPanel.deps, workspaceRoot);
rebuildExternalModulesFromIndex();
outputLog += "[deps] Library index updated.\n";
}

17
editor/src/LibraryIndexer.h
View File

@@ -1,6 +1,7 @@
#pragma once
#include "DependencyParser.h"
#include "LSPClient.h"
#include "StubParser.h"
#include "ast/ExternalModule.h"
#include "ast/TypeSignature.h"
#include "ast/Module.h"
@@ -80,3 +81,19 @@ static inline void rebuildExternalModules(Module* module,
module->addChild("externalModules", ext);
}
}
static inline void applyStubFallback(LibraryIndexData& index,
const std::vector<DependencySpec>& deps,
const std::string& workspaceRoot) {
for (const auto& dep : deps) {
if (dep.name.empty()) continue;
bool hasSymbols = index.symbolsByLibrary.count(dep.name) > 0;
bool hasCompletions = index.completionsByLibrary.count(dep.name) > 0;
if (hasSymbols || hasCompletions) continue;
std::string lang = languageForDependencySource(dep.source);
auto names = StubParser::scanWorkspaceForLibrary(workspaceRoot, dep.name, lang);
if (!names.empty()) {
index.completionsByLibrary[dep.name] = std::move(names);
}
}
}

231
editor/src/StubParser.h Normal file
View File

@@ -0,0 +1,231 @@
#pragma once
#include <string>
#include <vector>
#include <filesystem>
#include <fstream>
#include <sstream>
#include <unordered_set>
#include <algorithm>
#include <cctype>
class StubParser {
public:
static std::vector<std::string> parseFile(const std::string& path,
const std::string& languageHint) {
std::filesystem::path p(path);
std::string ext = p.extension().string();
if (ext == ".pyi") return parsePythonStub(path);
if (ext == ".d.ts") return parseTypeScriptStub(path);
if (ext == ".h" || ext == ".hpp" || ext == ".hh") return parseHeaderStub(path);
if (p.filename() == "lib.rs") return parseRustStub(path);
if (languageHint == "python") return parsePythonStub(path);
if (languageHint == "javascript" || languageHint == "typescript") return parseTypeScriptStub(path);
if (languageHint == "rust") return parseRustStub(path);
return parseHeaderStub(path);
}
static std::vector<std::string> scanWorkspaceForLibrary(const std::string& workspaceRoot,
const std::string& library,
const std::string& languageHint) {
std::vector<std::string> out;
if (workspaceRoot.empty() || library.empty()) return out;
std::filesystem::path root(workspaceRoot);
if (!std::filesystem::exists(root)) return out;
const int maxFiles = 8;
int scanned = 0;
std::string lowerLib = toLower(library);
for (auto it = std::filesystem::recursive_directory_iterator(root);
it != std::filesystem::recursive_directory_iterator(); ++it) {
if (scanned >= maxFiles) break;
if (!it->is_regular_file()) continue;
const auto& path = it->path();
std::string filename = path.filename().string();
std::string ext = path.extension().string();
if (!isStubCandidate(filename, ext, languageHint)) continue;
std::string pathLower = toLower(path.string());
if (pathLower.find(lowerLib) == std::string::npos) continue;
auto parsed = parseFile(path.string(), languageHint);
if (!parsed.empty()) {
out.insert(out.end(), parsed.begin(), parsed.end());
scanned++;
}
}
dedupe(out);
return out;
}
private:
static std::string trim(const std::string& s) {
size_t start = s.find_first_not_of(" \t\r\n");
size_t end = s.find_last_not_of(" \t\r\n");
if (start == std::string::npos || end == std::string::npos) return "";
return s.substr(start, end - start + 1);
}
static std::string toLower(const std::string& s) {
std::string out = s;
std::transform(out.begin(), out.end(), out.begin(),
[](unsigned char c) { return (char)std::tolower(c); });
return out;
}
static bool isIdentChar(char c) {
return std::isalnum((unsigned char)c) || c == '_' || c == ':';
}
static void dedupe(std::vector<std::string>& items) {
std::unordered_set<std::string> seen;
std::vector<std::string> out;
for (const auto& item : items) {
if (item.empty()) continue;
if (seen.insert(item).second) out.push_back(item);
}
items.swap(out);
}
static bool isStubCandidate(const std::string& filename,
const std::string& ext,
const std::string& languageHint) {
if (ext == ".pyi") return true;
if (ext == ".d.ts") return true;
if (ext == ".h" || ext == ".hpp" || ext == ".hh") return true;
if (filename == "lib.rs") return true;
if (languageHint == "python" && ext == ".pyi") return true;
if ((languageHint == "javascript" || languageHint == "typescript") && ext == ".d.ts") return true;
if (languageHint == "rust" && filename == "lib.rs") return true;
if (languageHint == "cpp" && (ext == ".h" || ext == ".hpp" || ext == ".hh")) return true;
return false;
}
static std::vector<std::string> parsePythonStub(const std::string& path) {
std::vector<std::string> out;
std::ifstream in(path);
if (!in.is_open()) return out;
std::string line;
while (std::getline(in, line)) {
line = trim(line);
if (line.empty() || line[0] == '#') continue;
if (line.rfind("def ", 0) == 0) {
std::string name = extractAfterKeyword(line, "def");
if (!name.empty()) out.push_back(name);
} else if (line.rfind("class ", 0) == 0) {
std::string name = extractAfterKeyword(line, "class");
if (!name.empty()) out.push_back(name);
} else if (line.rfind("CONST", 0) == 0 || line.find(":") != std::string::npos) {
std::string name = extractBefore(line, ':');
if (!name.empty()) out.push_back(name);
}
}
dedupe(out);
return out;
}
static std::vector<std::string> parseTypeScriptStub(const std::string& path) {
std::vector<std::string> out;
std::ifstream in(path);
if (!in.is_open()) return out;
std::string line;
while (std::getline(in, line)) {
line = trim(line);
if (line.empty() || line.rfind("//", 0) == 0) continue;
if (line.rfind("export function", 0) == 0 || line.rfind("function", 0) == 0) {
std::string name = extractAfterKeyword(line, "function");
if (!name.empty()) out.push_back(name);
} else if (line.rfind("export class", 0) == 0 || line.rfind("class", 0) == 0) {
std::string name = extractAfterKeyword(line, "class");
if (!name.empty()) out.push_back(name);
} else if (line.rfind("interface", 0) == 0 || line.rfind("export interface", 0) == 0) {
std::string name = extractAfterKeyword(line, "interface");
if (!name.empty()) out.push_back(name);
} else if (line.rfind("const ", 0) == 0 || line.rfind("export const", 0) == 0) {
std::string name = extractAfterKeyword(line, "const");
if (!name.empty()) out.push_back(name);
}
}
dedupe(out);
return out;
}
static std::vector<std::string> parseHeaderStub(const std::string& path) {
std::vector<std::string> out;
std::ifstream in(path);
if (!in.is_open()) return out;
std::string line;
while (std::getline(in, line)) {
line = stripCppComment(line);
line = trim(line);
if (line.empty()) continue;
auto pos = line.find('(');
if (pos == std::string::npos) continue;
std::string left = trim(line.substr(0, pos));
std::string name = lastIdentifier(left);
if (!name.empty() && name != "if" && name != "for" && name != "while") {
out.push_back(name);
}
}
dedupe(out);
return out;
}
static std::vector<std::string> parseRustStub(const std::string& path) {
std::vector<std::string> out;
std::ifstream in(path);
if (!in.is_open()) return out;
std::string line;
while (std::getline(in, line)) {
line = trim(line);
if (line.empty() || line.rfind("//", 0) == 0) continue;
if (line.rfind("pub fn", 0) == 0) {
std::string name = extractAfterKeyword(line, "fn");
if (!name.empty()) out.push_back(name);
} else if (line.rfind("pub struct", 0) == 0) {
std::string name = extractAfterKeyword(line, "struct");
if (!name.empty()) out.push_back(name);
} else if (line.rfind("pub enum", 0) == 0) {
std::string name = extractAfterKeyword(line, "enum");
if (!name.empty()) out.push_back(name);
} else if (line.rfind("pub trait", 0) == 0) {
std::string name = extractAfterKeyword(line, "trait");
if (!name.empty()) out.push_back(name);
}
}
dedupe(out);
return out;
}
static std::string extractAfterKeyword(const std::string& line, const std::string& keyword) {
auto pos = line.find(keyword);
if (pos == std::string::npos) return "";
pos += keyword.size();
while (pos < line.size() && std::isspace((unsigned char)line[pos])) pos++;
std::string name;
while (pos < line.size() && isIdentChar(line[pos])) {
name.push_back(line[pos]);
pos++;
}
return name;
}
static std::string extractBefore(const std::string& line, char delim) {
auto pos = line.find(delim);
if (pos == std::string::npos) return "";
return trim(line.substr(0, pos));
}
static std::string stripCppComment(const std::string& line) {
auto pos = line.find("//");
if (pos == std::string::npos) return line;
return line.substr(0, pos);
}
static std::string lastIdentifier(const std::string& text) {
int i = (int)text.size() - 1;
while (i >= 0 && !isIdentChar(text[i])) --i;
if (i < 0) return "";
int end = i;
while (i >= 0 && isIdentChar(text[i])) --i;
return text.substr(i + 1, end - i);
}
};

78
editor/tests/step131_test.cpp Normal file
View File

@@ -0,0 +1,78 @@
// Step 131 TDD Test: Library API indexing via type stubs
#include "StubParser.h"
#include "LibraryIndexer.h"
#include <iostream>
#include <filesystem>
#include <fstream>
static void expect(bool cond, const std::string& name, int& passed, int& failed) {
if (cond) {
std::cout << "Test " << (passed + failed + 1) << " PASS: " << name << "\n";
++passed;
} else {
std::cout << "Test " << (passed + failed + 1) << " FAIL: " << name << "\n";
++failed;
}
}
int main() {
int passed = 0;
int failed = 0;
std::filesystem::path dir = std::filesystem::current_path() / "stub_test_tmp";
std::filesystem::remove_all(dir);
std::filesystem::create_directories(dir / "mylib");
{
std::ofstream out(dir / "mylib" / "mylib.pyi");
out << "def foo(x: int) -> int: ...\nclass Bar: ...\nCONST: int\n";
}
{
std::ofstream out(dir / "mylib" / "index.d.ts");
out << "export function baz(x: number): number;\nexport interface Thing {}\n";
}
{
std::ofstream out(dir / "mylib" / "mylib.h");
out << "int add(int a, int b);\nclass Widget;\n";
}
{
std::filesystem::create_directories(dir / "mylib" / "src");
std::ofstream out(dir / "mylib" / "src" / "lib.rs");
out << "pub fn qux(x: i32) -> i32 { x }\npub struct Gizmo {}\n";
}
auto py = StubParser::parseFile((dir / "mylib" / "mylib.pyi").string(), "python");
expect(py.size() >= 2, "python stub parsed", passed, failed);
auto ts = StubParser::parseFile((dir / "mylib" / "index.d.ts").string(), "typescript");
expect(ts.size() >= 2, "typescript stub parsed", passed, failed);
auto hdr = StubParser::parseFile((dir / "mylib" / "mylib.h").string(), "cpp");
expect(!hdr.empty(), "header stub parsed", passed, failed);
auto rs = StubParser::parseFile((dir / "mylib" / "src" / "lib.rs").string(), "rust");
expect(rs.size() >= 2, "rust stub parsed", passed, failed);
auto scanned = StubParser::scanWorkspaceForLibrary(dir.string(), "mylib", "python");
expect(!scanned.empty(), "scan workspace finds stubs", passed, failed);
LibraryIndexData index;
std::vector<DependencySpec> deps;
deps.push_back({"mylib", "0.1.0", "requirements.txt"});
applyStubFallback(index, deps, dir.string());
expect(index.completionsByLibrary.count("mylib") == 1, "stub fallback fills index", passed, failed);
Module mod("m1", "Module", "python");
rebuildExternalModules(&mod, deps, index);
auto ext = mod.getChildren("externalModules");
expect(!ext.empty(), "external module created from stubs", passed, failed);
if (!ext.empty()) {
auto sigs = ext[0]->getChildren("signatures");
expect(!sigs.empty(), "signatures attached", passed, failed);
}
std::filesystem::remove_all(dir);
std::cout << "\n=== Step 131 Results: " << passed << " passed, " << failed << " failed ===\n";
return failed == 0 ? 0 : 1;
}

2
sprint5_plan.md
View File

@@ -61,7 +61,7 @@ The foundation: import libraries, parse their APIs, index their symbols.
Cache results in `ExternalModule` AST nodes (name, public symbols, type signatures).
*Modifies:* `LSPClient.h`, `ExternalModule` AST concept
- [ ] **Step 131: Library API indexing via type stubs**
- [x] **Step 131: Library API indexing via type stubs**
Fallback for when LSP isn't available or insufficient. Parse type stub files:
- Python: `.pyi` stub files (typeshed, bundled stubs)
- TypeScript: `.d.ts` declaration files