diff --git a/PROGRESS.md b/PROGRESS.md index 652964e..f82a565 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -382,7 +382,7 @@ vcpkg's imgui 1.91.9 removed the `sdl2-binding` feature (only `sdl3-binding` exi ## What's Next -Sprint 5 in progress. Step 130 (library API indexing via LSP) done. Next: Step 131 (library API indexing via type stubs). +Sprint 5 in progress. Step 131 (library API indexing via type stubs) done. Next: Step 132 (library symbol browser). --- @@ -476,3 +476,4 @@ Sprint 5 in progress. Step 130 (library API indexing via LSP) done. Next: Step 1 | 2026-02-09 | Codex | Step 128: Dependency file parsing with Import population. 6/6 tests pass. | | 2026-02-09 | Codex | Step 129: Dependency management UI panel with add/remove/update, source targeting, and writeback support for requirements.txt/package.json/Cargo.toml/go.mod/vcpkg.json. 8/8 tests pass. | | 2026-02-09 | Codex | Step 130: LSP workspace symbol + completion indexing for dependencies with ExternalModule population and library index polling. 7/7 tests pass. | +| 2026-02-09 | Codex | Step 131: Stub-based library indexing (pyi/d.ts/headers/lib.rs) with workspace scan fallback. 8/8 tests pass. | diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index ba75471..aa325b1 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -736,6 +736,10 @@ add_executable(step130_test tests/step130_test.cpp) target_include_directories(step130_test PRIVATE src) target_link_libraries(step130_test PRIVATE nlohmann_json::nlohmann_json) +add_executable(step131_test tests/step131_test.cpp) +target_include_directories(step131_test PRIVATE src) +target_link_libraries(step131_test PRIVATE nlohmann_json::nlohmann_json) + find_package(SDL2 CONFIG REQUIRED) find_package(OpenGL REQUIRED) find_package(glad CONFIG REQUIRED) diff --git a/editor/src/EditorState.h b/editor/src/EditorState.h index 488436a..b0dfc87 100644 --- a/editor/src/EditorState.h +++ b/editor/src/EditorState.h @@ -802,6 +802,7 @@ struct EditorState { return; } if (!lsp || !lspTransport || !lspTransport->isOpen()) { + applyStubFallback(libraryIndex, dependencyPanel.deps, workspaceRoot); rebuildExternalModulesFromIndex(); outputLog += "[deps] Library index uses cached symbols (LSP offline).\n"; return; @@ -822,6 +823,7 @@ struct EditorState { } libraryIndexRequests.push_back(req); } + applyStubFallback(libraryIndex, dependencyPanel.deps, workspaceRoot); rebuildExternalModulesFromIndex(); outputLog += "[deps] Library index requests queued: " + std::to_string(libraryIndexRequests.size()) + "\n"; @@ -856,6 +858,7 @@ struct EditorState { } } if (updated) { + applyStubFallback(libraryIndex, dependencyPanel.deps, workspaceRoot); rebuildExternalModulesFromIndex(); outputLog += "[deps] Library index updated.\n"; } diff --git a/editor/src/LibraryIndexer.h b/editor/src/LibraryIndexer.h index d5afa9f..91abeba 100644 --- a/editor/src/LibraryIndexer.h +++ b/editor/src/LibraryIndexer.h @@ -1,6 +1,7 @@ #pragma once #include "DependencyParser.h" #include "LSPClient.h" +#include "StubParser.h" #include "ast/ExternalModule.h" #include "ast/TypeSignature.h" #include "ast/Module.h" @@ -80,3 +81,19 @@ static inline void rebuildExternalModules(Module* module, module->addChild("externalModules", ext); } } + +static inline void applyStubFallback(LibraryIndexData& index, + const std::vector& deps, + const std::string& workspaceRoot) { + for (const auto& dep : deps) { + if (dep.name.empty()) continue; + bool hasSymbols = index.symbolsByLibrary.count(dep.name) > 0; + bool hasCompletions = index.completionsByLibrary.count(dep.name) > 0; + if (hasSymbols || hasCompletions) continue; + std::string lang = languageForDependencySource(dep.source); + auto names = StubParser::scanWorkspaceForLibrary(workspaceRoot, dep.name, lang); + if (!names.empty()) { + index.completionsByLibrary[dep.name] = std::move(names); + } + } +} diff --git a/editor/src/StubParser.h b/editor/src/StubParser.h new file mode 100644 index 0000000..0d157c2 --- /dev/null +++ b/editor/src/StubParser.h @@ -0,0 +1,231 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + +class StubParser { +public: + static std::vector parseFile(const std::string& path, + const std::string& languageHint) { + std::filesystem::path p(path); + std::string ext = p.extension().string(); + if (ext == ".pyi") return parsePythonStub(path); + if (ext == ".d.ts") return parseTypeScriptStub(path); + if (ext == ".h" || ext == ".hpp" || ext == ".hh") return parseHeaderStub(path); + if (p.filename() == "lib.rs") return parseRustStub(path); + if (languageHint == "python") return parsePythonStub(path); + if (languageHint == "javascript" || languageHint == "typescript") return parseTypeScriptStub(path); + if (languageHint == "rust") return parseRustStub(path); + return parseHeaderStub(path); + } + + static std::vector scanWorkspaceForLibrary(const std::string& workspaceRoot, + const std::string& library, + const std::string& languageHint) { + std::vector out; + if (workspaceRoot.empty() || library.empty()) return out; + std::filesystem::path root(workspaceRoot); + if (!std::filesystem::exists(root)) return out; + + const int maxFiles = 8; + int scanned = 0; + std::string lowerLib = toLower(library); + + for (auto it = std::filesystem::recursive_directory_iterator(root); + it != std::filesystem::recursive_directory_iterator(); ++it) { + if (scanned >= maxFiles) break; + if (!it->is_regular_file()) continue; + const auto& path = it->path(); + std::string filename = path.filename().string(); + std::string ext = path.extension().string(); + if (!isStubCandidate(filename, ext, languageHint)) continue; + std::string pathLower = toLower(path.string()); + if (pathLower.find(lowerLib) == std::string::npos) continue; + auto parsed = parseFile(path.string(), languageHint); + if (!parsed.empty()) { + out.insert(out.end(), parsed.begin(), parsed.end()); + scanned++; + } + } + dedupe(out); + return out; + } + +private: + static std::string trim(const std::string& s) { + size_t start = s.find_first_not_of(" \t\r\n"); + size_t end = s.find_last_not_of(" \t\r\n"); + if (start == std::string::npos || end == std::string::npos) return ""; + return s.substr(start, end - start + 1); + } + + static std::string toLower(const std::string& s) { + std::string out = s; + std::transform(out.begin(), out.end(), out.begin(), + [](unsigned char c) { return (char)std::tolower(c); }); + return out; + } + + static bool isIdentChar(char c) { + return std::isalnum((unsigned char)c) || c == '_' || c == ':'; + } + + static void dedupe(std::vector& items) { + std::unordered_set seen; + std::vector out; + for (const auto& item : items) { + if (item.empty()) continue; + if (seen.insert(item).second) out.push_back(item); + } + items.swap(out); + } + + static bool isStubCandidate(const std::string& filename, + const std::string& ext, + const std::string& languageHint) { + if (ext == ".pyi") return true; + if (ext == ".d.ts") return true; + if (ext == ".h" || ext == ".hpp" || ext == ".hh") return true; + if (filename == "lib.rs") return true; + if (languageHint == "python" && ext == ".pyi") return true; + if ((languageHint == "javascript" || languageHint == "typescript") && ext == ".d.ts") return true; + if (languageHint == "rust" && filename == "lib.rs") return true; + if (languageHint == "cpp" && (ext == ".h" || ext == ".hpp" || ext == ".hh")) return true; + return false; + } + + static std::vector parsePythonStub(const std::string& path) { + std::vector out; + std::ifstream in(path); + if (!in.is_open()) return out; + std::string line; + while (std::getline(in, line)) { + line = trim(line); + if (line.empty() || line[0] == '#') continue; + if (line.rfind("def ", 0) == 0) { + std::string name = extractAfterKeyword(line, "def"); + if (!name.empty()) out.push_back(name); + } else if (line.rfind("class ", 0) == 0) { + std::string name = extractAfterKeyword(line, "class"); + if (!name.empty()) out.push_back(name); + } else if (line.rfind("CONST", 0) == 0 || line.find(":") != std::string::npos) { + std::string name = extractBefore(line, ':'); + if (!name.empty()) out.push_back(name); + } + } + dedupe(out); + return out; + } + + static std::vector parseTypeScriptStub(const std::string& path) { + std::vector out; + std::ifstream in(path); + if (!in.is_open()) return out; + std::string line; + while (std::getline(in, line)) { + line = trim(line); + if (line.empty() || line.rfind("//", 0) == 0) continue; + if (line.rfind("export function", 0) == 0 || line.rfind("function", 0) == 0) { + std::string name = extractAfterKeyword(line, "function"); + if (!name.empty()) out.push_back(name); + } else if (line.rfind("export class", 0) == 0 || line.rfind("class", 0) == 0) { + std::string name = extractAfterKeyword(line, "class"); + if (!name.empty()) out.push_back(name); + } else if (line.rfind("interface", 0) == 0 || line.rfind("export interface", 0) == 0) { + std::string name = extractAfterKeyword(line, "interface"); + if (!name.empty()) out.push_back(name); + } else if (line.rfind("const ", 0) == 0 || line.rfind("export const", 0) == 0) { + std::string name = extractAfterKeyword(line, "const"); + if (!name.empty()) out.push_back(name); + } + } + dedupe(out); + return out; + } + + static std::vector parseHeaderStub(const std::string& path) { + std::vector out; + std::ifstream in(path); + if (!in.is_open()) return out; + std::string line; + while (std::getline(in, line)) { + line = stripCppComment(line); + line = trim(line); + if (line.empty()) continue; + auto pos = line.find('('); + if (pos == std::string::npos) continue; + std::string left = trim(line.substr(0, pos)); + std::string name = lastIdentifier(left); + if (!name.empty() && name != "if" && name != "for" && name != "while") { + out.push_back(name); + } + } + dedupe(out); + return out; + } + + static std::vector parseRustStub(const std::string& path) { + std::vector out; + std::ifstream in(path); + if (!in.is_open()) return out; + std::string line; + while (std::getline(in, line)) { + line = trim(line); + if (line.empty() || line.rfind("//", 0) == 0) continue; + if (line.rfind("pub fn", 0) == 0) { + std::string name = extractAfterKeyword(line, "fn"); + if (!name.empty()) out.push_back(name); + } else if (line.rfind("pub struct", 0) == 0) { + std::string name = extractAfterKeyword(line, "struct"); + if (!name.empty()) out.push_back(name); + } else if (line.rfind("pub enum", 0) == 0) { + std::string name = extractAfterKeyword(line, "enum"); + if (!name.empty()) out.push_back(name); + } else if (line.rfind("pub trait", 0) == 0) { + std::string name = extractAfterKeyword(line, "trait"); + if (!name.empty()) out.push_back(name); + } + } + dedupe(out); + return out; + } + + static std::string extractAfterKeyword(const std::string& line, const std::string& keyword) { + auto pos = line.find(keyword); + if (pos == std::string::npos) return ""; + pos += keyword.size(); + while (pos < line.size() && std::isspace((unsigned char)line[pos])) pos++; + std::string name; + while (pos < line.size() && isIdentChar(line[pos])) { + name.push_back(line[pos]); + pos++; + } + return name; + } + + static std::string extractBefore(const std::string& line, char delim) { + auto pos = line.find(delim); + if (pos == std::string::npos) return ""; + return trim(line.substr(0, pos)); + } + + static std::string stripCppComment(const std::string& line) { + auto pos = line.find("//"); + if (pos == std::string::npos) return line; + return line.substr(0, pos); + } + + static std::string lastIdentifier(const std::string& text) { + int i = (int)text.size() - 1; + while (i >= 0 && !isIdentChar(text[i])) --i; + if (i < 0) return ""; + int end = i; + while (i >= 0 && isIdentChar(text[i])) --i; + return text.substr(i + 1, end - i); + } +}; diff --git a/editor/tests/step131_test.cpp b/editor/tests/step131_test.cpp new file mode 100644 index 0000000..ac6c83e --- /dev/null +++ b/editor/tests/step131_test.cpp @@ -0,0 +1,78 @@ +// Step 131 TDD Test: Library API indexing via type stubs +#include "StubParser.h" +#include "LibraryIndexer.h" +#include +#include +#include + +static void expect(bool cond, const std::string& name, int& passed, int& failed) { + if (cond) { + std::cout << "Test " << (passed + failed + 1) << " PASS: " << name << "\n"; + ++passed; + } else { + std::cout << "Test " << (passed + failed + 1) << " FAIL: " << name << "\n"; + ++failed; + } +} + +int main() { + int passed = 0; + int failed = 0; + + std::filesystem::path dir = std::filesystem::current_path() / "stub_test_tmp"; + std::filesystem::remove_all(dir); + std::filesystem::create_directories(dir / "mylib"); + + { + std::ofstream out(dir / "mylib" / "mylib.pyi"); + out << "def foo(x: int) -> int: ...\nclass Bar: ...\nCONST: int\n"; + } + { + std::ofstream out(dir / "mylib" / "index.d.ts"); + out << "export function baz(x: number): number;\nexport interface Thing {}\n"; + } + { + std::ofstream out(dir / "mylib" / "mylib.h"); + out << "int add(int a, int b);\nclass Widget;\n"; + } + { + std::filesystem::create_directories(dir / "mylib" / "src"); + std::ofstream out(dir / "mylib" / "src" / "lib.rs"); + out << "pub fn qux(x: i32) -> i32 { x }\npub struct Gizmo {}\n"; + } + + auto py = StubParser::parseFile((dir / "mylib" / "mylib.pyi").string(), "python"); + expect(py.size() >= 2, "python stub parsed", passed, failed); + + auto ts = StubParser::parseFile((dir / "mylib" / "index.d.ts").string(), "typescript"); + expect(ts.size() >= 2, "typescript stub parsed", passed, failed); + + auto hdr = StubParser::parseFile((dir / "mylib" / "mylib.h").string(), "cpp"); + expect(!hdr.empty(), "header stub parsed", passed, failed); + + auto rs = StubParser::parseFile((dir / "mylib" / "src" / "lib.rs").string(), "rust"); + expect(rs.size() >= 2, "rust stub parsed", passed, failed); + + auto scanned = StubParser::scanWorkspaceForLibrary(dir.string(), "mylib", "python"); + expect(!scanned.empty(), "scan workspace finds stubs", passed, failed); + + LibraryIndexData index; + std::vector deps; + deps.push_back({"mylib", "0.1.0", "requirements.txt"}); + applyStubFallback(index, deps, dir.string()); + expect(index.completionsByLibrary.count("mylib") == 1, "stub fallback fills index", passed, failed); + + Module mod("m1", "Module", "python"); + rebuildExternalModules(&mod, deps, index); + auto ext = mod.getChildren("externalModules"); + expect(!ext.empty(), "external module created from stubs", passed, failed); + if (!ext.empty()) { + auto sigs = ext[0]->getChildren("signatures"); + expect(!sigs.empty(), "signatures attached", passed, failed); + } + + std::filesystem::remove_all(dir); + + std::cout << "\n=== Step 131 Results: " << passed << " passed, " << failed << " failed ===\n"; + return failed == 0 ? 0 : 1; +} diff --git a/sprint5_plan.md b/sprint5_plan.md index f74becb..fb47320 100644 --- a/sprint5_plan.md +++ b/sprint5_plan.md @@ -61,7 +61,7 @@ The foundation: import libraries, parse their APIs, index their symbols. Cache results in `ExternalModule` AST nodes (name, public symbols, type signatures). *Modifies:* `LSPClient.h`, `ExternalModule` AST concept -- [ ] **Step 131: Library API indexing via type stubs** +- [x] **Step 131: Library API indexing via type stubs** Fallback for when LSP isn't available or insufficient. Parse type stub files: - Python: `.pyi` stub files (typeshed, bundled stubs) - TypeScript: `.d.ts` declaration files