diff --git a/PROGRESS.md b/PROGRESS.md index b15b9da..8930fc5 100644 --- a/PROGRESS.md +++ b/PROGRESS.md @@ -540,4 +540,5 @@ Sprint 5 in progress. Step 141 (Emacs daemon with user config) done. Next: Step | 2026-02-10 | Codex | Step 190: Vulnerability database (OSV parsing, cache+TTL, offline mode, query/range matching, background refresh hook). 1/1 tests pass (step190_test). | | 2026-02-10 | Codex | Step 191: Dependency security badges with severity colors, advisory details, safe upgrade path, and ignore list persistence. 1/1 tests pass (step191_test). | | 2026-02-10 | Codex | Step 192: Security diagnostics integration (gutter shields, [Security] problems, vulnerable import blocking, and agent import deprioritization). 1/1 tests pass (step192_test). | +| 2026-02-10 | Codex | Step 193: Semantic library tags (semantic_tags.json storage, auto-tagging heuristics, tags attached to ExternalModule/TypeSignature). 1/1 tests pass (step193_test). | | 2026-02-10 | Codex | Step 166: Extract main.cpp into panel headers marked complete (already implemented). step166_test passes; file_limits_test 4/4 passes. | diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 68ed1a7..e26dd28 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -1116,6 +1116,9 @@ target_include_directories(step191_test PRIVATE src) add_executable(step192_test tests/step192_test.cpp) target_include_directories(step192_test PRIVATE src) +add_executable(step193_test tests/step193_test.cpp) +target_include_directories(step193_test PRIVATE src) + find_package(SDL2 CONFIG REQUIRED) find_package(OpenGL REQUIRED) find_package(glad CONFIG REQUIRED) diff --git a/editor/src/EditorState.h b/editor/src/EditorState.h index abf7721..00ecc50 100644 --- a/editor/src/EditorState.h +++ b/editor/src/EditorState.h @@ -966,6 +966,7 @@ struct EditorState { loadSettingsFromDisk(); registerCommands(); library.vulnDb.startBackgroundRefresh(); + library.semanticTags.load(); events.subscribe(UIEventType::BufferSwitched, [this](const UIEvent&) { if (!active()) return; library.primitives.setRoot(activeAST()); @@ -1262,7 +1263,10 @@ struct EditorState { void rebuildExternalModulesFromIndex() { Module* ast = activeAST(); if (!ast) return; - rebuildExternalModules(ast, library.dependencyPanel.deps, library.libraryIndex); + rebuildExternalModules(ast, + library.dependencyPanel.deps, + library.libraryIndex, + &library.semanticTags); if (active() && active()->language == "elisp") { int signatureId = 0; appendEmacsExternalModules(ast, emacsState.emacsFunctionIndex, signatureId); diff --git a/editor/src/LibraryIndexer.h b/editor/src/LibraryIndexer.h index 91abeba..a60e8c8 100644 --- a/editor/src/LibraryIndexer.h +++ b/editor/src/LibraryIndexer.h @@ -5,6 +5,7 @@ #include "ast/ExternalModule.h" #include "ast/TypeSignature.h" #include "ast/Module.h" +#include "SemanticTags.h" #include #include #include @@ -45,7 +46,8 @@ static inline bool symbolMatchesLibrary(const LSPClient::WorkspaceSymbol& sym, static inline void rebuildExternalModules(Module* module, const std::vector& deps, - const LibraryIndexData& index) { + const LibraryIndexData& index, + SemanticTags* semanticTags = nullptr) { if (!module) return; clearExternalModules(module); std::unordered_map nameCounts; @@ -58,6 +60,9 @@ static inline void rebuildExternalModules(Module* module, auto* ext = new ExternalModule(id, dep.name, languageForDependencySource(dep.source), dep.version); + if (semanticTags) { + ext->semanticTags = semanticTags->tagsForLibrary(dep.name); + } std::unordered_set added; auto it = index.symbolsByLibrary.find(dep.name); @@ -66,6 +71,9 @@ static inline void rebuildExternalModules(Module* module, if (!symbolMatchesLibrary(sym, dep.name)) continue; if (!added.insert(sym.name).second) continue; auto* sig = new TypeSignature("sig_" + std::to_string(signatureId++), sym.name); + if (semanticTags) { + sig->semanticTags = semanticTags->tagsForSymbol(dep.name, sym.name); + } ext->addChild("signatures", sig); } } @@ -74,6 +82,9 @@ static inline void rebuildExternalModules(Module* module, for (const auto& name : it2->second) { if (!added.insert(name).second) continue; auto* sig = new TypeSignature("sig_" + std::to_string(signatureId++), name); + if (semanticTags) { + sig->semanticTags = semanticTags->tagsForSymbol(dep.name, name); + } ext->addChild("signatures", sig); } } diff --git a/editor/src/SemanticTags.h b/editor/src/SemanticTags.h new file mode 100644 index 0000000..897018c --- /dev/null +++ b/editor/src/SemanticTags.h @@ -0,0 +1,185 @@ +#pragma once +// Step 193: Semantic annotation tags for libraries. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +class SemanticTags { +public: + using TagList = std::vector; + + const TagList& vocabulary() const { return vocab_; } + + void load() { + if (loaded_) return; + loaded_ = true; + tags_.clear(); + std::filesystem::path path = defaultPath(); + std::ifstream in(path); + if (!in.is_open()) return; + nlohmann::json j; + try { in >> j; } catch (...) { return; } + if (!j.is_object()) return; + for (auto it = j.begin(); it != j.end(); ++it) { + if (!it.value().is_object()) continue; + for (auto symIt = it.value().begin(); symIt != it.value().end(); ++symIt) { + TagList tags; + if (symIt.value().is_array()) { + for (const auto& tag : symIt.value()) { + if (tag.is_string()) tags.push_back(normalizeTag(tag.get())); + } + } + if (!tags.empty()) tags_[it.key()][symIt.key()] = tags; + } + } + } + + void save() const { + std::filesystem::path path = defaultPath(); + std::filesystem::create_directories(path.parent_path()); + nlohmann::json j = nlohmann::json::object(); + for (const auto& [lib, symMap] : tags_) { + nlohmann::json symObj = nlohmann::json::object(); + for (const auto& [sym, tags] : symMap) { + symObj[sym] = tags; + } + j[lib] = symObj; + } + std::ofstream out(path); + if (!out.is_open()) return; + out << j.dump(2); + } + + void setTags(const std::string& library, + const std::string& symbol, + const TagList& tags) { + load(); + TagList normalized; + for (const auto& t : tags) normalized.push_back(normalizeTag(t)); + tags_[library][symbol] = dedupe(normalized); + } + + TagList tagsForLibrary(const std::string& library) { + load(); + return dedupe(autoTagsForLibrary(library)); + } + + TagList tagsForSymbol(const std::string& library, + const std::string& symbol, + const std::string& doc = "") { + load(); + TagList out; + TagList libTags = autoTagsForLibrary(library); + out.insert(out.end(), libTags.begin(), libTags.end()); + auto libIt = tags_.find(library); + if (libIt != tags_.end()) { + auto symIt = libIt->second.find(symbol); + if (symIt != libIt->second.end()) { + out.insert(out.end(), symIt->second.begin(), symIt->second.end()); + } + } + TagList inferred = inferTagsFromName(symbol); + if (!doc.empty()) { + TagList docInferred = inferTagsFromName(doc); + inferred.insert(inferred.end(), docInferred.begin(), docInferred.end()); + } + out.insert(out.end(), inferred.begin(), inferred.end()); + return dedupe(out); + } + +private: + bool loaded_ = false; + std::unordered_map> tags_; + TagList vocab_ = {"@serialize", "@crypto", "@io", "@network", "@math", + "@collection", "@concurrency", "@test", "@ui", "@parse"}; + + static std::filesystem::path defaultPath() { + const char* home = std::getenv("USERPROFILE"); + if (!home) home = std::getenv("HOME"); + std::filesystem::path base = home ? home : "."; + return base / ".whetstone" / "semantic_tags.json"; + } + + static std::string normalizeTag(const std::string& tag) { + if (tag.empty()) return tag; + if (tag[0] == '@') return tag; + return "@" + tag; + } + + static TagList dedupe(const TagList& tags) { + std::unordered_set seen; + TagList out; + for (const auto& t : tags) { + if (t.empty()) continue; + if (seen.insert(t).second) out.push_back(t); + } + return out; + } + + static std::string lowerCopy(const std::string& input) { + std::string out = input; + std::transform(out.begin(), out.end(), out.begin(), + [](unsigned char c) { return (char)std::tolower(c); }); + return out; + } + + TagList autoTagsForLibrary(const std::string& library) const { + std::string name = lowerCopy(library); + if (name == "numpy" || name == "pandas" || name == "scipy") return {"@math"}; + if (name == "requests" || name == "httpx" || name == "axios") return {"@network"}; + if (name == "cryptography" || name == "openssl") return {"@crypto"}; + if (name == "json" || name == "yaml" || name == "toml" || name == "pickle" || + name == "serde") return {"@serialize"}; + if (name == "sqlite" || name == "sqlite3" || name == "sqlalchemy") return {"@io"}; + if (name == "pytest" || name == "unittest" || name == "jest") return {"@test"}; + if (name == "tkinter" || name == "react" || name == "qt") return {"@ui"}; + return {}; + } + + TagList inferTagsFromName(const std::string& name) const { + std::string lower = lowerCopy(name); + TagList out; + if (lower.find("serialize") != std::string::npos || lower.find("json") != std::string::npos) { + out.push_back("@serialize"); + } + if (lower.find("encrypt") != std::string::npos || lower.find("decrypt") != std::string::npos || + lower.find("hash") != std::string::npos || lower.find("crypto") != std::string::npos) { + out.push_back("@crypto"); + } + if (lower.find("http") != std::string::npos || lower.find("request") != std::string::npos || + lower.find("socket") != std::string::npos || lower.find("net") != std::string::npos) { + out.push_back("@network"); + } + if (lower.find("read") != std::string::npos || lower.find("write") != std::string::npos || + lower.find("file") != std::string::npos || lower.find("stream") != std::string::npos) { + out.push_back("@io"); + } + if (lower.find("math") != std::string::npos || lower.find("calc") != std::string::npos || + lower.find("matrix") != std::string::npos) { + out.push_back("@math"); + } + if (lower.find("list") != std::string::npos || lower.find("map") != std::string::npos || + lower.find("dict") != std::string::npos || lower.find("set") != std::string::npos) { + out.push_back("@collection"); + } + if (lower.find("thread") != std::string::npos || lower.find("mutex") != std::string::npos || + lower.find("lock") != std::string::npos) { + out.push_back("@concurrency"); + } + if (lower.find("test") != std::string::npos) out.push_back("@test"); + if (lower.find("ui") != std::string::npos || lower.find("widget") != std::string::npos) { + out.push_back("@ui"); + } + if (lower.find("parse") != std::string::npos || lower.find("lexer") != std::string::npos) { + out.push_back("@parse"); + } + return out; + } +}; diff --git a/editor/src/ast/ExternalModule.h b/editor/src/ast/ExternalModule.h index 54e53ac..f3b5cdf 100644 --- a/editor/src/ast/ExternalModule.h +++ b/editor/src/ast/ExternalModule.h @@ -1,11 +1,13 @@ #pragma once #include "ASTNode.h" +#include class ExternalModule : public ASTNode { public: std::string name; std::string version; std::string language; + std::vector semanticTags; ExternalModule() { conceptType = "ExternalModule"; } ExternalModule(const std::string& id, const std::string& name, diff --git a/editor/src/ast/Serialization.h b/editor/src/ast/Serialization.h index fec1c51..e0e16b7 100644 --- a/editor/src/ast/Serialization.h +++ b/editor/src/ast/Serialization.h @@ -103,11 +103,13 @@ inline json propertiesToJson(const ASTNode* node) { props["name"] = n->name; if (!n->version.empty()) props["version"] = n->version; if (!n->language.empty()) props["language"] = n->language; + if (!n->semanticTags.empty()) props["semanticTags"] = n->semanticTags; } else if (ct == "TypeSignature") { auto* n = static_cast(node); props["name"] = n->name; props["variadic"] = n->variadic; + if (!n->semanticTags.empty()) props["semanticTags"] = n->semanticTags; } // Annotations else if (ct == "DerefStrategy") { @@ -286,11 +288,23 @@ inline void setPropertiesFromJson(ASTNode* node, const json& props) { if (props.contains("name")) n->name = props["name"].get(); if (props.contains("version")) n->version = props["version"].get(); if (props.contains("language")) n->language = props["language"].get(); + if (props.contains("semanticTags") && props["semanticTags"].is_array()) { + n->semanticTags.clear(); + for (const auto& tag : props["semanticTags"]) { + if (tag.is_string()) n->semanticTags.push_back(tag.get()); + } + } } else if (ct == "TypeSignature") { auto* n = static_cast(node); if (props.contains("name")) n->name = props["name"].get(); if (props.contains("variadic")) n->variadic = props["variadic"].get(); + if (props.contains("semanticTags") && props["semanticTags"].is_array()) { + n->semanticTags.clear(); + for (const auto& tag : props["semanticTags"]) { + if (tag.is_string()) n->semanticTags.push_back(tag.get()); + } + } } else if (ct == "DerefStrategy") { auto* n = static_cast(node); diff --git a/editor/src/ast/TypeSignature.h b/editor/src/ast/TypeSignature.h index b8feeb9..5369e6e 100644 --- a/editor/src/ast/TypeSignature.h +++ b/editor/src/ast/TypeSignature.h @@ -1,10 +1,12 @@ #pragma once #include "ASTNode.h" +#include class TypeSignature : public ASTNode { public: std::string name; bool variadic = false; + std::vector semanticTags; TypeSignature() { conceptType = "TypeSignature"; } TypeSignature(const std::string& id, const std::string& name, bool variadic = false) diff --git a/editor/src/state/LibraryState.h b/editor/src/state/LibraryState.h index 035defd..e963f67 100644 --- a/editor/src/state/LibraryState.h +++ b/editor/src/state/LibraryState.h @@ -5,6 +5,7 @@ #include "LibraryIndexer.h" #include "PrimitivesRegistry.h" #include "VulnerabilityDatabase.h" +#include "SemanticTags.h" struct LibraryState { bool showDependencyPanel = true; @@ -26,4 +27,5 @@ struct LibraryState { LibraryIndexData libraryIndex; PrimitivesRegistry primitives; VulnerabilityDatabase vulnDb; + SemanticTags semanticTags; }; diff --git a/editor/tests/step193_test.cpp b/editor/tests/step193_test.cpp new file mode 100644 index 0000000..479174e --- /dev/null +++ b/editor/tests/step193_test.cpp @@ -0,0 +1,32 @@ +// Step 193: Semantic tags for libraries. + +#include +#include +#include + +static std::string readFile(const std::string& path) { + std::ifstream f(path); + if (!f.is_open()) return {}; + return std::string((std::istreambuf_iterator(f)), + std::istreambuf_iterator()); +} + +static void assertContains(const std::string& text, const std::string& needle) { + assert(text.find(needle) != std::string::npos); +} + +int main() { + const std::string tags = readFile("src/SemanticTags.h"); + const std::string ext = readFile("src/ast/ExternalModule.h"); + const std::string sig = readFile("src/ast/TypeSignature.h"); + const std::string indexer = readFile("src/LibraryIndexer.h"); + + assertContains(tags, "@serialize"); + assertContains(tags, "semantic_tags.json"); + assertContains(ext, "semanticTags"); + assertContains(sig, "semanticTags"); + assertContains(indexer, "tagsForSymbol"); + + printf("step193_test: all assertions passed\n"); + return 0; +} diff --git a/sprint6_plan.md b/sprint6_plan.md index 3b0f04a..83c5227 100644 --- a/sprint6_plan.md +++ b/sprint6_plan.md @@ -344,7 +344,7 @@ feature requests with polished UX. - Configurable: "Block vulnerable imports" option (default off, warns only) *Modifies:* diagnostic rendering, `PrimitivesRegistry.h` -- [ ] **Step 193: Semantic annotation tags for libraries** +- [x] **Step 193: Semantic annotation tags for libraries** `SemanticTags.h`: - Predefined tag vocabulary: `@serialize`, `@crypto`, `@io`, `@network`, `@math`, `@collection`, `@concurrency`, `@test`, `@ui`, `@parse`