bill/whetstone_DSL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Sprint 6 Step 193: semantic library tags

Browse Source
This commit is contained in:
Bill
2026-02-10 02:12:48 -07:00
parent 330fb9e668
commit 331999ce14
11 changed files with 259 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
PROGRESS.md
View File

@@ -540,4 +540,5 @@ Sprint 5 in progress. Step 141 (Emacs daemon with user config) done. Next: Step
| 2026-02-10 | Codex | Step 190: Vulnerability database (OSV parsing, cache+TTL, offline mode, query/range matching, background refresh hook). 1/1 tests pass (step190_test). |
| 2026-02-10 | Codex | Step 191: Dependency security badges with severity colors, advisory details, safe upgrade path, and ignore list persistence. 1/1 tests pass (step191_test). |
| 2026-02-10 | Codex | Step 192: Security diagnostics integration (gutter shields, [Security] problems, vulnerable import blocking, and agent import deprioritization). 1/1 tests pass (step192_test). |
| 2026-02-10 | Codex | Step 193: Semantic library tags (semantic_tags.json storage, auto-tagging heuristics, tags attached to ExternalModule/TypeSignature). 1/1 tests pass (step193_test). |
| 2026-02-10 | Codex | Step 166: Extract main.cpp into panel headers marked complete (already implemented). step166_test passes; file_limits_test 4/4 passes. |

3
editor/CMakeLists.txt
View File

@@ -1116,6 +1116,9 @@ target_include_directories(step191_test PRIVATE src)
add_executable(step192_test tests/step192_test.cpp)
target_include_directories(step192_test PRIVATE src)
add_executable(step193_test tests/step193_test.cpp)
target_include_directories(step193_test PRIVATE src)
find_package(SDL2 CONFIG REQUIRED)
find_package(OpenGL REQUIRED)
find_package(glad CONFIG REQUIRED)

6
editor/src/EditorState.h
View File

@@ -966,6 +966,7 @@ struct EditorState {
loadSettingsFromDisk();
registerCommands();
library.vulnDb.startBackgroundRefresh();
library.semanticTags.load();
events.subscribe(UIEventType::BufferSwitched, [this](const UIEvent&) {
if (!active()) return;
library.primitives.setRoot(activeAST());
@@ -1262,7 +1263,10 @@ struct EditorState {
void rebuildExternalModulesFromIndex() {
Module* ast = activeAST();
if (!ast) return;
rebuildExternalModules(ast, library.dependencyPanel.deps, library.libraryIndex);
rebuildExternalModules(ast,
library.dependencyPanel.deps,
library.libraryIndex,
&library.semanticTags);
if (active() && active()->language == "elisp") {
int signatureId = 0;
appendEmacsExternalModules(ast, emacsState.emacsFunctionIndex, signatureId);

13
editor/src/LibraryIndexer.h
View File

@@ -5,6 +5,7 @@
#include "ast/ExternalModule.h"
#include "ast/TypeSignature.h"
#include "ast/Module.h"
#include "SemanticTags.h"
#include <unordered_map>
#include <unordered_set>
#include <string>
@@ -45,7 +46,8 @@ static inline bool symbolMatchesLibrary(const LSPClient::WorkspaceSymbol& sym,
static inline void rebuildExternalModules(Module* module,
const std::vector<DependencySpec>& deps,
const LibraryIndexData& index) {
const LibraryIndexData& index,
SemanticTags* semanticTags = nullptr) {
if (!module) return;
clearExternalModules(module);
std::unordered_map<std::string, int> nameCounts;
@@ -58,6 +60,9 @@ static inline void rebuildExternalModules(Module* module,
auto* ext = new ExternalModule(id, dep.name,
languageForDependencySource(dep.source),
dep.version);
if (semanticTags) {
ext->semanticTags = semanticTags->tagsForLibrary(dep.name);
}
std::unordered_set<std::string> added;
auto it = index.symbolsByLibrary.find(dep.name);
@@ -66,6 +71,9 @@ static inline void rebuildExternalModules(Module* module,
if (!symbolMatchesLibrary(sym, dep.name)) continue;
if (!added.insert(sym.name).second) continue;
auto* sig = new TypeSignature("sig_" + std::to_string(signatureId++), sym.name);
if (semanticTags) {
sig->semanticTags = semanticTags->tagsForSymbol(dep.name, sym.name);
}
ext->addChild("signatures", sig);
}
}
@@ -74,6 +82,9 @@ static inline void rebuildExternalModules(Module* module,
for (const auto& name : it2->second) {
if (!added.insert(name).second) continue;
auto* sig = new TypeSignature("sig_" + std::to_string(signatureId++), name);
if (semanticTags) {
sig->semanticTags = semanticTags->tagsForSymbol(dep.name, name);
}
ext->addChild("signatures", sig);
}
}

185
editor/src/SemanticTags.h Normal file
View File

@@ -0,0 +1,185 @@
#pragma once
// Step 193: Semantic annotation tags for libraries.
#include <string>
#include <vector>
#include <unordered_map>
#include <unordered_set>
#include <filesystem>
#include <fstream>
#include <algorithm>
#include <cctype>
#include <nlohmann/json.hpp>
class SemanticTags {
public:
using TagList = std::vector<std::string>;
const TagList& vocabulary() const { return vocab_; }
void load() {
if (loaded_) return;
loaded_ = true;
tags_.clear();
std::filesystem::path path = defaultPath();
std::ifstream in(path);
if (!in.is_open()) return;
nlohmann::json j;
try { in >> j; } catch (...) { return; }
if (!j.is_object()) return;
for (auto it = j.begin(); it != j.end(); ++it) {
if (!it.value().is_object()) continue;
for (auto symIt = it.value().begin(); symIt != it.value().end(); ++symIt) {
TagList tags;
if (symIt.value().is_array()) {
for (const auto& tag : symIt.value()) {
if (tag.is_string()) tags.push_back(normalizeTag(tag.get<std::string>()));
}
}
if (!tags.empty()) tags_[it.key()][symIt.key()] = tags;
}
}
}
void save() const {
std::filesystem::path path = defaultPath();
std::filesystem::create_directories(path.parent_path());
nlohmann::json j = nlohmann::json::object();
for (const auto& [lib, symMap] : tags_) {
nlohmann::json symObj = nlohmann::json::object();
for (const auto& [sym, tags] : symMap) {
symObj[sym] = tags;
}
j[lib] = symObj;
}
std::ofstream out(path);
if (!out.is_open()) return;
out << j.dump(2);
}
void setTags(const std::string& library,
const std::string& symbol,
const TagList& tags) {
load();
TagList normalized;
for (const auto& t : tags) normalized.push_back(normalizeTag(t));
tags_[library][symbol] = dedupe(normalized);
}
TagList tagsForLibrary(const std::string& library) {
load();
return dedupe(autoTagsForLibrary(library));
}
TagList tagsForSymbol(const std::string& library,
const std::string& symbol,
const std::string& doc = "") {
load();
TagList out;
TagList libTags = autoTagsForLibrary(library);
out.insert(out.end(), libTags.begin(), libTags.end());
auto libIt = tags_.find(library);
if (libIt != tags_.end()) {
auto symIt = libIt->second.find(symbol);
if (symIt != libIt->second.end()) {
out.insert(out.end(), symIt->second.begin(), symIt->second.end());
}
}
TagList inferred = inferTagsFromName(symbol);
if (!doc.empty()) {
TagList docInferred = inferTagsFromName(doc);
inferred.insert(inferred.end(), docInferred.begin(), docInferred.end());
}
out.insert(out.end(), inferred.begin(), inferred.end());
return dedupe(out);
}
private:
bool loaded_ = false;
std::unordered_map<std::string, std::unordered_map<std::string, TagList>> tags_;
TagList vocab_ = {"@serialize", "@crypto", "@io", "@network", "@math",
"@collection", "@concurrency", "@test", "@ui", "@parse"};
static std::filesystem::path defaultPath() {
const char* home = std::getenv("USERPROFILE");
if (!home) home = std::getenv("HOME");
std::filesystem::path base = home ? home : ".";
return base / ".whetstone" / "semantic_tags.json";
}
static std::string normalizeTag(const std::string& tag) {
if (tag.empty()) return tag;
if (tag[0] == '@') return tag;
return "@" + tag;
}
static TagList dedupe(const TagList& tags) {
std::unordered_set<std::string> seen;
TagList out;
for (const auto& t : tags) {
if (t.empty()) continue;
if (seen.insert(t).second) out.push_back(t);
}
return out;
}
static std::string lowerCopy(const std::string& input) {
std::string out = input;
std::transform(out.begin(), out.end(), out.begin(),
[](unsigned char c) { return (char)std::tolower(c); });
return out;
}
TagList autoTagsForLibrary(const std::string& library) const {
std::string name = lowerCopy(library);
if (name == "numpy" || name == "pandas" || name == "scipy") return {"@math"};
if (name == "requests" || name == "httpx" || name == "axios") return {"@network"};
if (name == "cryptography" || name == "openssl") return {"@crypto"};
if (name == "json" || name == "yaml" || name == "toml" || name == "pickle" ||
name == "serde") return {"@serialize"};
if (name == "sqlite" || name == "sqlite3" || name == "sqlalchemy") return {"@io"};
if (name == "pytest" || name == "unittest" || name == "jest") return {"@test"};
if (name == "tkinter" || name == "react" || name == "qt") return {"@ui"};
return {};
}
TagList inferTagsFromName(const std::string& name) const {
std::string lower = lowerCopy(name);
TagList out;
if (lower.find("serialize") != std::string::npos || lower.find("json") != std::string::npos) {
out.push_back("@serialize");
}
if (lower.find("encrypt") != std::string::npos || lower.find("decrypt") != std::string::npos ||
lower.find("hash") != std::string::npos || lower.find("crypto") != std::string::npos) {
out.push_back("@crypto");
}
if (lower.find("http") != std::string::npos || lower.find("request") != std::string::npos ||
lower.find("socket") != std::string::npos || lower.find("net") != std::string::npos) {
out.push_back("@network");
}
if (lower.find("read") != std::string::npos || lower.find("write") != std::string::npos ||
lower.find("file") != std::string::npos || lower.find("stream") != std::string::npos) {
out.push_back("@io");
}
if (lower.find("math") != std::string::npos || lower.find("calc") != std::string::npos ||
lower.find("matrix") != std::string::npos) {
out.push_back("@math");
}
if (lower.find("list") != std::string::npos || lower.find("map") != std::string::npos ||
lower.find("dict") != std::string::npos || lower.find("set") != std::string::npos) {
out.push_back("@collection");
}
if (lower.find("thread") != std::string::npos || lower.find("mutex") != std::string::npos ||
lower.find("lock") != std::string::npos) {
out.push_back("@concurrency");
}
if (lower.find("test") != std::string::npos) out.push_back("@test");
if (lower.find("ui") != std::string::npos || lower.find("widget") != std::string::npos) {
out.push_back("@ui");
}
if (lower.find("parse") != std::string::npos || lower.find("lexer") != std::string::npos) {
out.push_back("@parse");
}
return out;
}
};

2
editor/src/ast/ExternalModule.h
View File

@@ -1,11 +1,13 @@
#pragma once
#include "ASTNode.h"
#include <vector>
class ExternalModule : public ASTNode {
public:
std::string name;
std::string version;
std::string language;
std::vector<std::string> semanticTags;
ExternalModule() { conceptType = "ExternalModule"; }
ExternalModule(const std::string& id, const std::string& name,

14
editor/src/ast/Serialization.h
View File

@@ -103,11 +103,13 @@ inline json propertiesToJson(const ASTNode* node) {
props["name"] = n->name;
if (!n->version.empty()) props["version"] = n->version;
if (!n->language.empty()) props["language"] = n->language;
if (!n->semanticTags.empty()) props["semanticTags"] = n->semanticTags;
}
else if (ct == "TypeSignature") {
auto* n = static_cast<const TypeSignature*>(node);
props["name"] = n->name;
props["variadic"] = n->variadic;
if (!n->semanticTags.empty()) props["semanticTags"] = n->semanticTags;
}
// Annotations
else if (ct == "DerefStrategy") {
@@ -286,11 +288,23 @@ inline void setPropertiesFromJson(ASTNode* node, const json& props) {
if (props.contains("name")) n->name = props["name"].get<std::string>();
if (props.contains("version")) n->version = props["version"].get<std::string>();
if (props.contains("language")) n->language = props["language"].get<std::string>();
if (props.contains("semanticTags") && props["semanticTags"].is_array()) {
n->semanticTags.clear();
for (const auto& tag : props["semanticTags"]) {
if (tag.is_string()) n->semanticTags.push_back(tag.get<std::string>());
}
}
}
else if (ct == "TypeSignature") {
auto* n = static_cast<TypeSignature*>(node);
if (props.contains("name")) n->name = props["name"].get<std::string>();
if (props.contains("variadic")) n->variadic = props["variadic"].get<bool>();
if (props.contains("semanticTags") && props["semanticTags"].is_array()) {
n->semanticTags.clear();
for (const auto& tag : props["semanticTags"]) {
if (tag.is_string()) n->semanticTags.push_back(tag.get<std::string>());
}
}
}
else if (ct == "DerefStrategy") {
auto* n = static_cast<DerefStrategy*>(node);

2
editor/src/ast/TypeSignature.h
View File

@@ -1,10 +1,12 @@
#pragma once
#include "ASTNode.h"
#include <vector>
class TypeSignature : public ASTNode {
public:
std::string name;
bool variadic = false;
std::vector<std::string> semanticTags;
TypeSignature() { conceptType = "TypeSignature"; }
TypeSignature(const std::string& id, const std::string& name, bool variadic = false)

2
editor/src/state/LibraryState.h
View File

@@ -5,6 +5,7 @@
#include "LibraryIndexer.h"
#include "PrimitivesRegistry.h"
#include "VulnerabilityDatabase.h"
#include "SemanticTags.h"
struct LibraryState {
bool showDependencyPanel = true;
@@ -26,4 +27,5 @@ struct LibraryState {
LibraryIndexData libraryIndex;
PrimitivesRegistry primitives;
VulnerabilityDatabase vulnDb;
SemanticTags semanticTags;
};

32
editor/tests/step193_test.cpp Normal file
View File

@@ -0,0 +1,32 @@
// Step 193: Semantic tags for libraries.
#include <cassert>
#include <fstream>
#include <string>
static std::string readFile(const std::string& path) {
std::ifstream f(path);
if (!f.is_open()) return {};
return std::string((std::istreambuf_iterator<char>(f)),
std::istreambuf_iterator<char>());
}
static void assertContains(const std::string& text, const std::string& needle) {
assert(text.find(needle) != std::string::npos);
}
int main() {
const std::string tags = readFile("src/SemanticTags.h");
const std::string ext = readFile("src/ast/ExternalModule.h");
const std::string sig = readFile("src/ast/TypeSignature.h");
const std::string indexer = readFile("src/LibraryIndexer.h");
assertContains(tags, "@serialize");
assertContains(tags, "semantic_tags.json");
assertContains(ext, "semanticTags");
assertContains(sig, "semanticTags");
assertContains(indexer, "tagsForSymbol");
printf("step193_test: all assertions passed\n");
return 0;
}

2
sprint6_plan.md
View File

@@ -344,7 +344,7 @@ feature requests with polished UX.
- Configurable: "Block vulnerable imports" option (default off, warns only)
*Modifies:* diagnostic rendering, `PrimitivesRegistry.h`
- [ ] **Step 193: Semantic annotation tags for libraries**
- [x] **Step 193: Semantic annotation tags for libraries**
`SemanticTags.h`:
- Predefined tag vocabulary: `@serialize`, `@crypto`, `@io`, `@network`,
`@math`, `@collection`, `@concurrency`, `@test`, `@ui`, `@parse`