Step 248: compact AST response format (subtree, diff, version tracking)

Adds token-efficient AST queries: compact mode (<30% of full size),
subtree extraction by nodeId, version-tracked AST diff after mutations,
and tokenEstimate on all responses. 12/12 tests pass, 17 tools total.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

editor/src/AgentPermissionPolicy.h

@@ -44,7 +44,9 @@ struct AgentPermissionPolicy {
             method == "projectLanguage" ||
             method == "fileRead" ||
             method == "workspaceList" ||
-            method == "fileDiff") {
+            method == "fileDiff" ||
+            method == "getASTSubtree" ||
+            method == "getASTDiff") {
             return true;
         }
 

editor/src/CompactAST.h

@@ -0,0 +1,191 @@
+#pragma once
+// Step 248: Compact AST response format
+//
+// Token-efficient AST serialization for agent consumption.
+// Provides compact mode, subtree extraction, and AST diff support.
+
+#include "ast/ASTNode.h"
+#include "ast/Serialization.h"
+#include <nlohmann/json.hpp>
+#include <string>
+#include <vector>
+#include <map>
+
+using json = nlohmann::json;
+
+// --- Extract a human-readable name from any AST node ---
+inline std::string getNodeName(const ASTNode* node) {
+    if (!node) return "";
+    const auto& ct = node->conceptType;
+    if (ct == "Module")
+        return static_cast<const Module*>(node)->name;
+    if (ct == "Function")
+        return static_cast<const Function*>(node)->name;
+    if (ct == "Variable")
+        return static_cast<const Variable*>(node)->name;
+    if (ct == "Parameter")
+        return static_cast<const Parameter*>(node)->name;
+    if (ct == "FunctionCall")
+        return static_cast<const FunctionCall*>(node)->functionName;
+    if (ct == "VariableReference")
+        return static_cast<const VariableReference*>(node)->variableName;
+    if (ct == "BinaryOperation")
+        return static_cast<const BinaryOperation*>(node)->op;
+    if (ct == "UnaryOperation")
+        return static_cast<const UnaryOperation*>(node)->op;
+    if (ct == "StringLiteral")
+        return static_cast<const StringLiteral*>(node)->value;
+    if (ct == "Import")
+        return static_cast<const Import*>(node)->moduleName;
+    if (ct == "ExternalModule")
+        return static_cast<const ExternalModule*>(node)->name;
+    if (ct == "PrimitiveType")
+        return static_cast<const PrimitiveType*>(node)->kind;
+    if (ct == "CustomType")
+        return static_cast<const CustomType*>(node)->typeName;
+    if (ct == "MemberAccess")
+        return static_cast<const MemberAccess*>(node)->memberName;
+    if (ct == "TypeSignature")
+        return static_cast<const TypeSignature*>(node)->name;
+    if (ct == "ForLoop")
+        return static_cast<const ForLoop*>(node)->iteratorName;
+    if (ct == "IntegerLiteral")
+        return std::to_string(
+            static_cast<const IntegerLiteral*>(node)->value);
+    if (ct == "BooleanLiteral")
+        return static_cast<const BooleanLiteral*>(node)->value
+            ? "true" : "false";
+    return "";
+}
+
+// --- Compact AST serialization ---
+// Returns: {id, type, name, line, childCount, children: [child_ids]}
+// Uses short keys and omits empty fields for minimal token usage.
+inline json toJsonCompact(const ASTNode* node) {
+    if (!node) return json();
+    json j;
+    j["id"] = node->id;
+    j["type"] = node->conceptType;
+    std::string name = getNodeName(node);
+    if (!name.empty()) j["name"] = name;
+    if (node->hasSpan()) j["line"] = node->spanStartLine;
+
+    auto kids = node->allChildren();
+    if (!kids.empty()) {
+        j["childCount"] = (int)kids.size();
+        json childIds = json::array();
+        for (const auto* child : kids)
+            childIds.push_back(child->id);
+        j["children"] = childIds;
+    }
+
+    return j;
+}
+
+// Compact summary: top-level nodes only (Module + direct children).
+// Deeper nodes omitted — use getASTSubtree for detail.
+inline json toJsonCompactSummary(const ASTNode* root) {
+    if (!root) return json::array();
+    json nodes = json::array();
+    // Root node
+    json rootJ;
+    rootJ["id"] = root->id;
+    rootJ["type"] = root->conceptType;
+    std::string rname = getNodeName(root);
+    if (!rname.empty()) rootJ["name"] = rname;
+    if (root->hasSpan()) rootJ["line"] = root->spanStartLine;
+    auto rootKids = root->allChildren();
+    if (!rootKids.empty()) rootJ["childCount"] = (int)rootKids.size();
+    nodes.push_back(rootJ);
+    // Direct children (depth 1 only — functions, imports, etc.)
+    for (const auto* child : rootKids) {
+        json cj;
+        cj["id"] = child->id;
+        cj["type"] = child->conceptType;
+        std::string cname = getNodeName(child);
+        if (!cname.empty()) cj["name"] = cname;
+        if (child->hasSpan()) cj["line"] = child->spanStartLine;
+        auto grandkids = child->allChildren();
+        if (!grandkids.empty())
+            cj["childCount"] = (int)grandkids.size();
+        nodes.push_back(cj);
+    }
+    return nodes;
+}
+
+// Collect all nodes in compact format (flat list)
+inline json toJsonCompactTree(const ASTNode* node) {
+    if (!node) return json::array();
+    json nodes = json::array();
+    nodes.push_back(toJsonCompact(node));
+    for (const auto* child : node->allChildren()) {
+        json childNodes = toJsonCompactTree(child);
+        for (auto& cn : childNodes)
+            nodes.push_back(std::move(cn));
+    }
+    return nodes;
+}
+
+// --- Subtree extraction ---
+// Returns full JSON for the subtree rooted at nodeId
+inline json toJsonSubtree(ASTNode* root, const std::string& nodeId) {
+    ASTNode* target = findNodeById(root, nodeId);
+    if (!target) return json();
+    return toJson(target);
+}
+
+// --- Token estimate ---
+// Rough estimate: characters / 4 (approximates LLM tokens)
+inline int tokenEstimate(const json& j) {
+    std::string s = j.dump();
+    return (int)s.size() / 4;
+}
+
+// --- AST version tracking ---
+// Stored in HeadlessBufferState, records which node IDs changed per version.
+struct ASTVersionTracker {
+    int version = 0;
+    // version -> list of affected node IDs
+    std::map<int, std::vector<std::string>> changes;
+
+    void recordMutation(const std::vector<std::string>& affectedIds) {
+        ++version;
+        changes[version] = affectedIds;
+    }
+
+    // Get all node IDs that changed since a given version
+    std::vector<std::string> changedSince(int sinceVersion) const {
+        std::vector<std::string> result;
+        for (const auto& [v, ids] : changes) {
+            if (v > sinceVersion) {
+                for (const auto& id : ids)
+                    result.push_back(id);
+            }
+        }
+        return result;
+    }
+
+    // Build a diff response: full JSON for changed nodes only
+    json buildDiff(ASTNode* root, int sinceVersion) const {
+        auto changedIds = changedSince(sinceVersion);
+        json nodes = json::array();
+        for (const auto& id : changedIds) {
+            ASTNode* node = findNodeById(root, id);
+            if (node)
+                nodes.push_back(toJson(node));
+        }
+        return {
+            {"sinceVersion", sinceVersion},
+            {"currentVersion", version},
+            {"changedCount", (int)nodes.size()},
+            {"nodes", nodes}
+        };
+    }
+
+    // Prune old entries to prevent unbounded growth
+    void pruneOlderThan(int keepVersion) {
+        auto it = changes.begin();
+        while (it != changes.end() && it->first < keepVersion)
+            it = changes.erase(it);
+    }
+};

editor/src/HeadlessAgentRPCHandler.h

@@ -62,11 +62,26 @@ inline json handleHeadlessAgentRequest(HeadlessEditorState& state,
     if (method == "getAST") {
         auto err = headlessRequireAST(state, id);
         if (!err.is_null()) return err;
-        return headlessRpcResult(id, {
-            {"ast", toJson(state.activeAST())},
-            {"annotationCount", countAnnotationNodes(state.activeAST())},
-            {"diagnostics", state.buildDiagnosticsJson()}
-        });
+        auto params = request.contains("params") ? request["params"]
+                                                  : json::object();
+        bool compact = params.value("compact", false);
+        json result;
+        if (compact) {
+            json nodes = toJsonCompactSummary(state.activeAST());
+            result = {{"nodes", nodes}, {"nodeCount", (int)nodes.size()},
+                      {"totalNodes", (int)toJsonCompactTree(
+                          state.activeAST()).size()}};
+        } else {
+            result = {
+                {"ast", toJson(state.activeAST())},
+                {"annotationCount",
+                 countAnnotationNodes(state.activeAST())},
+                {"diagnostics", state.buildDiagnosticsJson()}
+            };
+        }
+        result["version"] = state.active()->versionTracker.version;
+        result["tokenEstimate"] = tokenEstimate(result);
+        return headlessRpcResult(id, result);
     }
 
     // --- parseSource ---
@@ -296,11 +311,14 @@ inline json handleHeadlessAgentRequest(HeadlessEditorState& state,
             state.active()->incrementalOptimizer.recordExternalTransform(
                 "agent-mutation:" + type, affectedIds,
                 state.agentActorLabel(sessionId));
+            state.active()->versionTracker.recordMutation(affectedIds);
         }
         return headlessRpcResult(id, {
             {"success", true}, {"warning", res.warning},
             {"libraryWarning", policy.warning},
-            {"unknownFunctions", policy.unknownFunctions}
+            {"unknownFunctions", policy.unknownFunctions},
+            {"version", state.active()
+                ? state.active()->versionTracker.version : 0}
         });
     }
 
@@ -349,9 +367,15 @@ inline json handleHeadlessAgentRequest(HeadlessEditorState& state,
             state.active()->incrementalOptimizer.recordExternalTransform(
                 "agent-batch", {},
                 state.agentActorLabel(sessionId));
+            std::vector<std::string> batchIds;
+            for (const auto& m : mutations)
+                if (!m.nodeId.empty()) batchIds.push_back(m.nodeId);
+            state.active()->versionTracker.recordMutation(batchIds);
         }
         return headlessRpcResult(id,
-            {{"success", true}, {"appliedCount", batchRes.appliedCount}});
+            {{"success", true}, {"appliedCount", batchRes.appliedCount},
+             {"version", state.active()
+                 ? state.active()->versionTracker.version : 0}});
     }
 
     // --- getInScopeSymbols ---
@@ -554,6 +578,39 @@ inline json handleHeadlessAgentRequest(HeadlessEditorState& state,
         });
     }
 
+    // --- getASTSubtree ---
+    if (method == "getASTSubtree") {
+        auto err = headlessRequireAST(state, id);
+        if (!err.is_null()) return err;
+        auto params = request.contains("params") ? request["params"]
+                                                  : json::object();
+        std::string nodeId = params.value("nodeId", "");
+        if (nodeId.empty())
+            return headlessRpcError(id, -32602,
+                                     "Missing nodeId parameter");
+        json subtree = toJsonSubtree(state.activeAST(), nodeId);
+        if (subtree.is_null())
+            return headlessRpcError(id, -32002,
+                "Node not found: " + nodeId);
+        json result = {{"subtree", subtree}};
+        result["version"] = state.active()->versionTracker.version;
+        result["tokenEstimate"] = tokenEstimate(result);
+        return headlessRpcResult(id, result);
+    }
+
+    // --- getASTDiff ---
+    if (method == "getASTDiff") {
+        auto err = headlessRequireAST(state, id);
+        if (!err.is_null()) return err;
+        auto params = request.contains("params") ? request["params"]
+                                                  : json::object();
+        int sinceVersion = params.value("sinceVersion", 0);
+        json diff = state.active()->versionTracker.buildDiff(
+            state.activeAST(), sinceVersion);
+        diff["tokenEstimate"] = tokenEstimate(diff);
+        return headlessRpcResult(id, diff);
+    }
+
     // --- fileRead ---
     if (method == "fileRead") {
         if (!AgentPermissionPolicy::canInvoke(role, method))

editor/src/HeadlessEditorState.h

@@ -23,6 +23,7 @@
 #include "AgentLibraryPolicy.h"
 #include "WorkflowRecorder.h"
 #include "FileOperations.h"
+#include "CompactAST.h"
 #include "Orchestrator.h"
 #include "IncrementalOptimizer.h"
 #include "TextASTSync.h"
@@ -64,6 +65,7 @@ struct HeadlessBufferState {
     TextASTSync          sync;
     Orchestrator         orchestrator;
     IncrementalOptimizer incrementalOptimizer;
+    ASTVersionTracker    versionTracker;
     std::string          language    = "python";
     std::string          path        = "(untitled)";
     std::string          editBuf;

editor/src/MCPServer.h

@@ -309,12 +309,15 @@ private:
     void registerASTTools() {
         // whetstone_get_ast
         tools_.push_back({"whetstone_get_ast",
-            "Get the current AST (Abstract Syntax Tree) of the active buffer as JSON. "
-            "Returns the full tree structure with all nodes, annotations, and metadata.",
-            {{"type", "object"}, {"properties", json::object()}}
+            "Get the current AST of the active buffer. Set compact=true for "
+            "a token-efficient flat list of {id, type, name, line, children}. "
+            "Full mode returns complete tree with properties and spans.",
+            {{"type", "object"}, {"properties", {
+                {"compact", {{"type", "boolean"}, {"description", "Compact mode: flat list with minimal fields (default false)"}}}
+            }}}
         });
-        toolHandlers_["whetstone_get_ast"] = [this](const json&) {
-            return callWhetstone("getAST");
+        toolHandlers_["whetstone_get_ast"] = [this](const json& args) {
+            return callWhetstone("getAST", args);
         };
 
         // whetstone_mutate
@@ -370,6 +373,30 @@ private:
         toolHandlers_["whetstone_get_call_hierarchy"] = [this](const json& args) {
             return callWhetstone("getCallHierarchy", args);
         };
+
+        // whetstone_get_ast_subtree
+        tools_.push_back({"whetstone_get_ast_subtree",
+            "Get only the subtree rooted at a specific node ID. Returns full "
+            "node detail for just that subtree, saving tokens vs full AST.",
+            {{"type", "object"}, {"properties", {
+                {"nodeId", {{"type", "string"}, {"description", "Root node ID for the subtree"}}}
+            }}, {"required", {"nodeId"}}}
+        });
+        toolHandlers_["whetstone_get_ast_subtree"] = [this](const json& args) {
+            return callWhetstone("getASTSubtree", args);
+        };
+
+        // whetstone_get_ast_diff
+        tools_.push_back({"whetstone_get_ast_diff",
+            "Get only the AST nodes that changed since a given version. "
+            "Use the version number from a previous getAST or mutation response.",
+            {{"type", "object"}, {"properties", {
+                {"sinceVersion", {{"type", "integer"}, {"description", "Version number to diff against (from previous response)"}}}
+            }}, {"required", {"sinceVersion"}}}
+        });
+        toolHandlers_["whetstone_get_ast_diff"] = [this](const json& args) {
+            return callWhetstone("getASTDiff", args);
+        };
     }
 
     // ---------------------------------------------------------------