Step 425: add batched agent task submission planning

editor/CMakeLists.txt

@@ -2704,4 +2704,13 @@ target_link_libraries(step424_test PRIVATE
     tree_sitter_javascript tree_sitter_typescript
     tree_sitter_java tree_sitter_rust tree_sitter_go)
 
+add_executable(step425_test tests/step425_test.cpp)
+target_include_directories(step425_test PRIVATE src)
+target_link_libraries(step425_test PRIVATE
+    nlohmann_json::nlohmann_json
+    unofficial::tree-sitter::tree-sitter
+    tree_sitter_python tree_sitter_cpp tree_sitter_elisp
+    tree_sitter_javascript tree_sitter_typescript
+    tree_sitter_java tree_sitter_rust tree_sitter_go)
+
 # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies)

editor/src/BatchTaskSubmitter.h

@@ -0,0 +1,123 @@
+#pragma once
+
+#include <algorithm>
+#include <map>
+#include <string>
+#include <vector>
+
+#include <nlohmann/json.hpp>
+using json = nlohmann::json;
+
+struct AgentTaskRequest {
+    std::string itemId;
+    std::string workerType;   // slm|llm
+    std::string language;
+    std::string bufferId;
+    std::string context;
+    std::string prompt;
+};
+
+struct AgentTaskBatch {
+    std::string workerType;
+    std::string language;
+    std::string bufferId;
+    std::string sharedContext;
+    std::vector<AgentTaskRequest> tasks;
+
+    json toJson() const {
+        json taskArr = json::array();
+        for (const auto& t : tasks) {
+            taskArr.push_back({
+                {"itemId", t.itemId},
+                {"prompt", t.prompt}
+            });
+        }
+        return {
+            {"workerType", workerType},
+            {"language", language},
+            {"bufferId", bufferId},
+            {"sharedContext", sharedContext},
+            {"tasks", taskArr}
+        };
+    }
+};
+
+struct BatchPlan {
+    std::vector<AgentTaskBatch> batches;
+    int estimatedTokensIndividual = 0;
+    int estimatedTokensBatched = 0;
+
+    double estimatedSavingsPercent() const {
+        if (estimatedTokensIndividual <= 0) return 0.0;
+        int saved = estimatedTokensIndividual - estimatedTokensBatched;
+        if (saved <= 0) return 0.0;
+        return (100.0 * (double)saved) / (double)estimatedTokensIndividual;
+    }
+};
+
+class BatchTaskSubmitter {
+public:
+    static BatchPlan planBatches(const std::vector<AgentTaskRequest>& tasks,
+                                 int maxBatchSize = 4) {
+        if (maxBatchSize < 1) maxBatchSize = 1;
+        BatchPlan plan;
+
+        std::map<std::string, std::vector<AgentTaskRequest>> groups;
+        for (const auto& t : tasks) {
+            std::string key = makeGroupKey(t);
+            groups[key].push_back(t);
+            plan.estimatedTokensIndividual += estimateTokens(t.context + "\n" + t.prompt);
+        }
+
+        for (auto& [_, group] : groups) {
+            // Stable order for deterministic payloads.
+            std::sort(group.begin(), group.end(),
+                      [](const AgentTaskRequest& a, const AgentTaskRequest& b) {
+                          return a.itemId < b.itemId;
+                      });
+
+            for (size_t i = 0; i < group.size(); i += (size_t)maxBatchSize) {
+                AgentTaskBatch batch;
+                batch.workerType = group[i].workerType;
+                batch.language = group[i].language;
+                batch.bufferId = group[i].bufferId;
+                batch.sharedContext = group[i].context;
+
+                size_t end = std::min(group.size(), i + (size_t)maxBatchSize);
+                for (size_t j = i; j < end; ++j) {
+                    batch.tasks.push_back(group[j]);
+                }
+                plan.estimatedTokensBatched += estimateBatchTokens(batch);
+                plan.batches.push_back(batch);
+            }
+        }
+
+        std::sort(plan.batches.begin(), plan.batches.end(),
+                  [](const AgentTaskBatch& a, const AgentTaskBatch& b) {
+                      if (a.workerType != b.workerType) return a.workerType < b.workerType;
+                      if (a.language != b.language) return a.language < b.language;
+                      return a.bufferId < b.bufferId;
+                  });
+        return plan;
+    }
+
+private:
+    static std::string makeGroupKey(const AgentTaskRequest& t) {
+        return t.workerType + "|" + t.language + "|" + t.bufferId;
+    }
+
+    static int estimateTokens(const std::string& text) {
+        if (text.empty()) return 0;
+        int chars = (int)text.size();
+        int est = chars / 4;
+        return est > 0 ? est : 1;
+    }
+
+    static int estimateBatchTokens(const AgentTaskBatch& batch) {
+        int total = estimateTokens(batch.sharedContext);
+        for (const auto& t : batch.tasks) {
+            total += estimateTokens(t.prompt);
+        }
+        return total;
+    }
+};

editor/tests/step425_test.cpp

@@ -0,0 +1,194 @@
+// Step 425: Batch Submission for Agent Tasks Tests (12 tests)
+
+#include "BatchTaskSubmitter.h"
+
+#include <iostream>
+#include <string>
+#include <vector>
+
+static int passed = 0, failed = 0;
+#define TEST(name) { std::cout << "  " << #name << "... "; }
+#define PASS() { std::cout << "PASS\n"; ++passed; }
+#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; }
+#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {}
+
+static AgentTaskRequest makeTask(const std::string& id,
+                                 const std::string& worker,
+                                 const std::string& lang,
+                                 const std::string& file,
+                                 const std::string& prompt) {
+    AgentTaskRequest t;
+    t.itemId = id;
+    t.workerType = worker;
+    t.language = lang;
+    t.bufferId = file;
+    t.context = "shared context for " + file;
+    t.prompt = prompt;
+    return t;
+}
+
+void test_groups_tasks_by_worker_language_buffer() {
+    TEST(groups_tasks_by_worker_language_buffer);
+    std::vector<AgentTaskRequest> tasks = {
+        makeTask("a1", "llm", "cpp", "a.cpp", "do A"),
+        makeTask("a2", "llm", "cpp", "a.cpp", "do B"),
+        makeTask("b1", "llm", "cpp", "b.cpp", "do C")
+    };
+    auto plan = BatchTaskSubmitter::planBatches(tasks, 8);
+    CHECK(plan.batches.size() == 2, "expected two groups");
+    PASS();
+}
+
+void test_separates_by_worker_type() {
+    TEST(separates_by_worker_type);
+    std::vector<AgentTaskRequest> tasks = {
+        makeTask("a1", "llm", "cpp", "a.cpp", "p1"),
+        makeTask("a2", "slm", "cpp", "a.cpp", "p2")
+    };
+    auto plan = BatchTaskSubmitter::planBatches(tasks, 8);
+    CHECK(plan.batches.size() == 2, "different workers should not mix");
+    PASS();
+}
+
+void test_separates_by_language() {
+    TEST(separates_by_language);
+    std::vector<AgentTaskRequest> tasks = {
+        makeTask("a1", "llm", "cpp", "a.cpp", "p1"),
+        makeTask("a2", "llm", "python", "a.py", "p2")
+    };
+    auto plan = BatchTaskSubmitter::planBatches(tasks, 8);
+    CHECK(plan.batches.size() == 2, "different languages should not mix");
+    PASS();
+}
+
+void test_honors_max_batch_size() {
+    TEST(honors_max_batch_size);
+    std::vector<AgentTaskRequest> tasks = {
+        makeTask("a1", "llm", "cpp", "a.cpp", "p1"),
+        makeTask("a2", "llm", "cpp", "a.cpp", "p2"),
+        makeTask("a3", "llm", "cpp", "a.cpp", "p3"),
+        makeTask("a4", "llm", "cpp", "a.cpp", "p4"),
+        makeTask("a5", "llm", "cpp", "a.cpp", "p5")
+    };
+    auto plan = BatchTaskSubmitter::planBatches(tasks, 2);
+    CHECK(plan.batches.size() == 3, "5 tasks with size 2 => 3 batches");
+    CHECK(plan.batches[0].tasks.size() <= 2, "batch 0 exceeds max");
+    PASS();
+}
+
+void test_batch_keeps_shared_context_once() {
+    TEST(batch_keeps_shared_context_once);
+    std::vector<AgentTaskRequest> tasks = {
+        makeTask("a1", "llm", "cpp", "a.cpp", "p1"),
+        makeTask("a2", "llm", "cpp", "a.cpp", "p2")
+    };
+    auto plan = BatchTaskSubmitter::planBatches(tasks, 8);
+    CHECK(plan.batches.size() == 1, "expected one batch");
+    CHECK(plan.batches[0].sharedContext.find("a.cpp") != std::string::npos,
+          "shared context not preserved");
+    PASS();
+}
+
+void test_batch_payload_json_contains_tasks() {
+    TEST(batch_payload_json_contains_tasks);
+    std::vector<AgentTaskRequest> tasks = {
+        makeTask("a1", "llm", "cpp", "a.cpp", "p1"),
+        makeTask("a2", "llm", "cpp", "a.cpp", "p2")
+    };
+    auto plan = BatchTaskSubmitter::planBatches(tasks, 8);
+    auto j = plan.batches[0].toJson();
+    CHECK(j.contains("tasks"), "missing tasks json");
+    CHECK((int)j["tasks"].size() == 2, "task count mismatch");
+    PASS();
+}
+
+void test_deterministic_item_order_in_batch() {
+    TEST(deterministic_item_order_in_batch);
+    std::vector<AgentTaskRequest> tasks = {
+        makeTask("z2", "llm", "cpp", "a.cpp", "p2"),
+        makeTask("a1", "llm", "cpp", "a.cpp", "p1")
+    };
+    auto plan = BatchTaskSubmitter::planBatches(tasks, 8);
+    CHECK(plan.batches[0].tasks[0].itemId == "a1", "tasks should be sorted by itemId");
+    PASS();
+}
+
+void test_empty_input_produces_empty_plan() {
+    TEST(empty_input_produces_empty_plan);
+    auto plan = BatchTaskSubmitter::planBatches({}, 4);
+    CHECK(plan.batches.empty(), "expected no batches");
+    CHECK(plan.estimatedTokensIndividual == 0, "expected zero individual tokens");
+    PASS();
+}
+
+void test_max_batch_size_floor_to_one() {
+    TEST(max_batch_size_floor_to_one);
+    std::vector<AgentTaskRequest> tasks = {
+        makeTask("a1", "llm", "cpp", "a.cpp", "p1"),
+        makeTask("a2", "llm", "cpp", "a.cpp", "p2")
+    };
+    auto plan = BatchTaskSubmitter::planBatches(tasks, 0);
+    CHECK(plan.batches.size() == 2, "size<=0 should behave as maxBatchSize=1");
+    PASS();
+}
+
+void test_estimated_tokens_batched_not_exceed_individual() {
+    TEST(estimated_tokens_batched_not_exceed_individual);
+    std::vector<AgentTaskRequest> tasks = {
+        makeTask("a1", "llm", "cpp", "a.cpp", "rewrite alpha"),
+        makeTask("a2", "llm", "cpp", "a.cpp", "rewrite beta"),
+        makeTask("a3", "llm", "cpp", "a.cpp", "rewrite gamma")
+    };
+    auto plan = BatchTaskSubmitter::planBatches(tasks, 8);
+    CHECK(plan.estimatedTokensBatched <= plan.estimatedTokensIndividual,
+          "batched should not exceed individual");
+    PASS();
+}
+
+void test_savings_percent_positive_for_shared_context_batch() {
+    TEST(savings_percent_positive_for_shared_context_batch);
+    std::vector<AgentTaskRequest> tasks = {
+        makeTask("a1", "llm", "cpp", "a.cpp", "rewrite alpha"),
+        makeTask("a2", "llm", "cpp", "a.cpp", "rewrite beta"),
+        makeTask("a3", "llm", "cpp", "a.cpp", "rewrite gamma")
+    };
+    auto plan = BatchTaskSubmitter::planBatches(tasks, 8);
+    CHECK(plan.estimatedSavingsPercent() > 0.0, "expected positive savings");
+    PASS();
+}
+
+void test_multiple_groups_compute_savings_without_crash() {
+    TEST(multiple_groups_compute_savings_without_crash);
+    std::vector<AgentTaskRequest> tasks = {
+        makeTask("a1", "llm", "cpp", "a.cpp", "p1"),
+        makeTask("a2", "llm", "cpp", "a.cpp", "p2"),
+        makeTask("b1", "llm", "python", "b.py", "p3"),
+        makeTask("c1", "slm", "cpp", "c.cpp", "p4")
+    };
+    auto plan = BatchTaskSubmitter::planBatches(tasks, 8);
+    CHECK(plan.batches.size() >= 3, "expected multiple groups");
+    CHECK(plan.estimatedTokensIndividual >= plan.estimatedTokensBatched,
+          "individual should be >= batched");
+    PASS();
+}
+
+int main() {
+    std::cout << "Step 425: Batch Submission for Agent Tasks Tests\n";
+
+    test_groups_tasks_by_worker_language_buffer();            // 1
+    test_separates_by_worker_type();                          // 2
+    test_separates_by_language();                             // 3
+    test_honors_max_batch_size();                             // 4
+    test_batch_keeps_shared_context_once();                   // 5
+    test_batch_payload_json_contains_tasks();                 // 6
+    test_deterministic_item_order_in_batch();                 // 7
+    test_empty_input_produces_empty_plan();                   // 8
+    test_max_batch_size_floor_to_one();                       // 9
+    test_estimated_tokens_batched_not_exceed_individual();    // 10
+    test_savings_percent_positive_for_shared_context_batch(); // 11
+    test_multiple_groups_compute_savings_without_crash();     // 12
+
+    std::cout << "\nResults: " << passed << "/" << (passed + failed)
+              << " passed\n";
+    return failed == 0 ? 0 : 1;
+}

progress.md

@@ -4614,6 +4614,50 @@ with deterministic truncation behavior.
   - `editor/src/MCPServer.h` (`1940` > `600`)
   - `editor/src/HeadlessAgentRPCHandler.h` (`2768` > `600`)
 
+### Step 425: Batch Submission for Agent Tasks
+**Status:** PASS (12/12 tests)
+
+Added batching utilities for external agent tasks to reduce duplicated context
+tokens by grouping requests with shared worker/language/file context.
+
+**Files created:**
+- `editor/src/BatchTaskSubmitter.h` — task batching support:
+  - `AgentTaskRequest` / `AgentTaskBatch` / `BatchPlan` types
+  - deterministic grouping key (`workerType|language|bufferId`)
+  - stable item ordering within batches
+  - max batch size splitting
+  - token-cost estimation for individual vs batched plans
+  - estimated savings percentage reporting
+- `editor/tests/step425_test.cpp` — 12 tests covering:
+  1. grouping by worker/language/buffer
+  2. worker-type separation
+  3. language separation
+  4. max batch size enforcement
+  5. shared-context carryover in batch payload
+  6. batch JSON task payload shape
+  7. deterministic in-batch ordering
+  8. empty-input edge case
+  9. max-size floor behavior
+  10. batched token estimate <= individual
+  11. positive savings for shared-context groups
+  12. multi-group savings computation stability
+
+**Files modified:**
+- `editor/CMakeLists.txt` — `step425_test` target
+
+**Verification run:**
+- `step425_test` — PASS (12/12) new step coverage
+- `step424_test` — PASS (12/12) regression coverage
+- `step423_test` — PASS (12/12) regression coverage
+
+**Architecture gate check:**
+- `editor/src/BatchTaskSubmitter.h` within header-size limit (`123` <= `600`)
+- `editor/tests/step425_test.cpp` within test-file size guidance (`194` lines)
+- Legacy oversized headers persist:
+  - `editor/src/ast/Serialization.h` (`1427` > `600`)
+  - `editor/src/MCPServer.h` (`1940` > `600`)
+  - `editor/src/HeadlessAgentRPCHandler.h` (`2768` > `600`)
+
 # Roadmap Planning — Sprints 12-25+
 
 ## Status: Planning Complete (Sprints 12-19 detailed, 20-25 in roadmap.md)