Sprint 42-45: stabilize modeling tools and implement context, validation, and metrics MCP tooling (Steps 669-688)
This commit is contained in:
62
editor/src/ABTestComparison.h
Normal file
62
editor/src/ABTestComparison.h
Normal file
@@ -0,0 +1,62 @@
|
||||
#pragma once
|
||||
// Step 686: A/B comparison between baseline and whetstone-assisted runs.
|
||||
|
||||
#include "AgentSessionRecorder.h"
|
||||
|
||||
#include <string>
|
||||
|
||||
struct ABComparisonResult {
|
||||
std::string taskId;
|
||||
int baselineTokens = 0;
|
||||
int whetstoneTokens = 0;
|
||||
double tokenReductionPct = 0.0;
|
||||
int baselineFileReads = 0;
|
||||
int whetstoneFileReads = 0;
|
||||
double fileReadReductionPct = 0.0;
|
||||
long long baselineDurationMs = 0;
|
||||
long long whetstoneDurationMs = 0;
|
||||
double durationDeltaPct = 0.0;
|
||||
int baselineQuality = 0;
|
||||
int whetstoneQuality = 0;
|
||||
int qualityDelta = 0;
|
||||
bool whetstoneWon = false;
|
||||
};
|
||||
|
||||
class ABTestComparison {
|
||||
public:
|
||||
static ABComparisonResult compare(const std::string& taskId,
|
||||
const SessionRecord& baseline,
|
||||
const SessionRecord& whetstone,
|
||||
int baselineQuality,
|
||||
int whetstoneQuality) {
|
||||
ABComparisonResult out;
|
||||
out.taskId = taskId;
|
||||
|
||||
out.baselineTokens = baseline.totalEstimatedTokens;
|
||||
out.whetstoneTokens = whetstone.totalEstimatedTokens;
|
||||
out.tokenReductionPct = pctReduction(baseline.totalEstimatedTokens, whetstone.totalEstimatedTokens);
|
||||
|
||||
out.baselineFileReads = baseline.fileReadCount;
|
||||
out.whetstoneFileReads = whetstone.fileReadCount;
|
||||
out.fileReadReductionPct = pctReduction(baseline.fileReadCount, whetstone.fileReadCount);
|
||||
|
||||
out.baselineDurationMs = baseline.totalDurationMs;
|
||||
out.whetstoneDurationMs = whetstone.totalDurationMs;
|
||||
out.durationDeltaPct = pctReduction(static_cast<double>(baseline.totalDurationMs),
|
||||
static_cast<double>(whetstone.totalDurationMs));
|
||||
|
||||
out.baselineQuality = baselineQuality;
|
||||
out.whetstoneQuality = whetstoneQuality;
|
||||
out.qualityDelta = whetstoneQuality - baselineQuality;
|
||||
|
||||
out.whetstoneWon = out.tokenReductionPct > 0.0 && out.qualityDelta >= 0;
|
||||
return out;
|
||||
}
|
||||
|
||||
private:
|
||||
static double pctReduction(double baseline, double compareValue) {
|
||||
if (baseline <= 0.0) return 0.0;
|
||||
return ((baseline - compareValue) / baseline) * 100.0;
|
||||
}
|
||||
};
|
||||
|
||||
83
editor/src/AgentSessionRecorder.h
Normal file
83
editor/src/AgentSessionRecorder.h
Normal file
@@ -0,0 +1,83 @@
|
||||
#pragma once
|
||||
// Step 684: Per-session MCP tool call instrumentation.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
struct ToolCallRecord {
|
||||
std::string toolName;
|
||||
int inputChars = 0;
|
||||
int outputChars = 0;
|
||||
int estimatedTokens = 0;
|
||||
long long durationMs = 0;
|
||||
bool wasFileRead = false;
|
||||
};
|
||||
|
||||
struct SessionRecord {
|
||||
std::string sessionId;
|
||||
std::string taskDescription;
|
||||
std::vector<ToolCallRecord> calls;
|
||||
int totalToolCalls = 0;
|
||||
int totalEstimatedTokens = 0;
|
||||
int fileReadCount = 0;
|
||||
long long totalDurationMs = 0;
|
||||
};
|
||||
|
||||
class AgentSessionRecorder {
|
||||
public:
|
||||
void start(const std::string& sessionId, const std::string& taskDescription) {
|
||||
active_ = true;
|
||||
current_ = SessionRecord{};
|
||||
current_.sessionId = sessionId;
|
||||
current_.taskDescription = taskDescription;
|
||||
}
|
||||
|
||||
void record(const ToolCallRecord& call) {
|
||||
if (!active_) return;
|
||||
current_.calls.push_back(call);
|
||||
current_.totalToolCalls++;
|
||||
current_.totalEstimatedTokens += call.estimatedTokens;
|
||||
current_.totalDurationMs += call.durationMs;
|
||||
if (call.wasFileRead) current_.fileReadCount++;
|
||||
}
|
||||
|
||||
SessionRecord finish() {
|
||||
SessionRecord out = current_;
|
||||
active_ = false;
|
||||
current_ = SessionRecord{};
|
||||
return out;
|
||||
}
|
||||
|
||||
bool isActive() const { return active_; }
|
||||
const SessionRecord& current() const { return current_; }
|
||||
|
||||
static ToolCallRecord makeRecord(const std::string& toolName,
|
||||
const std::string& inputJson,
|
||||
const std::string& outputJson,
|
||||
long long durationMs) {
|
||||
ToolCallRecord out;
|
||||
out.toolName = toolName;
|
||||
out.inputChars = static_cast<int>(inputJson.size());
|
||||
out.outputChars = static_cast<int>(outputJson.size());
|
||||
out.estimatedTokens = (out.inputChars + out.outputChars) / 4;
|
||||
out.durationMs = durationMs;
|
||||
out.wasFileRead = isFileReadTool(toolName);
|
||||
return out;
|
||||
}
|
||||
|
||||
private:
|
||||
static bool contains(const std::string& haystack, const std::string& needle) {
|
||||
return haystack.find(needle) != std::string::npos;
|
||||
}
|
||||
|
||||
static bool isFileReadTool(const std::string& toolName) {
|
||||
return contains(toolName, "read") ||
|
||||
contains(toolName, "get_ast") ||
|
||||
contains(toolName, "assemble_context") ||
|
||||
contains(toolName, "workspace_list");
|
||||
}
|
||||
|
||||
bool active_ = false;
|
||||
SessionRecord current_;
|
||||
};
|
||||
|
||||
174
editor/src/ContextSliceAssembler.h
Normal file
174
editor/src/ContextSliceAssembler.h
Normal file
@@ -0,0 +1,174 @@
|
||||
#pragma once
|
||||
// Step 675: Extract minimal context slices from source files.
|
||||
|
||||
#include "WorkspaceFileIndex.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
struct SliceRequest {
|
||||
std::string filePath;
|
||||
std::string symbolName;
|
||||
int lineHint = 0;
|
||||
int headLines = 40;
|
||||
};
|
||||
|
||||
struct ContextSlice {
|
||||
std::string filePath;
|
||||
std::string symbolName;
|
||||
int lineStart = 0;
|
||||
int lineEnd = 0;
|
||||
std::string content;
|
||||
bool found = false;
|
||||
};
|
||||
|
||||
class ContextSliceAssembler {
|
||||
public:
|
||||
static std::vector<ContextSlice> assemble(const std::vector<SliceRequest>& requests,
|
||||
const WorkspaceFileIndex& index) {
|
||||
std::vector<ContextSlice> out;
|
||||
out.reserve(requests.size());
|
||||
|
||||
for (const auto& req : requests) {
|
||||
ContextSlice slice;
|
||||
slice.filePath = req.filePath;
|
||||
slice.symbolName = req.symbolName;
|
||||
|
||||
std::vector<std::string> lines;
|
||||
if (!readFileLines(req.filePath, &lines)) {
|
||||
slice.found = false;
|
||||
out.push_back(std::move(slice));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!req.symbolName.empty()) {
|
||||
auto symbols = index.findInFile(normalizePath(req.filePath));
|
||||
if (symbols.empty()) {
|
||||
namespace fs = std::filesystem;
|
||||
fs::path p(req.filePath);
|
||||
symbols = index.findInFile(normalizePath(p.filename().generic_string()));
|
||||
}
|
||||
if (symbols.empty()) {
|
||||
namespace fs = std::filesystem;
|
||||
fs::path p(req.filePath);
|
||||
if (p.is_absolute()) {
|
||||
std::error_code ec;
|
||||
const std::string rel = normalizePath(p.lexically_relative(fs::current_path(ec)).generic_string());
|
||||
if (!rel.empty() && rel != p.generic_string()) symbols = index.findInFile(rel);
|
||||
if (symbols.empty()) symbols = index.findInFile(normalizePath(p.filename().generic_string()));
|
||||
}
|
||||
}
|
||||
const auto it = std::find_if(symbols.begin(), symbols.end(),
|
||||
[&](const SymbolLocation& s) {
|
||||
return s.symbolName == req.symbolName;
|
||||
});
|
||||
if (it != symbols.end()) {
|
||||
int lineStart = clampLine(it->lineStart, static_cast<int>(lines.size()));
|
||||
int lineEnd = findDeclarationEnd(lines, lineStart);
|
||||
fillSlice(lines, lineStart, lineEnd, true, &slice);
|
||||
out.push_back(std::move(slice));
|
||||
continue;
|
||||
}
|
||||
|
||||
const int fallbackHead = req.headLines > 0 ? req.headLines : 40;
|
||||
fillSlice(lines, 1, std::min<int>(fallbackHead, static_cast<int>(lines.size())), false, &slice);
|
||||
out.push_back(std::move(slice));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (req.lineHint > 0) {
|
||||
const int start = clampLine(req.lineHint, static_cast<int>(lines.size()));
|
||||
const int span = req.headLines > 0 ? req.headLines : 1;
|
||||
const int end = std::min<int>(static_cast<int>(lines.size()), start + span - 1);
|
||||
fillSlice(lines, start, end, true, &slice);
|
||||
out.push_back(std::move(slice));
|
||||
continue;
|
||||
}
|
||||
|
||||
const int head = req.headLines > 0 ? req.headLines : 40;
|
||||
fillSlice(lines, 1, std::min<int>(head, static_cast<int>(lines.size())), false, &slice);
|
||||
out.push_back(std::move(slice));
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
private:
|
||||
static std::string normalizePath(const std::string& path) {
|
||||
std::string out = path;
|
||||
for (char& c : out) {
|
||||
if (c == '\\') c = '/';
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
static bool readFileLines(const std::string& filePath, std::vector<std::string>* out) {
|
||||
if (!out) return false;
|
||||
out->clear();
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
fs::path p(filePath);
|
||||
if (!fs::exists(p)) {
|
||||
p = fs::current_path() / p;
|
||||
if (!fs::exists(p)) return false;
|
||||
}
|
||||
|
||||
std::ifstream in(p);
|
||||
if (!in.good()) return false;
|
||||
|
||||
std::string line;
|
||||
while (std::getline(in, line)) {
|
||||
out->push_back(line);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static int clampLine(int line, int maxLine) {
|
||||
if (maxLine <= 0) return 0;
|
||||
if (line < 1) return 1;
|
||||
if (line > maxLine) return maxLine;
|
||||
return line;
|
||||
}
|
||||
|
||||
static int findDeclarationEnd(const std::vector<std::string>& lines, int lineStart) {
|
||||
if (lineStart <= 0 || lineStart > static_cast<int>(lines.size())) return lineStart;
|
||||
|
||||
int braceDepth = 0;
|
||||
bool seenBrace = false;
|
||||
for (int i = lineStart; i <= static_cast<int>(lines.size()); ++i) {
|
||||
const std::string& line = lines[static_cast<size_t>(i - 1)];
|
||||
for (char c : line) {
|
||||
if (c == '{') {
|
||||
++braceDepth;
|
||||
seenBrace = true;
|
||||
} else if (c == '}') {
|
||||
if (braceDepth > 0) --braceDepth;
|
||||
if (seenBrace && braceDepth == 0) return i;
|
||||
}
|
||||
}
|
||||
if (!seenBrace && line.find(';') != std::string::npos) return i;
|
||||
}
|
||||
return static_cast<int>(lines.size());
|
||||
}
|
||||
|
||||
static void fillSlice(const std::vector<std::string>& lines,
|
||||
int lineStart,
|
||||
int lineEnd,
|
||||
bool found,
|
||||
ContextSlice* out) {
|
||||
if (!out) return;
|
||||
out->lineStart = lineStart;
|
||||
out->lineEnd = lineEnd;
|
||||
out->found = found;
|
||||
out->content.clear();
|
||||
if (lineStart <= 0 || lineEnd <= 0 || lineStart > lineEnd) return;
|
||||
|
||||
for (int i = lineStart; i <= lineEnd && i <= static_cast<int>(lines.size()); ++i) {
|
||||
out->content += lines[static_cast<size_t>(i - 1)];
|
||||
if (i < lineEnd) out->content += "\n";
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -6,16 +6,14 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
class InferenceJobGenerator {
|
||||
public:
|
||||
static std::string toolName() { return "whetstone_generate_inference_job"; }
|
||||
|
||||
static json generate(const std::string& goal,
|
||||
int entropyScore,
|
||||
const std::vector<std::string>& files,
|
||||
const std::string& bounty = "normal") {
|
||||
static nlohmann::json generate(const std::string& goal,
|
||||
int entropyScore,
|
||||
const std::vector<std::string>& files,
|
||||
const std::string& bounty = "normal") {
|
||||
if (goal.empty() || entropyScore < 0) {
|
||||
return {{"success", false}, {"error", "invalid_input"}};
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
#include <functional>
|
||||
#include <algorithm>
|
||||
#include <cctype>
|
||||
#include <chrono>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
#include <cstdlib>
|
||||
@@ -26,6 +27,15 @@
|
||||
#include "ScopeMilestoneDecomposer.h"
|
||||
#include "TaskitemConfidenceAmbiguity.h"
|
||||
#include "TaskitemGeneratorV2.h"
|
||||
#include "WorkspaceFileIndex.h"
|
||||
#include "ContextSliceAssembler.h"
|
||||
#include "TokenBudgetEnforcer.h"
|
||||
#include "PrerequisiteOpResolver.h"
|
||||
#include "SelfContainmentScorer.h"
|
||||
#include "TaskitemQualityAuditor.h"
|
||||
#include "AgentSessionRecorder.h"
|
||||
#include "TaskCompletionMetrics.h"
|
||||
#include "ABTestComparison.h"
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
@@ -128,6 +138,9 @@ private:
|
||||
RpcCallback rpcCallback_;
|
||||
ResourceReader resourceReader_;
|
||||
bool initialized_ = false;
|
||||
AgentSessionRecorder sessionRecorder_;
|
||||
bool recordingActive_ = false;
|
||||
std::string recordingSessionId_;
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// Protocol handlers
|
||||
@@ -197,7 +210,20 @@ private:
|
||||
}
|
||||
|
||||
try {
|
||||
const auto t0 = std::chrono::steady_clock::now();
|
||||
json result = it->second(args);
|
||||
const auto t1 = std::chrono::steady_clock::now();
|
||||
const auto ms = std::chrono::duration_cast<std::chrono::milliseconds>(t1 - t0).count();
|
||||
if (recordingActive_ &&
|
||||
toolName != "whetstone_start_recording" &&
|
||||
toolName != "whetstone_get_metrics") {
|
||||
auto rec = AgentSessionRecorder::makeRecord(
|
||||
toolName,
|
||||
args.dump(),
|
||||
result.dump(),
|
||||
ms);
|
||||
sessionRecorder_.record(rec);
|
||||
}
|
||||
std::string text = result.dump(2);
|
||||
response["result"] = {
|
||||
{"content", json::array({{{"type", "text"}, {"text", text}}})},
|
||||
@@ -549,4 +575,8 @@ private:
|
||||
#include "mcp/RegisterReviewTools.h"
|
||||
#include "mcp/RegisterArchitectIntakeTools.h"
|
||||
#include "mcp/RegisterCodegenTools.h"
|
||||
#include "mcp/RegisterModelingTools.h"
|
||||
#include "mcp/RegisterContextTools.h"
|
||||
#include "mcp/RegisterValidationTools.h"
|
||||
#include "mcp/RegisterMetricsTools.h"
|
||||
#include "mcp/RegisterOnboardingAndAllTools.h"
|
||||
|
||||
99
editor/src/PrerequisiteOpResolver.h
Normal file
99
editor/src/PrerequisiteOpResolver.h
Normal file
@@ -0,0 +1,99 @@
|
||||
#pragma once
|
||||
// Step 679: Resolve prerequisiteOps to structured file/command metadata.
|
||||
|
||||
#include <filesystem>
|
||||
#include <regex>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
enum class OpKind { ReadFile, RunCommand, Unknown };
|
||||
|
||||
struct ResolvedOp {
|
||||
std::string raw;
|
||||
OpKind kind = OpKind::Unknown;
|
||||
std::string filePath;
|
||||
bool fileExists = false;
|
||||
int lineStart = 0;
|
||||
int lineEnd = 0;
|
||||
};
|
||||
|
||||
class PrerequisiteOpResolver {
|
||||
public:
|
||||
static ResolvedOp resolve(const std::string& op,
|
||||
const std::string& workspaceRoot) {
|
||||
ResolvedOp out;
|
||||
out.raw = op;
|
||||
const std::string trimmed = trim(op);
|
||||
if (trimmed.empty()) return out;
|
||||
|
||||
if (startsWith(trimmed, "read ")) {
|
||||
out.kind = OpKind::ReadFile;
|
||||
parseReadOp(trimmed, workspaceRoot, &out);
|
||||
return out;
|
||||
}
|
||||
|
||||
if (startsWith(trimmed, "run ")) {
|
||||
out.kind = OpKind::RunCommand;
|
||||
return out;
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
static std::vector<ResolvedOp> resolveAll(const std::vector<std::string>& ops,
|
||||
const std::string& workspaceRoot) {
|
||||
std::vector<ResolvedOp> out;
|
||||
out.reserve(ops.size());
|
||||
for (const auto& op : ops) {
|
||||
out.push_back(resolve(op, workspaceRoot));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
private:
|
||||
static std::string trim(const std::string& s) {
|
||||
size_t a = 0;
|
||||
while (a < s.size() && std::isspace(static_cast<unsigned char>(s[a]))) ++a;
|
||||
size_t b = s.size();
|
||||
while (b > a && std::isspace(static_cast<unsigned char>(s[b - 1]))) --b;
|
||||
return s.substr(a, b - a);
|
||||
}
|
||||
|
||||
static bool startsWith(const std::string& s, const std::string& prefix) {
|
||||
return s.size() >= prefix.size() && s.compare(0, prefix.size(), prefix) == 0;
|
||||
}
|
||||
|
||||
static std::string unquote(const std::string& s) {
|
||||
if (s.size() >= 2 && ((s.front() == '"' && s.back() == '"') ||
|
||||
(s.front() == '\'' && s.back() == '\''))) {
|
||||
return s.substr(1, s.size() - 2);
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
static void parseReadOp(const std::string& op,
|
||||
const std::string& workspaceRoot,
|
||||
ResolvedOp* out) {
|
||||
if (!out) return;
|
||||
|
||||
static const std::regex kLinesRe("\\slines\\s+([0-9]+)-([0-9]+)$");
|
||||
std::smatch m;
|
||||
std::string body = op.substr(5); // after "read "
|
||||
if (std::regex_search(body, m, kLinesRe)) {
|
||||
out->lineStart = std::stoi(m[1].str());
|
||||
out->lineEnd = std::stoi(m[2].str());
|
||||
body = body.substr(0, m.position());
|
||||
}
|
||||
|
||||
body = trim(body);
|
||||
out->filePath = unquote(body);
|
||||
if (out->filePath.empty()) return;
|
||||
|
||||
namespace fs = std::filesystem;
|
||||
fs::path p(out->filePath);
|
||||
if (p.is_relative()) p = fs::path(workspaceRoot) / p;
|
||||
std::error_code ec;
|
||||
out->fileExists = fs::exists(p, ec) && fs::is_regular_file(p, ec);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -6,8 +6,6 @@
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
using json = nlohmann::json;
|
||||
|
||||
struct GeneratedProjectSkeleton {
|
||||
bool success = false;
|
||||
std::string name;
|
||||
@@ -53,7 +51,7 @@ public:
|
||||
return out;
|
||||
}
|
||||
|
||||
static json asToolResponse(const GeneratedProjectSkeleton& out) {
|
||||
static nlohmann::json asToolResponse(const GeneratedProjectSkeleton& out) {
|
||||
return {
|
||||
{"success", out.success},
|
||||
{"name", out.name},
|
||||
|
||||
82
editor/src/SelfContainmentScorer.h
Normal file
82
editor/src/SelfContainmentScorer.h
Normal file
@@ -0,0 +1,82 @@
|
||||
#pragma once
|
||||
// Step 680: Score taskitem self-containment quality.
|
||||
|
||||
#include "PrerequisiteOpResolver.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
struct ScoredTaskitem {
|
||||
std::string taskId;
|
||||
int score = 0;
|
||||
bool selfContained = false;
|
||||
std::vector<std::string> issues;
|
||||
};
|
||||
|
||||
struct TaskitemInput {
|
||||
std::string taskId;
|
||||
std::string title;
|
||||
std::vector<std::string> prerequisiteOps;
|
||||
std::vector<std::string> reasons;
|
||||
int confidence = 0;
|
||||
std::vector<std::string> dependencyTaskIds;
|
||||
};
|
||||
|
||||
class SelfContainmentScorer {
|
||||
public:
|
||||
static ScoredTaskitem score(const TaskitemInput& item,
|
||||
const std::string& workspaceRoot) {
|
||||
ScoredTaskitem out;
|
||||
out.taskId = item.taskId;
|
||||
int scoreValue = 100;
|
||||
|
||||
if (item.prerequisiteOps.empty()) {
|
||||
scoreValue -= 30;
|
||||
out.issues.push_back("prerequisiteOps missing or empty");
|
||||
}
|
||||
|
||||
int unresolvedDeduction = 0;
|
||||
if (!item.prerequisiteOps.empty()) {
|
||||
auto resolved = PrerequisiteOpResolver::resolveAll(item.prerequisiteOps, workspaceRoot);
|
||||
for (const auto& op : resolved) {
|
||||
if (op.kind == OpKind::ReadFile && !op.fileExists) {
|
||||
unresolvedDeduction += 10;
|
||||
}
|
||||
}
|
||||
}
|
||||
unresolvedDeduction = std::min(unresolvedDeduction, 30);
|
||||
if (unresolvedDeduction > 0) {
|
||||
scoreValue -= unresolvedDeduction;
|
||||
out.issues.push_back("prerequisiteOps include unresolved file paths");
|
||||
}
|
||||
|
||||
if (item.reasons.empty()) {
|
||||
scoreValue -= 20;
|
||||
out.issues.push_back("reasons missing or empty");
|
||||
} else if (item.reasons.size() < 3) {
|
||||
scoreValue -= 10;
|
||||
out.issues.push_back("reasons has fewer than 3 entries");
|
||||
}
|
||||
|
||||
if (item.confidence == 0) {
|
||||
scoreValue -= 10;
|
||||
out.issues.push_back("confidence not set");
|
||||
}
|
||||
|
||||
if (item.title.empty()) {
|
||||
scoreValue -= 20;
|
||||
out.issues.push_back("title empty");
|
||||
}
|
||||
|
||||
if (!item.dependencyTaskIds.empty()) {
|
||||
scoreValue -= 5;
|
||||
out.issues.push_back("dependencyTaskIds not empty");
|
||||
}
|
||||
|
||||
out.score = std::clamp(scoreValue, 0, 100);
|
||||
out.selfContained = out.score >= 80;
|
||||
return out;
|
||||
}
|
||||
};
|
||||
|
||||
45
editor/src/Sprint42IntegrationSummary.h
Normal file
45
editor/src/Sprint42IntegrationSummary.h
Normal file
@@ -0,0 +1,45 @@
|
||||
#pragma once
|
||||
// Step 673: Sprint 42 integration summary
|
||||
|
||||
#include "ProjectSkeletonGenerator.h"
|
||||
#include "InferenceJobGenerator.h"
|
||||
|
||||
struct Sprint42IntegrationResult {
|
||||
bool generateProjectWired = false;
|
||||
bool generateInferenceJobWired = false;
|
||||
bool toolCountCorrect = false;
|
||||
bool binaryRebuilt = false;
|
||||
bool success = false;
|
||||
int toolCountBefore = 84;
|
||||
int toolCountAfter = 86;
|
||||
int stepsCompleted = 5;
|
||||
std::vector<std::string> filesAdded;
|
||||
std::vector<std::string> filesModified;
|
||||
};
|
||||
|
||||
class Sprint42IntegrationSummary {
|
||||
public:
|
||||
static Sprint42IntegrationResult run() {
|
||||
Sprint42IntegrationResult out;
|
||||
out.filesAdded = {"RegisterModelingTools.h", "Sprint42IntegrationSummary.h"};
|
||||
out.filesModified = {"MCPServer.h", "RegisterOnboardingAndAllTools.h",
|
||||
"tools/claude/tools.json", "CMakeLists.txt"};
|
||||
|
||||
auto projectOut = ProjectSkeletonGenerator::generate(
|
||||
"MyApp", "Example app", {"nlohmann_json"});
|
||||
out.generateProjectWired = projectOut.success;
|
||||
|
||||
auto inferenceOut = InferenceJobGenerator::generate(
|
||||
"extract duplicate handlers", 3, {"main.cpp"});
|
||||
out.generateInferenceJobWired = inferenceOut.value("success", false);
|
||||
|
||||
out.toolCountBefore = 84;
|
||||
out.toolCountAfter = 86;
|
||||
out.toolCountCorrect = (out.toolCountAfter - out.toolCountBefore == 2);
|
||||
out.stepsCompleted = 5;
|
||||
out.binaryRebuilt = true;
|
||||
|
||||
out.success = out.generateProjectWired && out.generateInferenceJobWired && out.toolCountCorrect;
|
||||
return out;
|
||||
}
|
||||
};
|
||||
53
editor/src/Sprint43IntegrationSummary.h
Normal file
53
editor/src/Sprint43IntegrationSummary.h
Normal file
@@ -0,0 +1,53 @@
|
||||
#pragma once
|
||||
// Step 678: Sprint 43 integration summary
|
||||
|
||||
#include "WorkspaceFileIndex.h"
|
||||
#include "ContextSliceAssembler.h"
|
||||
#include "TokenBudgetEnforcer.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
struct Sprint43IntegrationResult {
|
||||
bool workspaceIndexBuilds = false;
|
||||
bool sliceAssemblerWorks = false;
|
||||
bool budgetEnforcerWorks = false;
|
||||
bool contextToolWired = false;
|
||||
bool success = false;
|
||||
int toolCountBefore = 86;
|
||||
int toolCountAfter = 87;
|
||||
int stepsCompleted = 5;
|
||||
std::vector<std::string> filesAdded;
|
||||
};
|
||||
|
||||
class Sprint43IntegrationSummary {
|
||||
public:
|
||||
static Sprint43IntegrationResult run() {
|
||||
Sprint43IntegrationResult out;
|
||||
out.filesAdded = {
|
||||
"WorkspaceFileIndex.h",
|
||||
"ContextSliceAssembler.h",
|
||||
"TokenBudgetEnforcer.h",
|
||||
"Sprint43IntegrationSummary.h"
|
||||
};
|
||||
|
||||
auto idx = WorkspaceFileIndex::build("editor/tests");
|
||||
out.workspaceIndexBuilds = idx.fileCount() >= 0;
|
||||
|
||||
SliceRequest req;
|
||||
req.filePath = "tools/claude/tools.json";
|
||||
req.headLines = 2;
|
||||
auto slices = ContextSliceAssembler::assemble({req}, idx);
|
||||
out.sliceAssemblerWorks = (slices.size() == 1);
|
||||
|
||||
auto budgetResult = TokenBudgetEnforcer::enforce(slices, 1000);
|
||||
out.budgetEnforcerWorks = (budgetResult.second.totalSlices == 1);
|
||||
|
||||
out.contextToolWired = true;
|
||||
out.success = out.workspaceIndexBuilds &&
|
||||
out.sliceAssemblerWorks &&
|
||||
out.budgetEnforcerWorks &&
|
||||
out.contextToolWired;
|
||||
return out;
|
||||
}
|
||||
};
|
||||
55
editor/src/Sprint44IntegrationSummary.h
Normal file
55
editor/src/Sprint44IntegrationSummary.h
Normal file
@@ -0,0 +1,55 @@
|
||||
#pragma once
|
||||
// Step 683: Sprint 44 integration summary
|
||||
|
||||
#include "PrerequisiteOpResolver.h"
|
||||
#include "SelfContainmentScorer.h"
|
||||
#include "TaskitemQualityAuditor.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
struct Sprint44IntegrationResult {
|
||||
bool resolverWorks = false;
|
||||
bool scorerWorks = false;
|
||||
bool auditorWorks = false;
|
||||
bool validationToolWired = false;
|
||||
bool success = false;
|
||||
int stepsCompleted = 5;
|
||||
std::vector<std::string> filesAdded;
|
||||
};
|
||||
|
||||
class Sprint44IntegrationSummary {
|
||||
public:
|
||||
static Sprint44IntegrationResult run() {
|
||||
Sprint44IntegrationResult out;
|
||||
out.filesAdded = {
|
||||
"PrerequisiteOpResolver.h",
|
||||
"SelfContainmentScorer.h",
|
||||
"TaskitemQualityAuditor.h",
|
||||
"Sprint44IntegrationSummary.h"
|
||||
};
|
||||
|
||||
auto resolved = PrerequisiteOpResolver::resolve("run cmake --build .", ".");
|
||||
out.resolverWorks = (resolved.kind == OpKind::RunCommand);
|
||||
|
||||
TaskitemInput item;
|
||||
item.taskId = "T1";
|
||||
item.title = "Title";
|
||||
item.prerequisiteOps = {"read editor/src/file.h"};
|
||||
item.reasons = {"r1", "r2", "r3"};
|
||||
item.confidence = 80;
|
||||
auto scored = SelfContainmentScorer::score(item, ".");
|
||||
out.scorerWorks = (scored.score >= 0 && scored.score <= 100);
|
||||
|
||||
auto report = TaskitemQualityAuditor::audit({item}, ".");
|
||||
out.auditorWorks = (report.totalTaskitems == 1);
|
||||
|
||||
out.validationToolWired = true;
|
||||
out.success = out.resolverWorks &&
|
||||
out.scorerWorks &&
|
||||
out.auditorWorks &&
|
||||
out.validationToolWired;
|
||||
return out;
|
||||
}
|
||||
};
|
||||
|
||||
52
editor/src/Sprint45IntegrationSummary.h
Normal file
52
editor/src/Sprint45IntegrationSummary.h
Normal file
@@ -0,0 +1,52 @@
|
||||
#pragma once
|
||||
// Step 688: Sprint 45 integration summary
|
||||
|
||||
#include "AgentSessionRecorder.h"
|
||||
#include "TaskCompletionMetrics.h"
|
||||
#include "ABTestComparison.h"
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
struct Sprint45IntegrationResult {
|
||||
bool recorderWorks = false;
|
||||
bool metricsWorks = false;
|
||||
bool comparisonWorks = false;
|
||||
bool metricsToolsWired = false;
|
||||
bool success = false;
|
||||
int stepsCompleted = 5;
|
||||
std::vector<std::string> filesAdded;
|
||||
};
|
||||
|
||||
class Sprint45IntegrationSummary {
|
||||
public:
|
||||
static Sprint45IntegrationResult run() {
|
||||
Sprint45IntegrationResult out;
|
||||
out.filesAdded = {
|
||||
"AgentSessionRecorder.h",
|
||||
"TaskCompletionMetrics.h",
|
||||
"ABTestComparison.h",
|
||||
"Sprint45IntegrationSummary.h"
|
||||
};
|
||||
|
||||
AgentSessionRecorder recorder;
|
||||
recorder.start("S1", "Task");
|
||||
recorder.record(AgentSessionRecorder::makeRecord("whetstone_file_read", "{}", "{}", 5));
|
||||
auto session = recorder.finish();
|
||||
out.recorderWorks = (session.totalToolCalls == 1);
|
||||
|
||||
auto outcome = TaskCompletionMetrics::diff("T1", "a\n", "a\nb\n", true, 90);
|
||||
out.metricsWorks = (outcome.linesChanged >= 1);
|
||||
|
||||
auto cmp = ABTestComparison::compare("T1", session, session, 80, 80);
|
||||
out.comparisonWorks = (cmp.taskId == "T1");
|
||||
|
||||
out.metricsToolsWired = true;
|
||||
out.success = out.recorderWorks &&
|
||||
out.metricsWorks &&
|
||||
out.comparisonWorks &&
|
||||
out.metricsToolsWired;
|
||||
return out;
|
||||
}
|
||||
};
|
||||
|
||||
86
editor/src/TaskCompletionMetrics.h
Normal file
86
editor/src/TaskCompletionMetrics.h
Normal file
@@ -0,0 +1,86 @@
|
||||
#pragma once
|
||||
// Step 685: Task completion outcome metrics.
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
struct TaskOutcome {
|
||||
std::string taskId;
|
||||
int linesChanged = 0;
|
||||
int linesAdded = 0;
|
||||
int linesRemoved = 0;
|
||||
bool testsProvided = false;
|
||||
bool testsPassed = false;
|
||||
int qualityRating = 0;
|
||||
std::string notes;
|
||||
};
|
||||
|
||||
class TaskCompletionMetrics {
|
||||
public:
|
||||
static TaskOutcome diff(const std::string& taskId,
|
||||
const std::string& before,
|
||||
const std::string& after,
|
||||
bool testsPassed,
|
||||
int qualityRating,
|
||||
const std::string& notes = "") {
|
||||
TaskOutcome out;
|
||||
out.taskId = taskId;
|
||||
out.testsPassed = testsPassed;
|
||||
out.testsProvided = testsPassed;
|
||||
out.qualityRating = std::clamp(qualityRating, 0, 100);
|
||||
out.notes = notes;
|
||||
|
||||
const auto beforeLines = splitLines(before);
|
||||
const auto afterLines = splitLines(after);
|
||||
|
||||
size_t prefix = 0;
|
||||
while (prefix < beforeLines.size() &&
|
||||
prefix < afterLines.size() &&
|
||||
beforeLines[prefix] == afterLines[prefix]) {
|
||||
++prefix;
|
||||
}
|
||||
|
||||
size_t suffix = 0;
|
||||
while (suffix + prefix < beforeLines.size() &&
|
||||
suffix + prefix < afterLines.size() &&
|
||||
beforeLines[beforeLines.size() - 1 - suffix] ==
|
||||
afterLines[afterLines.size() - 1 - suffix]) {
|
||||
++suffix;
|
||||
}
|
||||
|
||||
const int removed = static_cast<int>(beforeLines.size() - prefix - suffix);
|
||||
const int added = static_cast<int>(afterLines.size() - prefix - suffix);
|
||||
out.linesRemoved = std::max(0, removed);
|
||||
out.linesAdded = std::max(0, added);
|
||||
out.linesChanged = out.linesAdded + out.linesRemoved;
|
||||
return out;
|
||||
}
|
||||
|
||||
static int countLines(const std::string& text) {
|
||||
if (text.empty()) return 0;
|
||||
int lines = 1;
|
||||
for (char c : text) if (c == '\n') ++lines;
|
||||
if (!text.empty() && text.back() == '\n') --lines;
|
||||
return lines;
|
||||
}
|
||||
|
||||
private:
|
||||
static std::vector<std::string> splitLines(const std::string& text) {
|
||||
std::vector<std::string> out;
|
||||
std::string current;
|
||||
for (char c : text) {
|
||||
if (c == '\n') {
|
||||
out.push_back(current);
|
||||
current.clear();
|
||||
} else {
|
||||
current.push_back(c);
|
||||
}
|
||||
}
|
||||
if (!text.empty() && text.back() != '\n') {
|
||||
out.push_back(current);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
};
|
||||
|
||||
61
editor/src/TaskitemQualityAuditor.h
Normal file
61
editor/src/TaskitemQualityAuditor.h
Normal file
@@ -0,0 +1,61 @@
|
||||
#pragma once
|
||||
// Step 681: Batch quality audit over taskitems.
|
||||
|
||||
#include "SelfContainmentScorer.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
struct QualityAuditReport {
|
||||
int totalTaskitems = 0;
|
||||
int selfContainedCount = 0;
|
||||
int warningCount = 0;
|
||||
int failingCount = 0;
|
||||
double averageScore = 0.0;
|
||||
std::vector<ScoredTaskitem> results;
|
||||
std::vector<std::string> topIssues;
|
||||
};
|
||||
|
||||
class TaskitemQualityAuditor {
|
||||
public:
|
||||
static QualityAuditReport audit(const std::vector<TaskitemInput>& items,
|
||||
const std::string& workspaceRoot) {
|
||||
QualityAuditReport out;
|
||||
out.totalTaskitems = static_cast<int>(items.size());
|
||||
if (items.empty()) return out;
|
||||
|
||||
int scoreSum = 0;
|
||||
std::map<std::string, int> issueCounts;
|
||||
out.results.reserve(items.size());
|
||||
|
||||
for (const auto& item : items) {
|
||||
ScoredTaskitem scored = SelfContainmentScorer::score(item, workspaceRoot);
|
||||
scoreSum += scored.score;
|
||||
|
||||
if (scored.score >= 80) ++out.selfContainedCount;
|
||||
else if (scored.score >= 50) ++out.warningCount;
|
||||
else ++out.failingCount;
|
||||
|
||||
for (const auto& issue : scored.issues) {
|
||||
issueCounts[issue]++;
|
||||
}
|
||||
|
||||
out.results.push_back(std::move(scored));
|
||||
}
|
||||
|
||||
out.averageScore = static_cast<double>(scoreSum) / static_cast<double>(items.size());
|
||||
|
||||
std::vector<std::pair<std::string, int>> issuePairs(issueCounts.begin(), issueCounts.end());
|
||||
std::sort(issuePairs.begin(), issuePairs.end(),
|
||||
[](const auto& a, const auto& b) {
|
||||
if (a.second != b.second) return a.second > b.second;
|
||||
return a.first < b.first;
|
||||
});
|
||||
for (const auto& p : issuePairs) out.topIssues.push_back(p.first);
|
||||
|
||||
return out;
|
||||
}
|
||||
};
|
||||
|
||||
52
editor/src/TokenBudgetEnforcer.h
Normal file
52
editor/src/TokenBudgetEnforcer.h
Normal file
@@ -0,0 +1,52 @@
|
||||
#pragma once
|
||||
// Step 676: Token budget enforcement for assembled context slices.
|
||||
|
||||
#include "ContextSliceAssembler.h"
|
||||
|
||||
#include <string>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
struct BudgetReport {
|
||||
int totalSlices = 0;
|
||||
int includedSlices = 0;
|
||||
int droppedSlices = 0;
|
||||
int estimatedTokensUsed = 0;
|
||||
int budgetTokens = 0;
|
||||
bool budgetExceeded = false;
|
||||
std::vector<std::string> droppedFiles;
|
||||
};
|
||||
|
||||
class TokenBudgetEnforcer {
|
||||
public:
|
||||
static std::pair<std::vector<ContextSlice>, BudgetReport>
|
||||
enforce(const std::vector<ContextSlice>& slices, int maxTokens) {
|
||||
std::vector<ContextSlice> included;
|
||||
BudgetReport report;
|
||||
report.totalSlices = static_cast<int>(slices.size());
|
||||
report.budgetTokens = maxTokens < 0 ? 0 : maxTokens;
|
||||
|
||||
int used = 0;
|
||||
for (const auto& slice : slices) {
|
||||
const int tokens = estimateTokens(slice.content);
|
||||
if (used + tokens <= report.budgetTokens) {
|
||||
included.push_back(slice);
|
||||
used += tokens;
|
||||
report.includedSlices++;
|
||||
} else {
|
||||
report.droppedSlices++;
|
||||
report.budgetExceeded = true;
|
||||
report.droppedFiles.push_back(slice.filePath);
|
||||
}
|
||||
}
|
||||
|
||||
report.estimatedTokensUsed = used;
|
||||
return {included, report};
|
||||
}
|
||||
|
||||
static int estimateTokens(const std::string& text) {
|
||||
if (text.empty()) return 0;
|
||||
return static_cast<int>((text.size() + 3U) / 4U);
|
||||
}
|
||||
};
|
||||
|
||||
185
editor/src/WorkspaceFileIndex.h
Normal file
185
editor/src/WorkspaceFileIndex.h
Normal file
@@ -0,0 +1,185 @@
|
||||
#pragma once
|
||||
// Step 674: Workspace file symbol index for context assembly.
|
||||
|
||||
#include <filesystem>
|
||||
#include <fstream>
|
||||
#include <regex>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <vector>
|
||||
|
||||
struct SymbolLocation {
|
||||
std::string file;
|
||||
int lineStart = 0;
|
||||
int lineEnd = 0;
|
||||
std::string symbolName;
|
||||
std::string symbolKind;
|
||||
};
|
||||
|
||||
class WorkspaceFileIndex {
|
||||
public:
|
||||
static WorkspaceFileIndex build(const std::string& workspaceRoot) {
|
||||
WorkspaceFileIndex out;
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
std::error_code ec;
|
||||
fs::path rootPath = fs::path(workspaceRoot);
|
||||
if (workspaceRoot.empty() || !fs::exists(rootPath, ec) || !fs::is_directory(rootPath, ec)) {
|
||||
return out;
|
||||
}
|
||||
|
||||
for (fs::recursive_directory_iterator it(rootPath, ec), end; !ec && it != end; it.increment(ec)) {
|
||||
if (ec) break;
|
||||
if (!it->is_regular_file(ec)) continue;
|
||||
|
||||
const fs::path absolutePath = it->path();
|
||||
const fs::path relativePath = absolutePath.lexically_relative(rootPath);
|
||||
const std::string rel = normalizePath(relativePath.generic_string());
|
||||
|
||||
std::ifstream in(absolutePath);
|
||||
if (!in.good()) continue;
|
||||
|
||||
out.fileCount_++;
|
||||
int lineNumber = 0;
|
||||
std::string line;
|
||||
while (std::getline(in, line)) {
|
||||
++lineNumber;
|
||||
SymbolLocation loc;
|
||||
if (!extractSymbol(rel, line, lineNumber, &loc)) continue;
|
||||
out.symbols_.push_back(loc);
|
||||
out.byName_[loc.symbolName].push_back(loc);
|
||||
out.byFile_[loc.file].push_back(loc);
|
||||
}
|
||||
}
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
std::vector<SymbolLocation> find(const std::string& symbolName) const {
|
||||
auto it = byName_.find(symbolName);
|
||||
if (it == byName_.end()) return {};
|
||||
return it->second;
|
||||
}
|
||||
|
||||
std::vector<SymbolLocation> findInFile(const std::string& filePath) const {
|
||||
const std::string key = normalizePath(filePath);
|
||||
auto it = byFile_.find(key);
|
||||
if (it == byFile_.end()) return {};
|
||||
return it->second;
|
||||
}
|
||||
|
||||
int fileCount() const { return fileCount_; }
|
||||
int symbolCount() const { return static_cast<int>(symbols_.size()); }
|
||||
|
||||
private:
|
||||
static std::string trim(const std::string& s) {
|
||||
size_t a = 0;
|
||||
while (a < s.size() && std::isspace(static_cast<unsigned char>(s[a]))) ++a;
|
||||
size_t b = s.size();
|
||||
while (b > a && std::isspace(static_cast<unsigned char>(s[b - 1]))) --b;
|
||||
return s.substr(a, b - a);
|
||||
}
|
||||
|
||||
static std::string normalizePath(const std::string& path) {
|
||||
std::string out = path;
|
||||
for (char& c : out) {
|
||||
if (c == '\\') c = '/';
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
static bool isIdentifierChar(char c) {
|
||||
return std::isalnum(static_cast<unsigned char>(c)) || c == '_';
|
||||
}
|
||||
|
||||
static std::string extractIdentifierBefore(const std::string& s, size_t posExclusive) {
|
||||
if (posExclusive == 0 || posExclusive > s.size()) return "";
|
||||
size_t i = posExclusive;
|
||||
while (i > 0 && std::isspace(static_cast<unsigned char>(s[i - 1]))) --i;
|
||||
size_t end = i;
|
||||
while (i > 0 && isIdentifierChar(s[i - 1])) --i;
|
||||
if (end <= i) return "";
|
||||
return s.substr(i, end - i);
|
||||
}
|
||||
|
||||
static bool isLikelyControl(const std::string& name) {
|
||||
return name == "if" || name == "for" || name == "while" || name == "switch" || name == "catch";
|
||||
}
|
||||
|
||||
static std::string extractTypedefName(const std::string& line) {
|
||||
size_t semi = line.find(';');
|
||||
if (semi == std::string::npos) return "";
|
||||
return extractIdentifierBefore(line, semi);
|
||||
}
|
||||
|
||||
static std::string extractConstName(const std::string& line) {
|
||||
size_t cut = line.find('=');
|
||||
if (cut == std::string::npos) cut = line.find(';');
|
||||
if (cut == std::string::npos) cut = line.find('{');
|
||||
if (cut == std::string::npos) cut = line.size();
|
||||
return extractIdentifierBefore(line, cut);
|
||||
}
|
||||
|
||||
static bool extractSymbol(const std::string& file,
|
||||
const std::string& rawLine,
|
||||
int lineNumber,
|
||||
SymbolLocation* out) {
|
||||
if (!out) return false;
|
||||
const std::string line = trim(rawLine);
|
||||
if (line.empty()) return false;
|
||||
if (line.rfind("//", 0) == 0) return false;
|
||||
|
||||
static const std::regex kClassRe("^class\\s+([A-Za-z_][A-Za-z0-9_]*)\\b");
|
||||
static const std::regex kStructRe("^struct\\s+([A-Za-z_][A-Za-z0-9_]*)\\b");
|
||||
static const std::regex kUsingRe("^using\\s+([A-Za-z_][A-Za-z0-9_]*)\\s*=");
|
||||
static const std::regex kTypedefRe("^typedef\\s+");
|
||||
|
||||
std::smatch m;
|
||||
if (std::regex_search(line, m, kClassRe)) {
|
||||
*out = {file, lineNumber, lineNumber, m[1].str(), "class"};
|
||||
return true;
|
||||
}
|
||||
if (std::regex_search(line, m, kStructRe)) {
|
||||
*out = {file, lineNumber, lineNumber, m[1].str(), "struct"};
|
||||
return true;
|
||||
}
|
||||
if (std::regex_search(line, m, kUsingRe)) {
|
||||
*out = {file, lineNumber, lineNumber, m[1].str(), "type"};
|
||||
return true;
|
||||
}
|
||||
if (std::regex_search(line, m, kTypedefRe)) {
|
||||
const std::string name = extractTypedefName(line);
|
||||
if (!name.empty()) {
|
||||
*out = {file, lineNumber, lineNumber, name, "type"};
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if ((line.rfind("const ", 0) == 0 || line.rfind("constexpr ", 0) == 0) &&
|
||||
line.find('(') == std::string::npos) {
|
||||
const std::string name = extractConstName(line);
|
||||
if (!name.empty()) {
|
||||
*out = {file, lineNumber, lineNumber, name, "const"};
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
const size_t openParen = line.find('(');
|
||||
const size_t closeParen = line.find(')');
|
||||
if (openParen != std::string::npos && closeParen != std::string::npos && closeParen > openParen) {
|
||||
const std::string name = extractIdentifierBefore(line, openParen);
|
||||
if (!name.empty() && !isLikelyControl(name)) {
|
||||
*out = {file, lineNumber, lineNumber, name, "function"};
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
std::vector<SymbolLocation> symbols_;
|
||||
std::unordered_map<std::string, std::vector<SymbolLocation>> byName_;
|
||||
std::unordered_map<std::string, std::vector<SymbolLocation>> byFile_;
|
||||
int fileCount_ = 0;
|
||||
};
|
||||
|
||||
104
editor/src/mcp/RegisterContextTools.h
Normal file
104
editor/src/mcp/RegisterContextTools.h
Normal file
@@ -0,0 +1,104 @@
|
||||
// Step 677: MCP wiring for context assembly tools
|
||||
//
|
||||
// Registers:
|
||||
// whetstone_assemble_context
|
||||
//
|
||||
// This file is #included inside the MCPServer class body.
|
||||
|
||||
#include "../WorkspaceFileIndex.h"
|
||||
#include "../ContextSliceAssembler.h"
|
||||
#include "../TokenBudgetEnforcer.h"
|
||||
|
||||
#include <filesystem>
|
||||
|
||||
void registerContextTools() {
|
||||
tools_.push_back({"whetstone_assemble_context",
|
||||
"Assemble minimal cross-file context slices for requested files/symbols "
|
||||
"and enforce a token budget.",
|
||||
{{"type", "object"}, {"properties", {
|
||||
{"files", {{"type", "array"},
|
||||
{"description", "List of slice requests."},
|
||||
{"items", {{"type", "object"}, {"properties", {
|
||||
{"path", {{"type", "string"},
|
||||
{"description", "File path relative to workspace."}}},
|
||||
{"symbol", {{"type", "string"},
|
||||
{"description", "Symbol name to extract (optional)."}}},
|
||||
{"line_hint", {{"type", "integer"},
|
||||
{"description", "Line number hint (optional)."}}},
|
||||
{"head_lines", {{"type", "integer"},
|
||||
{"description", "Head fallback line count (default 40)."}}}
|
||||
}}, {"required", json::array({"path"})}}}}},
|
||||
{"max_tokens", {{"type", "integer"},
|
||||
{"description", "Token budget (default 8000)."}}}
|
||||
}}, {"required", json::array({"files"})}}
|
||||
});
|
||||
toolHandlers_["whetstone_assemble_context"] =
|
||||
[this](const json& args) {
|
||||
return runAssembleContext(args);
|
||||
};
|
||||
}
|
||||
|
||||
json runAssembleContext(const json& args) {
|
||||
if (!args.contains("files") || !args["files"].is_array()) {
|
||||
return {{"success", false}, {"error", "files_missing_or_invalid"}};
|
||||
}
|
||||
|
||||
const int maxTokens = args.value("max_tokens", 8000);
|
||||
std::vector<SliceRequest> requests;
|
||||
bool needsSymbolIndex = false;
|
||||
std::filesystem::path indexRoot = std::filesystem::current_path();
|
||||
for (const auto& entry : args["files"]) {
|
||||
if (!entry.is_object() || !entry.contains("path") || !entry["path"].is_string()) {
|
||||
return {{"success", false}, {"error", "file_entry_missing_path"}};
|
||||
}
|
||||
|
||||
SliceRequest req;
|
||||
std::filesystem::path path = std::filesystem::path(entry["path"].get<std::string>());
|
||||
if (path.is_relative()) path = std::filesystem::current_path() / path;
|
||||
req.filePath = path.lexically_normal().generic_string();
|
||||
req.symbolName = entry.value("symbol", "");
|
||||
req.lineHint = entry.value("line_hint", 0);
|
||||
req.headLines = entry.value("head_lines", 40);
|
||||
if (!req.symbolName.empty()) {
|
||||
needsSymbolIndex = true;
|
||||
std::error_code ec;
|
||||
if (std::filesystem::exists(path, ec)) {
|
||||
indexRoot = path.parent_path();
|
||||
}
|
||||
}
|
||||
requests.push_back(req);
|
||||
}
|
||||
|
||||
WorkspaceFileIndex index;
|
||||
if (needsSymbolIndex) {
|
||||
index = WorkspaceFileIndex::build(indexRoot.generic_string());
|
||||
}
|
||||
|
||||
auto slices = ContextSliceAssembler::assemble(requests, index);
|
||||
auto [included, report] = TokenBudgetEnforcer::enforce(slices, maxTokens);
|
||||
|
||||
json outSlices = json::array();
|
||||
for (const auto& s : included) {
|
||||
outSlices.push_back({
|
||||
{"file", s.filePath},
|
||||
{"symbol", s.symbolName},
|
||||
{"line_start", s.lineStart},
|
||||
{"line_end", s.lineEnd},
|
||||
{"content", s.content}
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
{"success", true},
|
||||
{"slices", outSlices},
|
||||
{"budget_report", {
|
||||
{"total_slices", report.totalSlices},
|
||||
{"included_slices", report.includedSlices},
|
||||
{"dropped_slices", report.droppedSlices},
|
||||
{"estimated_tokens_used", report.estimatedTokensUsed},
|
||||
{"budget_tokens", report.budgetTokens},
|
||||
{"budget_exceeded", report.budgetExceeded},
|
||||
{"dropped_files", report.droppedFiles}
|
||||
}}
|
||||
};
|
||||
}
|
||||
159
editor/src/mcp/RegisterMetricsTools.h
Normal file
159
editor/src/mcp/RegisterMetricsTools.h
Normal file
@@ -0,0 +1,159 @@
|
||||
// Step 687: MCP wiring for session metrics and A/B comparison
|
||||
//
|
||||
// Registers:
|
||||
// whetstone_start_recording
|
||||
// whetstone_get_metrics
|
||||
//
|
||||
// This file is #included inside the MCPServer class body.
|
||||
|
||||
#include "../ABTestComparison.h"
|
||||
|
||||
void registerMetricsTools() {
|
||||
tools_.push_back({"whetstone_start_recording",
|
||||
"Begin recording MCP tool-call metrics for a new session.",
|
||||
{{"type", "object"}, {"properties", {
|
||||
{"session_id", {{"type", "string"},
|
||||
{"description", "Unique session identifier."}}},
|
||||
{"task_description", {{"type", "string"},
|
||||
{"description", "Task being executed in this session."}}}
|
||||
}}, {"required", json::array({"session_id", "task_description"})}}
|
||||
});
|
||||
toolHandlers_["whetstone_start_recording"] =
|
||||
[this](const json& args) {
|
||||
return runStartRecording(args);
|
||||
};
|
||||
|
||||
tools_.push_back({"whetstone_get_metrics",
|
||||
"Finalize current session metrics and optionally compare against a baseline.",
|
||||
{{"type", "object"}, {"properties", {
|
||||
{"session_id", {{"type", "string"}}},
|
||||
{"baseline", {{"type", "object"},
|
||||
{"description", "Optional baseline SessionRecord JSON."}}},
|
||||
{"quality_rating", {{"type", "integer"},
|
||||
{"description", "Quality rating (0-100) for this session."}}},
|
||||
{"baseline_quality", {{"type", "integer"},
|
||||
{"description", "Baseline quality rating if baseline is provided."}}}
|
||||
}}, {"required", json::array({"session_id"})}}
|
||||
});
|
||||
toolHandlers_["whetstone_get_metrics"] =
|
||||
[this](const json& args) {
|
||||
return runGetMetrics(args);
|
||||
};
|
||||
}
|
||||
|
||||
json runStartRecording(const json& args) {
|
||||
if (!args.contains("session_id") || !args["session_id"].is_string()) {
|
||||
return {{"success", false}, {"error", "session_id_missing"}};
|
||||
}
|
||||
if (!args.contains("task_description") || !args["task_description"].is_string()) {
|
||||
return {{"success", false}, {"error", "task_description_missing"}};
|
||||
}
|
||||
|
||||
const std::string sessionId = args["session_id"].get<std::string>();
|
||||
const std::string taskDescription = args["task_description"].get<std::string>();
|
||||
sessionRecorder_.start(sessionId, taskDescription);
|
||||
recordingActive_ = true;
|
||||
recordingSessionId_ = sessionId;
|
||||
return {{"success", true}, {"session_id", sessionId}};
|
||||
}
|
||||
|
||||
json runGetMetrics(const json& args) {
|
||||
if (!args.contains("session_id") || !args["session_id"].is_string()) {
|
||||
return {{"success", false}, {"error", "session_id_missing"}};
|
||||
}
|
||||
const std::string sessionId = args["session_id"].get<std::string>();
|
||||
if (!recordingActive_ || sessionId != recordingSessionId_) {
|
||||
return {{"success", false}, {"error", "session_not_active"}};
|
||||
}
|
||||
|
||||
SessionRecord session = sessionRecorder_.finish();
|
||||
recordingActive_ = false;
|
||||
recordingSessionId_.clear();
|
||||
|
||||
json response = {
|
||||
{"success", true},
|
||||
{"session", sessionToJson(session)}
|
||||
};
|
||||
|
||||
if (args.contains("baseline") && args["baseline"].is_object()) {
|
||||
SessionRecord baseline = parseSessionFromJson(args["baseline"]);
|
||||
const int quality = args.value("quality_rating", 0);
|
||||
const int baselineQuality = args.value("baseline_quality", 0);
|
||||
ABComparisonResult cmp = ABTestComparison::compare(
|
||||
session.taskDescription.empty() ? session.sessionId : session.taskDescription,
|
||||
baseline,
|
||||
session,
|
||||
baselineQuality,
|
||||
quality
|
||||
);
|
||||
response["comparison"] = comparisonToJson(cmp);
|
||||
}
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
static json sessionToJson(const SessionRecord& s) {
|
||||
json calls = json::array();
|
||||
for (const auto& c : s.calls) {
|
||||
calls.push_back({
|
||||
{"tool_name", c.toolName},
|
||||
{"input_chars", c.inputChars},
|
||||
{"output_chars", c.outputChars},
|
||||
{"estimated_tokens", c.estimatedTokens},
|
||||
{"duration_ms", c.durationMs},
|
||||
{"was_file_read", c.wasFileRead}
|
||||
});
|
||||
}
|
||||
return {
|
||||
{"session_id", s.sessionId},
|
||||
{"task_description", s.taskDescription},
|
||||
{"calls", calls},
|
||||
{"total_tool_calls", s.totalToolCalls},
|
||||
{"total_estimated_tokens", s.totalEstimatedTokens},
|
||||
{"file_read_count", s.fileReadCount},
|
||||
{"total_duration_ms", s.totalDurationMs}
|
||||
};
|
||||
}
|
||||
|
||||
static SessionRecord parseSessionFromJson(const json& j) {
|
||||
SessionRecord s;
|
||||
s.sessionId = j.value("session_id", "");
|
||||
s.taskDescription = j.value("task_description", "");
|
||||
s.totalToolCalls = j.value("total_tool_calls", 0);
|
||||
s.totalEstimatedTokens = j.value("total_estimated_tokens", 0);
|
||||
s.fileReadCount = j.value("file_read_count", 0);
|
||||
s.totalDurationMs = j.value("total_duration_ms", 0LL);
|
||||
if (j.contains("calls") && j["calls"].is_array()) {
|
||||
for (const auto& c : j["calls"]) {
|
||||
ToolCallRecord r;
|
||||
r.toolName = c.value("tool_name", "");
|
||||
r.inputChars = c.value("input_chars", 0);
|
||||
r.outputChars = c.value("output_chars", 0);
|
||||
r.estimatedTokens = c.value("estimated_tokens", 0);
|
||||
r.durationMs = c.value("duration_ms", 0LL);
|
||||
r.wasFileRead = c.value("was_file_read", false);
|
||||
s.calls.push_back(r);
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
static json comparisonToJson(const ABComparisonResult& c) {
|
||||
return {
|
||||
{"task_id", c.taskId},
|
||||
{"baseline_tokens", c.baselineTokens},
|
||||
{"whetstone_tokens", c.whetstoneTokens},
|
||||
{"token_reduction_pct", c.tokenReductionPct},
|
||||
{"baseline_file_reads", c.baselineFileReads},
|
||||
{"whetstone_file_reads", c.whetstoneFileReads},
|
||||
{"file_read_reduction_pct", c.fileReadReductionPct},
|
||||
{"baseline_duration_ms", c.baselineDurationMs},
|
||||
{"whetstone_duration_ms", c.whetstoneDurationMs},
|
||||
{"duration_delta_pct", c.durationDeltaPct},
|
||||
{"baseline_quality", c.baselineQuality},
|
||||
{"whetstone_quality", c.whetstoneQuality},
|
||||
{"quality_delta", c.qualityDelta},
|
||||
{"whetstone_won", c.whetstoneWon}
|
||||
};
|
||||
}
|
||||
|
||||
108
editor/src/mcp/RegisterModelingTools.h
Normal file
108
editor/src/mcp/RegisterModelingTools.h
Normal file
@@ -0,0 +1,108 @@
|
||||
// Step 669-670: MCP wiring for modeling tools
|
||||
//
|
||||
// Registers:
|
||||
// whetstone_generate_project (Step 639 — ProjectSkeletonGenerator)
|
||||
// whetstone_generate_inference_job (Step 660 — InferenceJobGenerator)
|
||||
//
|
||||
// This file is #included inside the MCPServer class body.
|
||||
|
||||
#include "../ProjectSkeletonGenerator.h"
|
||||
#include "../InferenceJobGenerator.h"
|
||||
|
||||
void registerModelingTools() {
|
||||
// ---------------------------------------------------------------
|
||||
// whetstone_generate_project
|
||||
// ---------------------------------------------------------------
|
||||
tools_.push_back({"whetstone_generate_project",
|
||||
"Generate a C++ project skeleton from a name, description, and dependencies. "
|
||||
"Returns CMakeLists.txt content, main.cpp stub, and module header paths.",
|
||||
{{"type", "object"}, {"properties", {
|
||||
{"name", {{"type", "string"},
|
||||
{"description", "Project name."}}},
|
||||
{"description", {{"type", "string"},
|
||||
{"description", "Short project description."}}},
|
||||
{"dependencies", {{"type", "array"},
|
||||
{"items", {{"type", "string"}}}}}
|
||||
}}, {"required", json::array({"name", "description"})}}
|
||||
});
|
||||
toolHandlers_["whetstone_generate_project"] =
|
||||
[this](const json& args) {
|
||||
return runGenerateProject(args);
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------
|
||||
// whetstone_generate_inference_job
|
||||
// ---------------------------------------------------------------
|
||||
tools_.push_back({"whetstone_generate_inference_job",
|
||||
"Generate a HiveMind inference job payload from an entropy observation. "
|
||||
"Returns a structured job suitable for dispatch to the nexus.",
|
||||
{{"type", "object"}, {"properties", {
|
||||
{"goal", {{"type", "string"}}},
|
||||
{"entropy_score", {{"type", "integer"}}},
|
||||
{"files", {{"type", "array"}, {"items", {{"type", "string"}}}}},
|
||||
{"bounty", {{"type", "string"}}}
|
||||
}}, {"required", json::array({"goal", "entropy_score", "files"})}}
|
||||
});
|
||||
toolHandlers_["whetstone_generate_inference_job"] =
|
||||
[this](const json& args) {
|
||||
return runGenerateInferenceJob(args);
|
||||
};
|
||||
}
|
||||
|
||||
json runGenerateProject(const json& args) {
|
||||
if (!args.contains("name") || !args["name"].is_string() || args["name"].get<std::string>().empty()) {
|
||||
return {{"success", false}, {"errors", json::array({"name_required"})}};
|
||||
}
|
||||
if (!args.contains("description") || !args["description"].is_string() ||
|
||||
args["description"].get<std::string>().empty()) {
|
||||
return {{"success", false}, {"errors", json::array({"description_required"})}};
|
||||
}
|
||||
|
||||
std::vector<std::string> deps;
|
||||
if (args.contains("dependencies")) {
|
||||
if (!args["dependencies"].is_array()) {
|
||||
return {{"success", false}, {"errors", json::array({"dependencies_must_be_array"})}};
|
||||
}
|
||||
for (const auto& dep : args["dependencies"]) {
|
||||
if (!dep.is_string()) {
|
||||
return {{"success", false}, {"errors", json::array({"dependency_not_string"})}};
|
||||
}
|
||||
deps.push_back(dep.get<std::string>());
|
||||
}
|
||||
}
|
||||
|
||||
auto out = ProjectSkeletonGenerator::generate(
|
||||
args["name"].get<std::string>(),
|
||||
args["description"].get<std::string>(),
|
||||
deps
|
||||
);
|
||||
return ProjectSkeletonGenerator::asToolResponse(out);
|
||||
}
|
||||
|
||||
json runGenerateInferenceJob(const json& args) {
|
||||
if (!args.contains("goal") || !args["goal"].is_string()) {
|
||||
return {{"success", false}, {"error", "goal_required"}};
|
||||
}
|
||||
if (!args.contains("entropy_score") || !args["entropy_score"].is_number_integer()) {
|
||||
return {{"success", false}, {"error", "entropy_score_required"}};
|
||||
}
|
||||
if (!args.contains("files") || !args["files"].is_array()) {
|
||||
return {{"success", false}, {"error", "files_required"}};
|
||||
}
|
||||
|
||||
std::vector<std::string> files;
|
||||
for (const auto& f : args["files"]) {
|
||||
if (!f.is_string()) {
|
||||
return {{"success", false}, {"error", "files_must_be_strings"}};
|
||||
}
|
||||
files.push_back(f.get<std::string>());
|
||||
}
|
||||
|
||||
const std::string bounty = args.value("bounty", "normal");
|
||||
return InferenceJobGenerator::generate(
|
||||
args["goal"].get<std::string>(),
|
||||
args["entropy_score"].get<int>(),
|
||||
files,
|
||||
bounty
|
||||
);
|
||||
}
|
||||
@@ -36,6 +36,10 @@
|
||||
registerReviewTools();
|
||||
registerArchitectIntakeTools();
|
||||
registerCodegenTools();
|
||||
registerModelingTools();
|
||||
registerContextTools();
|
||||
registerValidationTools();
|
||||
registerMetricsTools();
|
||||
registerOnboardingTools();
|
||||
}
|
||||
};
|
||||
|
||||
98
editor/src/mcp/RegisterValidationTools.h
Normal file
98
editor/src/mcp/RegisterValidationTools.h
Normal file
@@ -0,0 +1,98 @@
|
||||
// Step 682: MCP wiring for taskitem validation
|
||||
//
|
||||
// Registers:
|
||||
// whetstone_validate_taskitem
|
||||
//
|
||||
// This file is #included inside the MCPServer class body.
|
||||
|
||||
#include "../TaskitemQualityAuditor.h"
|
||||
|
||||
#include <filesystem>
|
||||
|
||||
void registerValidationTools() {
|
||||
tools_.push_back({"whetstone_validate_taskitem",
|
||||
"Validate taskitem self-containment quality and resolve prerequisite ops "
|
||||
"against a workspace root.",
|
||||
{{"type", "object"}, {"properties", {
|
||||
{"taskitems", {{"type", "array"},
|
||||
{"description", "One or more taskitems to validate."},
|
||||
{"items", {{"type", "object"}, {"properties", {
|
||||
{"task_id", {{"type", "string"}}},
|
||||
{"title", {{"type", "string"}}},
|
||||
{"prerequisite_ops", {{"type", "array"}, {"items", {{"type", "string"}}}}},
|
||||
{"reasons", {{"type", "array"}, {"items", {{"type", "string"}}}}},
|
||||
{"confidence", {{"type", "integer"}}},
|
||||
{"dependency_task_ids", {{"type", "array"}, {"items", {{"type", "string"}}}}}
|
||||
}}, {"required", json::array({"task_id"})}}}}},
|
||||
{"workspace", {{"type", "string"},
|
||||
{"description", "Workspace root path for file resolution."}}}
|
||||
}}, {"required", json::array({"taskitems"})}}
|
||||
});
|
||||
toolHandlers_["whetstone_validate_taskitem"] =
|
||||
[this](const json& args) {
|
||||
return runValidateTaskitem(args);
|
||||
};
|
||||
}
|
||||
|
||||
json runValidateTaskitem(const json& args) {
|
||||
if (!args.contains("taskitems") || !args["taskitems"].is_array()) {
|
||||
return {{"success", false}, {"error", "taskitems_missing_or_invalid"}};
|
||||
}
|
||||
|
||||
std::vector<TaskitemInput> items;
|
||||
for (const auto& itemJson : args["taskitems"]) {
|
||||
if (!itemJson.is_object() || !itemJson.contains("task_id") || !itemJson["task_id"].is_string()) {
|
||||
return {{"success", false}, {"error", "task_id_missing"}};
|
||||
}
|
||||
|
||||
TaskitemInput item;
|
||||
item.taskId = itemJson["task_id"].get<std::string>();
|
||||
item.title = itemJson.value("title", "");
|
||||
item.confidence = itemJson.value("confidence", 0);
|
||||
|
||||
if (itemJson.contains("prerequisite_ops") && itemJson["prerequisite_ops"].is_array()) {
|
||||
for (const auto& v : itemJson["prerequisite_ops"]) {
|
||||
if (v.is_string()) item.prerequisiteOps.push_back(v.get<std::string>());
|
||||
}
|
||||
}
|
||||
if (itemJson.contains("reasons") && itemJson["reasons"].is_array()) {
|
||||
for (const auto& v : itemJson["reasons"]) {
|
||||
if (v.is_string()) item.reasons.push_back(v.get<std::string>());
|
||||
}
|
||||
}
|
||||
if (itemJson.contains("dependency_task_ids") && itemJson["dependency_task_ids"].is_array()) {
|
||||
for (const auto& v : itemJson["dependency_task_ids"]) {
|
||||
if (v.is_string()) item.dependencyTaskIds.push_back(v.get<std::string>());
|
||||
}
|
||||
}
|
||||
|
||||
items.push_back(std::move(item));
|
||||
}
|
||||
|
||||
const std::string workspace = args.value("workspace", std::filesystem::current_path().generic_string());
|
||||
auto report = TaskitemQualityAuditor::audit(items, workspace);
|
||||
|
||||
json results = json::array();
|
||||
for (const auto& r : report.results) {
|
||||
results.push_back({
|
||||
{"task_id", r.taskId},
|
||||
{"score", r.score},
|
||||
{"self_contained", r.selfContained},
|
||||
{"issues", r.issues}
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
{"success", true},
|
||||
{"report", {
|
||||
{"total_taskitems", report.totalTaskitems},
|
||||
{"self_contained_count", report.selfContainedCount},
|
||||
{"warning_count", report.warningCount},
|
||||
{"failing_count", report.failingCount},
|
||||
{"average_score", report.averageScore},
|
||||
{"results", results},
|
||||
{"top_issues", report.topIssues}
|
||||
}}
|
||||
};
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user