bill/whetstone_DSL
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Complete Step 486: adaptive test plan generator

Browse Source
This commit is contained in:
Bill
2026-02-16 21:30:47 -07:00
parent 49126896ee
commit cd983a692e
4 changed files with 360 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
editor/CMakeLists.txt
View File

@@ -3253,4 +3253,13 @@ target_link_libraries(step485_test PRIVATE
tree_sitter_javascript tree_sitter_typescript
tree_sitter_java tree_sitter_rust tree_sitter_go)
add_executable(step486_test tests/step486_test.cpp)
target_include_directories(step486_test PRIVATE src)
target_link_libraries(step486_test PRIVATE
nlohmann_json::nlohmann_json
unofficial::tree-sitter::tree-sitter
tree_sitter_python tree_sitter_cpp tree_sitter_elisp
tree_sitter_javascript tree_sitter_typescript
tree_sitter_java tree_sitter_rust tree_sitter_go)
# Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies)

191
editor/src/TestPlanGenerator.h Normal file
View File

@@ -0,0 +1,191 @@
#pragma once
// Step 486: Test Plan Generator
// Proposes typed tests from step description heuristics + complexity budget.
#include "StepSpecExpander.h"
#include <algorithm>
#include <cctype>
#include <string>
#include <vector>
struct TestPatternSample {
std::string path;
std::string content;
};
struct GeneratedTestPlan {
StepComplexity complexity = StepComplexity::Standard;
std::vector<PlannedTestStub> tests;
std::vector<std::string> rationale;
int countType(PlannedTestType t) const {
int n = 0;
for (const auto& x : tests) if (x.type == t) ++n;
return n;
}
bool hasType(PlannedTestType t) const {
return countType(t) > 0;
}
};
struct TestPlanInput {
std::string stepMarkdown;
std::string stepNumber;
std::string componentName;
std::vector<TestPatternSample> priorTests;
StepComplexity complexityHint = StepComplexity::Standard;
bool useComplexityHint = false;
};
class TestPlanGenerator {
public:
static GeneratedTestPlan generate(const TestPlanInput& in) {
GeneratedTestPlan out;
const std::string text = lower(in.stepMarkdown);
out.complexity = in.useComplexityHint ? in.complexityHint : inferComplexity(in);
out.rationale.push_back("Complexity classified as " + complexityLabel(out.complexity));
add(out, in, PlannedTestType::Unit, "core behavior correctness", "worker");
add(out, in, PlannedTestType::Unit, "API surface coverage", "worker");
if (hasAny(text, {"detect", "validate", "reject", "invalid"})) {
add(out, in, PlannedTestType::Negative, "rejects invalid input", "worker");
out.rationale.push_back("Validation keywords detected -> negative tests");
}
if (hasAny(text, {"pipeline", "workflow"})) {
add(out, in, PlannedTestType::Integration, "components compose in full flow", "architect");
out.rationale.push_back("Pipeline/workflow keywords detected -> integration tests");
}
if (hasAny(text, {"rpc", "mcp"})) {
add(out, in, PlannedTestType::Smoke, "basic RPC/MCP response viability", "worker");
out.rationale.push_back("RPC/MCP keywords detected -> smoke test");
}
if (hasAny(text, {"score", "arithmetic", "limit", "boundary", "max", "min"})) {
add(out, in, PlannedTestType::Boundary, "boundary limits remain safe", "worker");
out.rationale.push_back("Boundary-related keywords detected -> boundary test");
}
if (hasAny(text, {"contract", "@contract"})) {
add(out, in, PlannedTestType::Contract, "pre/post contract is preserved", "worker");
out.rationale.push_back("Contract keywords detected -> contract test");
}
if (hasAny(text, {"roundtrip", "invariant", "property", "serialize"})) {
add(out, in, PlannedTestType::Property, "invariant/property holds across inputs", "worker");
out.rationale.push_back("Invariant keywords detected -> property test");
}
if (hasAny(text, {"performance", "latency", "throughput", "hot path"})) {
add(out, in, PlannedTestType::Performance, "no major perf regression", "architect");
out.rationale.push_back("Performance keywords detected -> performance test");
}
if (hasAny(text, {"modify", "change existing", "regression"})) {
add(out, in, PlannedTestType::Regression, "existing behavior remains stable", "architect");
out.rationale.push_back("Modification keywords detected -> regression test");
}
if (priorSuggestsRegression(in.priorTests)) {
add(out, in, PlannedTestType::Regression, "prior-step regression continuity", "architect");
out.rationale.push_back("Prior tests indicate regression culture -> regression test");
}
if (hasAny(text, {"new struct", "add struct", "data struct"})) {
add(out, in, PlannedTestType::Unit, "constructor/accessor behavior", "worker");
out.rationale.push_back("New-struct signal detected -> extra unit test");
}
enforceBudget(out, in);
return out;
}
private:
static void add(GeneratedTestPlan& out, const TestPlanInput& in,
PlannedTestType type, const std::string& desc,
const std::string& routing) {
PlannedTestStub t;
t.type = type;
t.name = "test_step" + (in.stepNumber.empty() ? std::string("x") : in.stepNumber) +
"_" + typeLabel(type) + "_" + std::to_string(out.tests.size() + 1);
t.description = desc;
t.expectedRouting = routing;
out.tests.push_back(std::move(t));
}
static bool hasAny(const std::string& text, std::initializer_list<const char*> needles) {
for (const auto& n : needles) if (text.find(n) != std::string::npos) return true;
return false;
}
static std::string lower(const std::string& s) {
std::string out = s;
std::transform(out.begin(), out.end(), out.begin(),
[](unsigned char c) { return static_cast<char>(std::tolower(c)); });
return out;
}
static StepComplexity inferComplexity(const TestPlanInput& in) {
StepExpanderInput e;
e.stepMarkdown = in.stepMarkdown;
e.stepNumber = in.stepNumber;
e.componentName = in.componentName.empty() ? "Generated" : in.componentName;
auto spec = StepSpecExpander::expand(e);
return spec.complexity;
}
static bool priorSuggestsRegression(const std::vector<TestPatternSample>& samples) {
for (const auto& s : samples) {
const std::string t = lower(s.content + " " + s.path);
if (t.find("regression") != std::string::npos) return true;
}
return false;
}
static std::string typeLabel(PlannedTestType t) {
switch (t) {
case PlannedTestType::Unit: return "unit";
case PlannedTestType::Negative: return "negative";
case PlannedTestType::Integration: return "integration";
case PlannedTestType::Contract: return "contract";
case PlannedTestType::Regression: return "regression";
case PlannedTestType::Boundary: return "boundary";
case PlannedTestType::Property: return "property";
case PlannedTestType::Smoke: return "smoke";
case PlannedTestType::Performance: return "performance";
}
return "unit";
}
static std::string complexityLabel(StepComplexity c) {
switch (c) {
case StepComplexity::Trivial: return "trivial";
case StepComplexity::Standard: return "standard";
case StepComplexity::Complex: return "complex";
case StepComplexity::Pipeline: return "pipeline";
}
return "standard";
}
static void enforceBudget(GeneratedTestPlan& out, const TestPlanInput& in) {
(void)in;
size_t minCount = 5, maxCount = 10;
if (out.complexity == StepComplexity::Trivial) {
minCount = 1; maxCount = 4;
} else if (out.complexity == StepComplexity::Complex) {
minCount = 10; maxCount = 15;
} else if (out.complexity == StepComplexity::Pipeline) {
minCount = 6; maxCount = 10;
}
while (out.tests.size() < minCount) {
PlannedTestType filler = PlannedTestType::Unit;
std::string desc = "budget filler coverage";
std::string routing = "worker";
if (out.complexity == StepComplexity::Pipeline && out.countType(PlannedTestType::Integration) == 0) {
filler = PlannedTestType::Integration;
desc = "pipeline integration coverage";
routing = "architect";
}
add(out, in, filler, desc, routing);
}
if (out.tests.size() > maxCount) out.tests.resize(maxCount);
}
};

118
editor/tests/step486_test.cpp Normal file
View File

@@ -0,0 +1,118 @@
// Step 486: Test Plan Generator Tests (unit, boundary, negative)
#include "TestPlanGenerator.h"
#include <iostream>
#include <string>
static int passed = 0, failed = 0;
#define TEST(name) { std::cout << " " << #name << "... "; }
#define PASS() { std::cout << "PASS\n"; ++passed; }
#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; }
#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {}
static TestPlanInput baseInput(const std::string& text) {
TestPlanInput in;
in.stepMarkdown = text;
in.stepNumber = "486";
in.componentName = "TestPlanGenerator";
return in;
}
void test_validation_keywords_add_negative_tests() {
TEST(validation_keywords_add_negative_tests);
auto plan = TestPlanGenerator::generate(baseInput("Validate and reject invalid requests."));
CHECK(plan.hasType(PlannedTestType::Negative), "expected negative tests");
PASS();
}
void test_pipeline_keywords_add_integration_tests() {
TEST(pipeline_keywords_add_integration_tests);
auto plan = TestPlanGenerator::generate(baseInput("Build workflow pipeline for dispatch."));
CHECK(plan.complexity == StepComplexity::Pipeline, "expected pipeline complexity");
CHECK(plan.hasType(PlannedTestType::Integration), "expected integration tests");
CHECK(plan.tests.size() >= 6 && plan.tests.size() <= 10, "pipeline budget mismatch");
PASS();
}
void test_rpc_mcp_keywords_add_smoke_tests() {
TEST(rpc_mcp_keywords_add_smoke_tests);
auto plan = TestPlanGenerator::generate(baseInput("Add MCP RPC handler and routing support."));
CHECK(plan.hasType(PlannedTestType::Smoke), "expected smoke test");
PASS();
}
void test_boundary_keywords_add_boundary_tests() {
TEST(boundary_keywords_add_boundary_tests);
auto plan = TestPlanGenerator::generate(baseInput("Compute score with max/min boundary limits."));
CHECK(plan.hasType(PlannedTestType::Boundary), "expected boundary test");
PASS();
}
void test_modification_keywords_add_regression_tests() {
TEST(modification_keywords_add_regression_tests);
auto plan = TestPlanGenerator::generate(baseInput("Modify existing scoring rules."));
CHECK(plan.hasType(PlannedTestType::Regression), "expected regression test");
PASS();
}
void test_prior_test_patterns_can_force_regression_tests() {
TEST(prior_test_patterns_can_force_regression_tests);
auto in = baseInput("Add parser utility.");
in.priorTests.push_back({"editor/tests/step123_test.cpp", "regression coverage"});
auto plan = TestPlanGenerator::generate(in);
CHECK(plan.hasType(PlannedTestType::Regression), "expected regression from prior pattern");
PASS();
}
void test_contract_keywords_add_contract_tests() {
TEST(contract_keywords_add_contract_tests);
auto plan = TestPlanGenerator::generate(baseInput("Enforce @Contract pre/post checks."));
CHECK(plan.hasType(PlannedTestType::Contract), "expected contract test");
PASS();
}
void test_property_keywords_add_property_tests() {
TEST(property_keywords_add_property_tests);
auto plan = TestPlanGenerator::generate(baseInput("Roundtrip serialize invariant for nodes."));
CHECK(plan.hasType(PlannedTestType::Property), "expected property test");
PASS();
}
void test_complex_hint_enforces_10_to_15_budget_boundary() {
TEST(complex_hint_enforces_10_to_15_budget_boundary);
auto in = baseInput("Simple line.");
in.useComplexityHint = true;
in.complexityHint = StepComplexity::Complex;
auto plan = TestPlanGenerator::generate(in);
CHECK(plan.tests.size() >= 10, "expected complex minimum");
CHECK(plan.tests.size() <= 15, "expected complex maximum");
PASS();
}
void test_test_names_use_step_prefix_convention() {
TEST(test_names_use_step_prefix_convention);
auto plan = TestPlanGenerator::generate(baseInput("Validate behavior."));
CHECK(!plan.tests.empty(), "expected test stubs");
CHECK(plan.tests[0].name.find("test_step486_") == 0, "unexpected test name prefix");
PASS();
}
int main() {
std::cout << "Step 486: Test Plan Generator Tests\n";
test_validation_keywords_add_negative_tests(); // 1
test_pipeline_keywords_add_integration_tests(); // 2
test_rpc_mcp_keywords_add_smoke_tests(); // 3
test_boundary_keywords_add_boundary_tests(); // 4
test_modification_keywords_add_regression_tests(); // 5
test_prior_test_patterns_can_force_regression_tests(); // 6
test_contract_keywords_add_contract_tests(); // 7
test_property_keywords_add_property_tests(); // 8
test_complex_hint_enforces_10_to_15_budget_boundary(); // 9
test_test_names_use_step_prefix_convention(); // 10
std::cout << "\nResults: " << passed << "/" << (passed + failed)
<< " passed\n";
return failed == 0 ? 0 : 1;
}

42
progress.md
View File

@@ -7030,3 +7030,45 @@ budget, and escalate unresolved submissions to architect review.
- `editor/src/WorkerDispatchValidator.h` within header-size limit (`145` <= `600`)
- `editor/tests/step485_test.cpp` within test-file size guidance (`165` lines)
- Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md`
### Step 486: Test Plan Generator
**Status:** PASS (10/10 tests)
Implements the test-plan intelligence layer for Phase 23c: proposes typed,
named tests from step semantics and complexity budgets instead of fixed
test counts.
**Files added:**
- `editor/src/TestPlanGenerator.h` — adaptive test-plan engine:
- `TestPlanInput`, `GeneratedTestPlan`, `TestPatternSample`
- typed test proposal across taxonomy:
- unit / negative / integration / contract / regression / boundary /
property / smoke / performance
- keyword heuristics aligned to sprint plan:
- validate/detect -> negative
- pipeline/workflow -> integration
- RPC/MCP -> smoke
- score/boundary/max/min -> boundary
- modify-existing -> regression
- prior-test pattern influence for regression continuity
- complexity-aware budget enforcement:
- trivial: 1-4
- standard: 5-10
- complex: 10-15
- pipeline: 6-10
- `editor/tests/step486_test.cpp` — 10 tests (unit/boundary/negative focus):
- heuristic inclusion checks by keyword class
- regression inference from prior test patterns
- budget range checks under explicit complexity hint
- generated test name convention checks
- `editor/CMakeLists.txt` — `step486_test` target
**Verification run:**
- `cmake --build editor/build-native --target step486_test step485_test` — PASS
- `./editor/build-native/step486_test` — PASS (10/10)
- `./editor/build-native/step485_test` — PASS (10/10) regression coverage
**Architecture gate check:**
- `editor/src/TestPlanGenerator.h` within header-size limit (`191` <= `600`)
- `editor/tests/step486_test.cpp` within test-file size guidance (`118` lines)
- Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md`