Step 400: add self-hosting test harness with coverage tracking
Infrastructure to feed Whetstone .h files through the C++ parser pipeline. Tracks construct coverage (classes, functions, namespaces, type aliases), graceful degradation for unparseable sections, and coverage reporting. 12/12 tests passing. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Bill
@@ -2479,4 +2479,13 @@ target_link_libraries(step399_test PRIVATE
|
||||
tree_sitter_javascript tree_sitter_typescript
|
||||
tree_sitter_java tree_sitter_rust tree_sitter_go)
|
||||
|
||||
add_executable(step400_test tests/step400_test.cpp)
|
||||
target_include_directories(step400_test PRIVATE src)
|
||||
target_link_libraries(step400_test PRIVATE
|
||||
nlohmann_json::nlohmann_json
|
||||
unofficial::tree-sitter::tree-sitter
|
||||
tree_sitter_python tree_sitter_cpp tree_sitter_elisp
|
||||
tree_sitter_javascript tree_sitter_typescript
|
||||
tree_sitter_java tree_sitter_rust tree_sitter_go)
|
||||
|
||||
# Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies)
|
||||
|
||||
308
editor/src/SelfHostHarness.h
Normal file
308
editor/src/SelfHostHarness.h
Normal file
@@ -0,0 +1,308 @@
|
||||
#pragma once
|
||||
// Step 400: Self-Hosting Test Harness
|
||||
//
|
||||
// Infrastructure to feed actual Whetstone .h files through the pipeline.
|
||||
// Compare parsed AST against expected structure. Track coverage.
|
||||
// Graceful degradation: unparseable sections become opaque blocks, not errors.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
#include <map>
|
||||
#include <fstream>
|
||||
#include <sstream>
|
||||
#include <filesystem>
|
||||
#include "ast/ASTNode.h"
|
||||
#include "ast/Module.h"
|
||||
#include "ast/Function.h"
|
||||
#include "ast/Parser.h"
|
||||
#include "ast/ClassDeclaration.h"
|
||||
#include "ast/EnumNamespaceNodes.h"
|
||||
#include "ast/CppAdvancedNodes.h"
|
||||
|
||||
// --- Coverage tracking ---
|
||||
|
||||
struct ConstructCoverage {
|
||||
std::string constructType; // e.g. "class", "function", "struct", "enum"
|
||||
int expected = 0;
|
||||
int parsed = 0;
|
||||
int opaque = 0; // unparseable sections that became opaque blocks
|
||||
|
||||
double ratio() const {
|
||||
return expected > 0 ? static_cast<double>(parsed) / expected : 1.0;
|
||||
}
|
||||
};
|
||||
|
||||
struct SelfHostResult {
|
||||
std::string filePath;
|
||||
bool parseSuccess = false;
|
||||
std::unique_ptr<Module> ast;
|
||||
std::vector<ConstructCoverage> coverage;
|
||||
std::vector<std::string> warnings;
|
||||
int totalExpected = 0;
|
||||
int totalParsed = 0;
|
||||
int totalOpaque = 0;
|
||||
|
||||
double overallCoverage() const {
|
||||
return totalExpected > 0 ? static_cast<double>(totalParsed) / totalExpected : 1.0;
|
||||
}
|
||||
};
|
||||
|
||||
// --- Expected structure declaration ---
|
||||
|
||||
struct ExpectedConstruct {
|
||||
std::string type; // "class", "function", "struct", "namespace", "type_alias"
|
||||
std::string name; // expected name (empty = don't check)
|
||||
};
|
||||
|
||||
struct ExpectedStructure {
|
||||
std::string filePath;
|
||||
std::vector<ExpectedConstruct> constructs;
|
||||
};
|
||||
|
||||
// --- Self-Hosting Harness ---
|
||||
|
||||
class SelfHostHarness {
|
||||
public:
|
||||
|
||||
// Read a file from disk and return its contents
|
||||
static std::string readFile(const std::string& path) {
|
||||
std::ifstream ifs(path);
|
||||
if (!ifs.is_open()) return "";
|
||||
std::ostringstream ss;
|
||||
ss << ifs.rdbuf();
|
||||
return ss.str();
|
||||
}
|
||||
|
||||
// Parse a Whetstone .h file through the C++ parser
|
||||
static SelfHostResult parseFile(const std::string& path) {
|
||||
SelfHostResult result;
|
||||
result.filePath = path;
|
||||
|
||||
std::string source = readFile(path);
|
||||
if (source.empty()) {
|
||||
result.warnings.push_back("File not found or empty: " + path);
|
||||
return result;
|
||||
}
|
||||
|
||||
result.ast = TreeSitterParser::parseCpp(source);
|
||||
result.parseSuccess = (result.ast != nullptr);
|
||||
|
||||
if (!result.parseSuccess) {
|
||||
result.warnings.push_back("Parse returned null for: " + path);
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
// Parse source string directly (for testing without file I/O)
|
||||
static SelfHostResult parseSource(const std::string& source,
|
||||
const std::string& label = "<inline>") {
|
||||
SelfHostResult result;
|
||||
result.filePath = label;
|
||||
|
||||
if (source.empty()) {
|
||||
result.warnings.push_back("Empty source provided");
|
||||
return result;
|
||||
}
|
||||
|
||||
result.ast = TreeSitterParser::parseCpp(source);
|
||||
result.parseSuccess = (result.ast != nullptr);
|
||||
return result;
|
||||
}
|
||||
|
||||
// Compare parsed AST against expected structure
|
||||
static void checkExpected(SelfHostResult& result,
|
||||
const ExpectedStructure& expected) {
|
||||
if (!result.ast) {
|
||||
result.totalExpected = static_cast<int>(expected.constructs.size());
|
||||
result.totalOpaque = result.totalExpected;
|
||||
return;
|
||||
}
|
||||
|
||||
// Group expected constructs by type
|
||||
std::map<std::string, std::vector<std::string>> byType;
|
||||
for (const auto& c : expected.constructs) {
|
||||
byType[c.type].push_back(c.name);
|
||||
}
|
||||
|
||||
result.totalExpected = static_cast<int>(expected.constructs.size());
|
||||
|
||||
for (const auto& [type, names] : byType) {
|
||||
ConstructCoverage cov;
|
||||
cov.constructType = type;
|
||||
cov.expected = static_cast<int>(names.size());
|
||||
|
||||
if (type == "class" || type == "struct") {
|
||||
auto& classes = result.ast->getChildren("classes");
|
||||
for (const auto& name : names) {
|
||||
bool found = false;
|
||||
for (auto* c : classes) {
|
||||
auto* cls = static_cast<ClassDeclaration*>(c);
|
||||
if (name.empty() || cls->name == name) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (found) cov.parsed++;
|
||||
else cov.opaque++;
|
||||
}
|
||||
}
|
||||
else if (type == "function") {
|
||||
auto& fns = result.ast->getChildren("functions");
|
||||
for (const auto& name : names) {
|
||||
bool found = false;
|
||||
for (auto* f : fns) {
|
||||
auto* fn = static_cast<Function*>(f);
|
||||
if (name.empty() || fn->name == name) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (found) cov.parsed++;
|
||||
else cov.opaque++;
|
||||
}
|
||||
}
|
||||
else if (type == "namespace") {
|
||||
auto& stmts = result.ast->getChildren("statements");
|
||||
for (const auto& name : names) {
|
||||
bool found = false;
|
||||
for (auto* s : stmts) {
|
||||
if (s->conceptType == "NamespaceDeclaration") {
|
||||
auto* ns = static_cast<NamespaceDeclaration*>(s);
|
||||
if (name.empty() || ns->name == name) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (found) cov.parsed++;
|
||||
else cov.opaque++;
|
||||
}
|
||||
}
|
||||
else if (type == "type_alias") {
|
||||
auto& stmts = result.ast->getChildren("statements");
|
||||
for (const auto& name : names) {
|
||||
bool found = false;
|
||||
for (auto* s : stmts) {
|
||||
if (s->conceptType == "TypeAlias") {
|
||||
auto* ta = static_cast<TypeAlias*>(s);
|
||||
if (name.empty() || ta->aliasName == name) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (found) cov.parsed++;
|
||||
else cov.opaque++;
|
||||
}
|
||||
}
|
||||
else if (type == "enum") {
|
||||
auto& stmts = result.ast->getChildren("statements");
|
||||
for (const auto& name : names) {
|
||||
bool found = false;
|
||||
for (auto* s : stmts) {
|
||||
if (s->conceptType == "EnumDeclaration") {
|
||||
auto* en = static_cast<EnumDeclaration*>(s);
|
||||
if (name.empty() || en->name == name) {
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (found) cov.parsed++;
|
||||
else cov.opaque++;
|
||||
}
|
||||
}
|
||||
else {
|
||||
// Unknown type — mark all as opaque
|
||||
cov.opaque = cov.expected;
|
||||
result.warnings.push_back("Unknown construct type: " + type);
|
||||
}
|
||||
|
||||
result.totalParsed += cov.parsed;
|
||||
result.totalOpaque += cov.opaque;
|
||||
result.coverage.push_back(std::move(cov));
|
||||
}
|
||||
}
|
||||
|
||||
// Count all classes found in parsed AST
|
||||
static int countClasses(const Module* mod) {
|
||||
if (!mod) return 0;
|
||||
return static_cast<int>(mod->getChildren("classes").size());
|
||||
}
|
||||
|
||||
// Count all functions found in parsed AST
|
||||
static int countFunctions(const Module* mod) {
|
||||
if (!mod) return 0;
|
||||
return static_cast<int>(mod->getChildren("functions").size());
|
||||
}
|
||||
|
||||
// Count all statements of a specific concept type
|
||||
static int countStatements(const Module* mod, const std::string& conceptType) {
|
||||
if (!mod) return 0;
|
||||
int count = 0;
|
||||
for (auto* s : mod->getChildren("statements")) {
|
||||
if (s->conceptType == conceptType) count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
// Get list of all class names
|
||||
static std::vector<std::string> getClassNames(const Module* mod) {
|
||||
std::vector<std::string> names;
|
||||
if (!mod) return names;
|
||||
for (auto* c : mod->getChildren("classes")) {
|
||||
names.push_back(static_cast<ClassDeclaration*>(c)->name);
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
// Get list of all function names
|
||||
static std::vector<std::string> getFunctionNames(const Module* mod) {
|
||||
std::vector<std::string> names;
|
||||
if (!mod) return names;
|
||||
for (auto* f : mod->getChildren("functions")) {
|
||||
names.push_back(static_cast<Function*>(f)->name);
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
// Check if a specific construct was parsed (by type and name)
|
||||
static bool hasParsedConstruct(const Module* mod,
|
||||
const std::string& type,
|
||||
const std::string& name) {
|
||||
if (!mod) return false;
|
||||
if (type == "class" || type == "struct") {
|
||||
for (auto* c : mod->getChildren("classes")) {
|
||||
if (static_cast<ClassDeclaration*>(c)->name == name) return true;
|
||||
}
|
||||
} else if (type == "function") {
|
||||
for (auto* f : mod->getChildren("functions")) {
|
||||
if (static_cast<Function*>(f)->name == name) return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// Generate a coverage report string
|
||||
static std::string coverageReport(const SelfHostResult& result) {
|
||||
std::ostringstream out;
|
||||
out << "Self-Host Report: " << result.filePath << "\n";
|
||||
out << " Parse success: " << (result.parseSuccess ? "yes" : "no") << "\n";
|
||||
out << " Overall coverage: " << (result.overallCoverage() * 100) << "%"
|
||||
<< " (" << result.totalParsed << "/" << result.totalExpected << ")\n";
|
||||
|
||||
for (const auto& cov : result.coverage) {
|
||||
out << " " << cov.constructType << ": "
|
||||
<< cov.parsed << "/" << cov.expected
|
||||
<< " (" << (cov.ratio() * 100) << "%)";
|
||||
if (cov.opaque > 0) out << " [" << cov.opaque << " opaque]";
|
||||
out << "\n";
|
||||
}
|
||||
|
||||
for (const auto& w : result.warnings) {
|
||||
out << " WARNING: " << w << "\n";
|
||||
}
|
||||
return out.str();
|
||||
}
|
||||
};
|
||||
258
editor/tests/step400_test.cpp
Normal file
258
editor/tests/step400_test.cpp
Normal file
@@ -0,0 +1,258 @@
|
||||
// Step 400: Self-Hosting Test Harness (12 tests)
|
||||
|
||||
#include <cassert>
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include "SelfHostHarness.h"
|
||||
|
||||
int main() {
|
||||
int passed = 0;
|
||||
|
||||
// Test 1: Parse simple struct from source string
|
||||
{
|
||||
std::string src = R"(
|
||||
struct Foo {
|
||||
int x;
|
||||
std::string name;
|
||||
};
|
||||
)";
|
||||
auto result = SelfHostHarness::parseSource(src, "test_struct");
|
||||
assert(result.parseSuccess);
|
||||
assert(result.ast != nullptr);
|
||||
std::cout << "PASS: test 1 — parse simple struct\n";
|
||||
passed++;
|
||||
}
|
||||
|
||||
// Test 2: Parse class + free function from source string
|
||||
{
|
||||
std::string src = R"(
|
||||
class Widget {
|
||||
public:
|
||||
void draw() { }
|
||||
};
|
||||
|
||||
void process(Widget* w) {
|
||||
w->draw();
|
||||
}
|
||||
)";
|
||||
auto result = SelfHostHarness::parseSource(src, "test_class_func");
|
||||
assert(result.parseSuccess);
|
||||
assert(SelfHostHarness::countClasses(result.ast.get()) >= 1);
|
||||
assert(SelfHostHarness::countFunctions(result.ast.get()) >= 1);
|
||||
std::cout << "PASS: test 2 — parse class + free function\n";
|
||||
passed++;
|
||||
}
|
||||
|
||||
// Test 3: Coverage tracking — all constructs found
|
||||
{
|
||||
std::string src = R"(
|
||||
struct Point { int x; int y; };
|
||||
void doSomething() { }
|
||||
)";
|
||||
auto result = SelfHostHarness::parseSource(src, "test_coverage");
|
||||
ExpectedStructure expected;
|
||||
expected.filePath = "test_coverage";
|
||||
expected.constructs = {
|
||||
{"struct", "Point"},
|
||||
{"function", "doSomething"}
|
||||
};
|
||||
SelfHostHarness::checkExpected(result, expected);
|
||||
assert(result.totalExpected == 2);
|
||||
assert(result.totalParsed == 2);
|
||||
assert(result.totalOpaque == 0);
|
||||
assert(result.overallCoverage() == 1.0);
|
||||
std::cout << "PASS: test 3 — coverage tracking, all found\n";
|
||||
passed++;
|
||||
}
|
||||
|
||||
// Test 4: Coverage tracking — missing construct is opaque
|
||||
{
|
||||
std::string src = R"(
|
||||
void foo() { }
|
||||
)";
|
||||
auto result = SelfHostHarness::parseSource(src, "test_missing");
|
||||
ExpectedStructure expected;
|
||||
expected.filePath = "test_missing";
|
||||
expected.constructs = {
|
||||
{"function", "foo"},
|
||||
{"class", "MissingClass"}
|
||||
};
|
||||
SelfHostHarness::checkExpected(result, expected);
|
||||
assert(result.totalExpected == 2);
|
||||
assert(result.totalParsed == 1);
|
||||
assert(result.totalOpaque == 1);
|
||||
assert(result.overallCoverage() == 0.5);
|
||||
std::cout << "PASS: test 4 — missing construct is opaque\n";
|
||||
passed++;
|
||||
}
|
||||
|
||||
// Test 5: Coverage report generation
|
||||
{
|
||||
std::string src = R"(
|
||||
class Foo { };
|
||||
void bar() { }
|
||||
)";
|
||||
auto result = SelfHostHarness::parseSource(src, "report_test");
|
||||
ExpectedStructure expected;
|
||||
expected.filePath = "report_test";
|
||||
expected.constructs = {
|
||||
{"class", "Foo"},
|
||||
{"function", "bar"}
|
||||
};
|
||||
SelfHostHarness::checkExpected(result, expected);
|
||||
std::string report = SelfHostHarness::coverageReport(result);
|
||||
assert(!report.empty());
|
||||
assert(report.find("report_test") != std::string::npos);
|
||||
assert(report.find("100%") != std::string::npos);
|
||||
std::cout << "PASS: test 5 — coverage report generation\n";
|
||||
passed++;
|
||||
}
|
||||
|
||||
// Test 6: Graceful degradation — empty source doesn't crash
|
||||
{
|
||||
auto result = SelfHostHarness::parseSource("", "empty");
|
||||
assert(!result.parseSuccess || result.ast == nullptr ||
|
||||
result.warnings.size() > 0);
|
||||
std::cout << "PASS: test 6 — graceful degradation on empty source\n";
|
||||
passed++;
|
||||
}
|
||||
|
||||
// Test 7: hasParsedConstruct lookup
|
||||
{
|
||||
std::string src = R"(
|
||||
class Pipeline { };
|
||||
void run() { }
|
||||
)";
|
||||
auto result = SelfHostHarness::parseSource(src, "lookup");
|
||||
assert(SelfHostHarness::hasParsedConstruct(result.ast.get(), "class", "Pipeline"));
|
||||
assert(SelfHostHarness::hasParsedConstruct(result.ast.get(), "function", "run"));
|
||||
assert(!SelfHostHarness::hasParsedConstruct(result.ast.get(), "class", "Missing"));
|
||||
std::cout << "PASS: test 7 — hasParsedConstruct lookup\n";
|
||||
passed++;
|
||||
}
|
||||
|
||||
// Test 8: getClassNames and getFunctionNames
|
||||
{
|
||||
std::string src = R"(
|
||||
class Alpha { };
|
||||
class Beta { };
|
||||
void one() { }
|
||||
void two() { }
|
||||
)";
|
||||
auto result = SelfHostHarness::parseSource(src, "names");
|
||||
auto classNames = SelfHostHarness::getClassNames(result.ast.get());
|
||||
auto funcNames = SelfHostHarness::getFunctionNames(result.ast.get());
|
||||
assert(classNames.size() >= 2);
|
||||
assert(funcNames.size() >= 2);
|
||||
std::cout << "PASS: test 8 — getClassNames and getFunctionNames\n";
|
||||
passed++;
|
||||
}
|
||||
|
||||
// Test 9: Parse Whetstone-style header pattern (struct + inline function)
|
||||
{
|
||||
std::string src = R"(
|
||||
#pragma once
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
struct ConflictInfo {
|
||||
std::string type1;
|
||||
std::string type2;
|
||||
std::string message;
|
||||
};
|
||||
|
||||
inline void checkConflicts(const std::vector<ConflictInfo>& items) {
|
||||
for (const auto& item : items) {
|
||||
// process
|
||||
}
|
||||
}
|
||||
)";
|
||||
auto result = SelfHostHarness::parseSource(src, "whetstone_pattern");
|
||||
assert(result.parseSuccess);
|
||||
// Should find at least the struct and the function
|
||||
ExpectedStructure expected;
|
||||
expected.constructs = {
|
||||
{"struct", "ConflictInfo"},
|
||||
{"function", "checkConflicts"}
|
||||
};
|
||||
SelfHostHarness::checkExpected(result, expected);
|
||||
assert(result.totalParsed >= 1); // At least struct or function found
|
||||
std::cout << "PASS: test 9 — Whetstone-style header pattern\n";
|
||||
passed++;
|
||||
}
|
||||
|
||||
// Test 10: Type alias coverage tracking
|
||||
{
|
||||
std::string src = R"(
|
||||
using json = nlohmann::json;
|
||||
using StringVec = std::vector<std::string>;
|
||||
)";
|
||||
auto result = SelfHostHarness::parseSource(src, "alias_test");
|
||||
assert(result.parseSuccess);
|
||||
int aliases = SelfHostHarness::countStatements(result.ast.get(), "TypeAlias");
|
||||
assert(aliases == 2);
|
||||
|
||||
ExpectedStructure expected;
|
||||
expected.constructs = {
|
||||
{"type_alias", "json"},
|
||||
{"type_alias", "StringVec"}
|
||||
};
|
||||
SelfHostHarness::checkExpected(result, expected);
|
||||
assert(result.totalParsed == 2);
|
||||
std::cout << "PASS: test 10 — type alias coverage tracking\n";
|
||||
passed++;
|
||||
}
|
||||
|
||||
// Test 11: Namespace coverage tracking
|
||||
{
|
||||
std::string src = R"(
|
||||
namespace detail {
|
||||
void helper() { }
|
||||
}
|
||||
)";
|
||||
auto result = SelfHostHarness::parseSource(src, "ns_test");
|
||||
assert(result.parseSuccess);
|
||||
|
||||
ExpectedStructure expected;
|
||||
expected.constructs = {
|
||||
{"namespace", "detail"}
|
||||
};
|
||||
SelfHostHarness::checkExpected(result, expected);
|
||||
assert(result.totalParsed == 1);
|
||||
std::cout << "PASS: test 11 — namespace coverage tracking\n";
|
||||
passed++;
|
||||
}
|
||||
|
||||
// Test 12: Mixed coverage with partial success
|
||||
{
|
||||
std::string src = R"(
|
||||
class Validator {
|
||||
public:
|
||||
void validate() { }
|
||||
};
|
||||
|
||||
void helperFunc() { }
|
||||
)";
|
||||
auto result = SelfHostHarness::parseSource(src, "mixed_test");
|
||||
ExpectedStructure expected;
|
||||
expected.constructs = {
|
||||
{"class", "Validator"},
|
||||
{"function", "helperFunc"},
|
||||
{"class", "NonExistent"},
|
||||
{"function", "ghostFunc"}
|
||||
};
|
||||
SelfHostHarness::checkExpected(result, expected);
|
||||
assert(result.totalExpected == 4);
|
||||
assert(result.totalParsed == 2);
|
||||
assert(result.totalOpaque == 2);
|
||||
assert(result.overallCoverage() == 0.5);
|
||||
|
||||
std::string report = SelfHostHarness::coverageReport(result);
|
||||
assert(report.find("50%") != std::string::npos);
|
||||
std::cout << "PASS: test 12 — mixed coverage with partial success\n";
|
||||
passed++;
|
||||
}
|
||||
|
||||
std::cout << "\nStep 400 result: " << passed << "/12 tests passed\n";
|
||||
return (passed == 12) ? 0 : 1;
|
||||
}
|
||||
Reference in New Issue
Block a user