diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index 2a4604a..16292d7 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -3352,4 +3352,13 @@ target_link_libraries(step496_test PRIVATE tree_sitter_javascript tree_sitter_typescript tree_sitter_java tree_sitter_rust tree_sitter_go) +add_executable(step497_test tests/step497_test.cpp) +target_include_directories(step497_test PRIVATE src) +target_link_libraries(step497_test PRIVATE + nlohmann_json::nlohmann_json + unofficial::tree-sitter::tree-sitter + tree_sitter_python tree_sitter_cpp tree_sitter_elisp + tree_sitter_javascript tree_sitter_typescript + tree_sitter_java tree_sitter_rust tree_sitter_go) + # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies) diff --git a/editor/src/SelfHostingMetricsReport.h b/editor/src/SelfHostingMetricsReport.h new file mode 100644 index 0000000..12aa309 --- /dev/null +++ b/editor/src/SelfHostingMetricsReport.h @@ -0,0 +1,96 @@ +#pragma once + +// Step 497: Self-Hosting Metrics Report +// Aggregates parse, annotation, transpilation, and workflow viability metrics. + +#include "SelfHostAnnotationAudit.h" +#include "SelfHostCodebaseAudit.h" +#include "SelfHostTranspileAudit.h" +#include "SelfModernizationWorkflow.h" + +#include +#include + +struct SelfHostingMetrics { + int filesParsed = 0; + int filesTotal = 0; + double parseCoveragePercent = 0.0; + + int inferredAnnotations = 0; + int annotationSignals = 0; + double annotationSignalDensity = 0.0; + + float averageTranspileConfidence = 0.0f; + int transpileReviewRequired = 0; + int idiomaticTranslations = 0; + + int workflowTasks = 0; + int workflowDeterministicValid = 0; + bool workflowViable = false; + + std::vector gaps; + std::vector notes; +}; + +class SelfHostingMetricsReport { +public: + static SelfHostingMetrics generate(const CodebaseParseAudit& parseAudit, + const SelfAnnotationAuditReport& annotationAudit, + const SelfTranspileAuditReport& transpileAudit, + const ModernizationWorkflowReport& workflowAudit) { + SelfHostingMetrics out; + + out.filesParsed = parseAudit.parsedFiles; + out.filesTotal = parseAudit.totalFiles; + out.parseCoveragePercent = parseAudit.parseRate() * 100.0; + + out.inferredAnnotations = annotationAudit.inferredTotal; + out.annotationSignals = annotationAudit.complexityCount + annotationAudit.riskCount + + annotationAudit.ownerCount + annotationAudit.intentCount + + annotationAudit.tailCallCount; + if (annotationAudit.parsedFiles > 0) { + out.annotationSignalDensity = + static_cast(out.annotationSignals) / + static_cast(annotationAudit.parsedFiles); + } + + out.averageTranspileConfidence = transpileAudit.averageConfidence; + out.transpileReviewRequired = transpileAudit.reviewRequiredCount; + out.idiomaticTranslations = transpileAudit.idiomaticCount; + + out.workflowTasks = static_cast(workflowAudit.tasks.size()); + out.workflowDeterministicValid = workflowAudit.deterministicValidCount; + out.workflowViable = (workflowAudit.deterministicCount == workflowAudit.deterministicValidCount); + + fillGaps(out); + out.notes.push_back("Self-hosting metrics report generated"); + return out; + } + + static SelfHostingMetrics generateFromCurrentState() { + auto parse = SelfHostCodebaseAudit::auditDirectories({"editor/src", "editor/src/ast"}); + auto ann = SelfHostAnnotationAudit::auditDirectories({"editor/src", "editor/src/ast"}); + auto transpile = SelfHostTranspileAudit::runDefault(); + auto workflow = SelfModernizationWorkflow::createAndRun("editor/src/AnnotationValidator.h"); + return generate(parse, ann, transpile, workflow); + } + +private: + static void fillGaps(SelfHostingMetrics& out) { + if (out.parseCoveragePercent < 95.0) { + out.gaps.push_back("Parse coverage below 95% target"); + } + if (out.annotationSignalDensity < 1.0) { + out.gaps.push_back("Low annotation signal density"); + } + if (out.averageTranspileConfidence < 0.60f) { + out.gaps.push_back("Low average transpilation confidence"); + } + if (!out.workflowViable) { + out.gaps.push_back("Deterministic modernization outputs not fully valid"); + } + if (out.gaps.empty()) { + out.gaps.push_back("No blocking gaps detected"); + } + } +}; diff --git a/editor/tests/step497_test.cpp b/editor/tests/step497_test.cpp new file mode 100644 index 0000000..5d83d8e --- /dev/null +++ b/editor/tests/step497_test.cpp @@ -0,0 +1,165 @@ +// Step 497: Self-Hosting Metrics Report Tests (12 tests) + +#include "SelfHostingMetricsReport.h" + +#include + +static int passed = 0, failed = 0; +#define TEST(name) { std::cout << " " << #name << "... "; } +#define PASS() { std::cout << "PASS\n"; ++passed; } +#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; } +#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {} + +static SelfHostingMetrics runNow() { + return SelfHostingMetricsReport::generateFromCurrentState(); +} + +void test_report_populates_parse_coverage_metrics() { + TEST(report_populates_parse_coverage_metrics); + auto r = runNow(); + CHECK(r.filesTotal > 0, "filesTotal should be > 0"); + CHECK(r.filesParsed >= 0, "filesParsed should be >= 0"); + CHECK(r.parseCoveragePercent >= 0.0, "parse coverage should be non-negative"); + PASS(); +} + +void test_report_populates_annotation_metrics() { + TEST(report_populates_annotation_metrics); + auto r = runNow(); + CHECK(r.inferredAnnotations >= 0, "inferredAnnotations invalid"); + CHECK(r.annotationSignals >= 0, "annotationSignals invalid"); + CHECK(r.annotationSignalDensity >= 0.0, "annotation density invalid"); + PASS(); +} + +void test_report_populates_transpilation_metrics() { + TEST(report_populates_transpilation_metrics); + auto r = runNow(); + CHECK(r.averageTranspileConfidence >= 0.0f && r.averageTranspileConfidence <= 1.0f, + "confidence out of range"); + CHECK(r.transpileReviewRequired >= 0, "review count invalid"); + PASS(); +} + +void test_report_populates_workflow_viability_metrics() { + TEST(report_populates_workflow_viability_metrics); + auto r = runNow(); + CHECK(r.workflowTasks >= 0, "workflow task count invalid"); + CHECK(r.workflowDeterministicValid >= 0, "workflow deterministic valid invalid"); + PASS(); +} + +void test_gap_analysis_flags_parse_coverage_below_target() { + TEST(gap_analysis_flags_parse_coverage_below_target); + CodebaseParseAudit p; p.totalFiles = 100; p.parsedFiles = 90; + SelfAnnotationAuditReport a; + SelfTranspileAuditReport t; + ModernizationWorkflowReport w; + auto r = SelfHostingMetricsReport::generate(p, a, t, w); + bool found = false; + for (const auto& g : r.gaps) if (g.find("Parse coverage below 95%") != std::string::npos) found = true; + CHECK(found, "expected parse-coverage gap"); + PASS(); +} + +void test_gap_analysis_flags_low_annotation_density() { + TEST(gap_analysis_flags_low_annotation_density); + CodebaseParseAudit p; p.totalFiles = 10; p.parsedFiles = 10; + SelfAnnotationAuditReport a; a.parsedFiles = 10; a.complexityCount = 1; + SelfTranspileAuditReport t; t.averageConfidence = 0.9f; + ModernizationWorkflowReport w; w.deterministicCount = 1; w.deterministicValidCount = 1; + auto r = SelfHostingMetricsReport::generate(p, a, t, w); + bool found = false; + for (const auto& g : r.gaps) if (g.find("annotation signal density") != std::string::npos) found = true; + CHECK(found, "expected annotation density gap"); + PASS(); +} + +void test_gap_analysis_flags_low_transpilation_confidence() { + TEST(gap_analysis_flags_low_transpilation_confidence); + CodebaseParseAudit p; p.totalFiles = 10; p.parsedFiles = 10; + SelfAnnotationAuditReport a; a.parsedFiles = 10; a.complexityCount = 20; + SelfTranspileAuditReport t; t.averageConfidence = 0.2f; + ModernizationWorkflowReport w; w.deterministicCount = 1; w.deterministicValidCount = 1; + auto r = SelfHostingMetricsReport::generate(p, a, t, w); + bool found = false; + for (const auto& g : r.gaps) if (g.find("transpilation confidence") != std::string::npos) found = true; + CHECK(found, "expected transpilation confidence gap"); + PASS(); +} + +void test_gap_analysis_flags_nonviable_workflow() { + TEST(gap_analysis_flags_nonviable_workflow); + CodebaseParseAudit p; p.totalFiles = 10; p.parsedFiles = 10; + SelfAnnotationAuditReport a; a.parsedFiles = 10; a.complexityCount = 20; + SelfTranspileAuditReport t; t.averageConfidence = 0.9f; + ModernizationWorkflowReport w; w.deterministicCount = 3; w.deterministicValidCount = 1; + auto r = SelfHostingMetricsReport::generate(p, a, t, w); + bool found = false; + for (const auto& g : r.gaps) if (g.find("modernization outputs") != std::string::npos) found = true; + CHECK(found, "expected workflow viability gap"); + PASS(); +} + +void test_no_blocking_gaps_message_when_all_thresholds_pass() { + TEST(no_blocking_gaps_message_when_all_thresholds_pass); + CodebaseParseAudit p; p.totalFiles = 10; p.parsedFiles = 10; + SelfAnnotationAuditReport a; a.parsedFiles = 10; a.complexityCount = 20; + SelfTranspileAuditReport t; t.averageConfidence = 0.9f; + ModernizationWorkflowReport w; w.deterministicCount = 2; w.deterministicValidCount = 2; + auto r = SelfHostingMetricsReport::generate(p, a, t, w); + CHECK(r.gaps.size() == 1, "expected single non-blocking gap message"); + CHECK(r.gaps[0].find("No blocking gaps") != std::string::npos, "missing no-gap message"); + PASS(); +} + +void test_generated_report_includes_summary_note() { + TEST(generated_report_includes_summary_note); + auto r = runNow(); + CHECK(!r.notes.empty(), "expected notes"); + CHECK(r.notes[0].find("metrics report generated") != std::string::npos, "missing summary note"); + PASS(); +} + +void test_parse_coverage_percent_matches_ratio() { + TEST(parse_coverage_percent_matches_ratio); + CodebaseParseAudit p; p.totalFiles = 4; p.parsedFiles = 3; + SelfAnnotationAuditReport a; a.parsedFiles = 4; + SelfTranspileAuditReport t; + ModernizationWorkflowReport w; + auto r = SelfHostingMetricsReport::generate(p, a, t, w); + CHECK(r.parseCoveragePercent > 74.9 && r.parseCoveragePercent < 75.1, "expected 75%"); + PASS(); +} + +void test_workflow_viable_true_when_all_deterministic_outputs_valid() { + TEST(workflow_viable_true_when_all_deterministic_outputs_valid); + CodebaseParseAudit p; p.totalFiles = 10; p.parsedFiles = 10; + SelfAnnotationAuditReport a; a.parsedFiles = 10; a.complexityCount = 20; + SelfTranspileAuditReport t; t.averageConfidence = 0.9f; + ModernizationWorkflowReport w; w.deterministicCount = 4; w.deterministicValidCount = 4; + auto r = SelfHostingMetricsReport::generate(p, a, t, w); + CHECK(r.workflowViable, "workflow should be viable"); + PASS(); +} + +int main() { + std::cout << "Step 497: Self-Hosting Metrics Report Tests\n"; + + test_report_populates_parse_coverage_metrics(); // 1 + test_report_populates_annotation_metrics(); // 2 + test_report_populates_transpilation_metrics(); // 3 + test_report_populates_workflow_viability_metrics(); // 4 + test_gap_analysis_flags_parse_coverage_below_target(); // 5 + test_gap_analysis_flags_low_annotation_density(); // 6 + test_gap_analysis_flags_low_transpilation_confidence(); // 7 + test_gap_analysis_flags_nonviable_workflow(); // 8 + test_no_blocking_gaps_message_when_all_thresholds_pass(); // 9 + test_generated_report_includes_summary_note(); // 10 + test_parse_coverage_percent_matches_ratio(); // 11 + test_workflow_viable_true_when_all_deterministic_outputs_valid(); // 12 + + std::cout << "\nResults: " << passed << "/" << (passed + failed) + << " passed\n"; + return failed == 0 ? 0 : 1; +} diff --git a/progress.md b/progress.md index 9646f99..b2484c9 100644 --- a/progress.md +++ b/progress.md @@ -7511,3 +7511,48 @@ validity checks. - `editor/src/SelfModernizationWorkflow.h` within header-size limit (`157` <= `600`) - `editor/tests/step496_test.cpp` within test-file size guidance (`159` lines) - Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md` + +### Step 497: Self-Hosting Metrics Report +**Status:** PASS (12/12 tests) + +Adds a consolidated self-hosting metrics layer that aggregates parse coverage, +annotation signals, transpilation quality, and workflow viability into a +single report with gap analysis. + +**Files added:** +- `editor/src/SelfHostingMetricsReport.h` — metrics aggregation/reporting: + - `SelfHostingMetrics` model: + - parse coverage metrics + - annotation signal metrics/density + - transpilation confidence + review pressure + - workflow task/viability metrics + - gap analysis + notes + - `generate(...)` for composed report from: + - `CodebaseParseAudit` + - `SelfAnnotationAuditReport` + - `SelfTranspileAuditReport` + - `ModernizationWorkflowReport` + - `generateFromCurrentState()` convenience path that runs all upstream + self-host phase components + - threshold-based gap detection: + - parse coverage < 95% + - low annotation density + - low transpilation confidence + - non-viable deterministic modernization outputs +- `editor/tests/step497_test.cpp` — 12 tests covering: + - population of all metric categories + - deterministic gap-analysis behavior under synthetic thresholds + - no-blocking-gap path when all thresholds pass + - parse-coverage ratio math correctness + - workflow viability signal semantics +- `editor/CMakeLists.txt` — `step497_test` target + +**Verification run:** +- `cmake --build editor/build-native --target step497_test step496_test` — PASS +- `./editor/build-native/step497_test` — PASS (12/12) +- `./editor/build-native/step496_test` — PASS (12/12) regression coverage + +**Architecture gate check:** +- `editor/src/SelfHostingMetricsReport.h` within header-size limit (`96` <= `600`) +- `editor/tests/step497_test.cpp` within test-file size guidance (`165` lines) +- Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md`