diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index fc07cf3..4732c16 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -2983,4 +2983,13 @@ target_link_libraries(step455_test PRIVATE tree_sitter_javascript tree_sitter_typescript tree_sitter_java tree_sitter_rust tree_sitter_go) +add_executable(step456_test tests/step456_test.cpp) +target_include_directories(step456_test PRIVATE src) +target_link_libraries(step456_test PRIVATE + nlohmann_json::nlohmann_json + unofficial::tree-sitter::tree-sitter + tree_sitter_python tree_sitter_cpp tree_sitter_elisp + tree_sitter_javascript tree_sitter_typescript + tree_sitter_java tree_sitter_rust tree_sitter_go) + # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies) diff --git a/editor/src/TranspilationConfidence.h b/editor/src/TranspilationConfidence.h new file mode 100644 index 0000000..53a29eb --- /dev/null +++ b/editor/src/TranspilationConfidence.h @@ -0,0 +1,153 @@ +#pragma once + +// Step 456: Transpilation Confidence Scorer — scores translations 0.0-1.0 +// and auto-adds @Review(required,human) for low-confidence translations. + +#include +#include + +enum class TranslationCategory { + DirectStdLib, // 0.95 — direct standard library mapping + AlgorithmPattern, // 0.85 — recognized algorithm pattern + StructuralWithIntent,// 0.70 — structural translation with @Intent + StructuralNoIntent, // 0.50 — structural translation, no intent + Unknown // 0.30 — unknown/unrecognized pattern +}; + +struct ConfidenceScore { + std::string functionName; + std::string sourceLanguage; + std::string targetLanguage; + TranslationCategory category = TranslationCategory::Unknown; + float score = 0.0f; + bool reviewRequired = false; + std::string annotation; // e.g. @Confidence(0.85) + std::string reviewReason; +}; + +struct ConfidenceReport { + std::vector scores; + std::string sourceLanguage; + std::string targetLanguage; + + float averageScore() const { + if (scores.empty()) return 0.0f; + float sum = 0.0f; + for (const auto& s : scores) sum += s.score; + return sum / (float)scores.size(); + } + + int countReviewRequired() const { + int n = 0; + for (const auto& s : scores) + if (s.reviewRequired) ++n; + return n; + } + + int countByCategory(TranslationCategory cat) const { + int n = 0; + for (const auto& s : scores) + if (s.category == cat) ++n; + return n; + } + + ConfidenceScore getScore(const std::string& funcName) const { + for (const auto& s : scores) + if (s.functionName == funcName) return s; + return {}; + } +}; + +class TranspilationScorer { +public: + static ConfidenceScore score(const std::string& functionName, + const std::string& sourceCode, + const std::string& targetCode, + const std::string& srcLang, + const std::string& tgtLang, + bool hasIntent = false) { + ConfidenceScore cs; + cs.functionName = functionName; + cs.sourceLanguage = srcLang; + cs.targetLanguage = tgtLang; + cs.category = classify(sourceCode, targetCode, hasIntent); + cs.score = categoryScore(cs.category); + + if (cs.score < 0.60f) { + cs.reviewRequired = true; + cs.annotation = "@Confidence(" + std::to_string(cs.score).substr(0, 4) + + "), @Review(required,human)"; + cs.reviewReason = "Low confidence translation — manual review needed"; + } else { + cs.annotation = "@Confidence(" + std::to_string(cs.score).substr(0, 4) + ")"; + } + + return cs; + } + + static ConfidenceReport scoreAll( + const std::vector>>& functions, + const std::string& srcLang, + const std::string& tgtLang, + const std::vector& hasIntents = {}) { + + ConfidenceReport report; + report.sourceLanguage = srcLang; + report.targetLanguage = tgtLang; + + for (size_t i = 0; i < functions.size(); ++i) { + bool intent = (i < hasIntents.size()) ? hasIntents[i] : false; + auto cs = score(functions[i].first, + functions[i].second.first, + functions[i].second.second, + srcLang, tgtLang, intent); + report.scores.push_back(std::move(cs)); + } + return report; + } + + static float categoryScore(TranslationCategory cat) { + switch (cat) { + case TranslationCategory::DirectStdLib: return 0.95f; + case TranslationCategory::AlgorithmPattern: return 0.85f; + case TranslationCategory::StructuralWithIntent: return 0.70f; + case TranslationCategory::StructuralNoIntent: return 0.50f; + case TranslationCategory::Unknown: return 0.30f; + } + return 0.30f; + } + +private: + static TranslationCategory classify(const std::string& source, + const std::string& target, + bool hasIntent) { + // Direct stdlib mapping + if (target.find(".sort()") != std::string::npos || + target.find("sorted(") != std::string::npos || + target.find("Collections.sort") != std::string::npos || + target.find("std::sort") != std::string::npos || + target.find("binary_search") != std::string::npos || + target.find("read_to_string") != std::string::npos || + target.find("ReadFile") != std::string::npos) + return TranslationCategory::DirectStdLib; + + // Algorithm pattern: iterator chains, fold, filter, map + if (target.find(".iter()") != std::string::npos || + target.find(".stream()") != std::string::npos || + target.find(".fold(") != std::string::npos || + target.find(".filter(") != std::string::npos || + target.find(".map(") != std::string::npos || + target.find("accumulate") != std::string::npos) + return TranslationCategory::AlgorithmPattern; + + // Structural with or without intent + if (hasIntent) + return TranslationCategory::StructuralWithIntent; + + // Check if it looks like real code or just TODO + if (target.find("TODO") != std::string::npos || target.empty()) + return TranslationCategory::Unknown; + + return TranslationCategory::StructuralNoIntent; + } +}; diff --git a/editor/tests/step456_test.cpp b/editor/tests/step456_test.cpp new file mode 100644 index 0000000..e0532cc --- /dev/null +++ b/editor/tests/step456_test.cpp @@ -0,0 +1,155 @@ +// Step 456: Transpilation Confidence Scoring Tests (12 tests) + +#include "TranspilationConfidence.h" + +#include + +static int passed = 0, failed = 0; +#define TEST(name) { std::cout << " " << #name << "... "; } +#define PASS() { std::cout << "PASS\n"; ++passed; } +#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; } +#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {} + +void test_direct_stdlib_scores_point_95() { + TEST(direct_stdlib_scores_point_95); + auto s = TranspilationScorer::score("sort_values", "sort loop", "items.sort()", "python", "rust", true); + CHECK(s.category == TranslationCategory::DirectStdLib, "expected direct stdlib category"); + CHECK(s.score == 0.95f, "expected 0.95 score"); + CHECK(!s.reviewRequired, "high confidence should not require review"); + PASS(); +} + +void test_algorithm_pattern_scores_point_85() { + TEST(algorithm_pattern_scores_point_85); + auto s = TranspilationScorer::score("sum_values", "sum loop", "items.iter().fold(0, |a, x| a + x)", "python", "rust", true); + CHECK(s.category == TranslationCategory::AlgorithmPattern, "expected algorithm pattern"); + CHECK(s.score == 0.85f, "expected 0.85 score"); + PASS(); +} + +void test_structural_with_intent_scores_point_70() { + TEST(structural_with_intent_scores_point_70); + auto s = TranspilationScorer::score("transform", "custom transform", "transform(data)", "python", "java", true); + CHECK(s.category == TranslationCategory::StructuralWithIntent, "expected structural with intent"); + CHECK(s.score == 0.70f, "expected 0.70 score"); + CHECK(!s.reviewRequired, "0.70 should not require review"); + PASS(); +} + +void test_structural_no_intent_scores_point_50_and_reviews() { + TEST(structural_no_intent_scores_point_50_and_reviews); + auto s = TranspilationScorer::score("transform", "custom transform", "transform(data)", "python", "java", false); + CHECK(s.category == TranslationCategory::StructuralNoIntent, "expected structural no intent"); + CHECK(s.score == 0.50f, "expected 0.50 score"); + CHECK(s.reviewRequired, "0.50 should require review"); + PASS(); +} + +void test_unknown_todo_scores_point_30_and_reviews() { + TEST(unknown_todo_scores_point_30_and_reviews); + auto s = TranspilationScorer::score("mystery", "unknown", "TODO: implement", "python", "rust", false); + CHECK(s.category == TranslationCategory::Unknown, "expected unknown category"); + CHECK(s.score == 0.30f, "expected 0.30 score"); + CHECK(s.reviewRequired, "0.30 should require review"); + PASS(); +} + +void test_low_confidence_adds_review_annotation() { + TEST(low_confidence_adds_review_annotation); + auto s = TranspilationScorer::score("mystery", "unknown", "TODO", "python", "rust", false); + CHECK(s.annotation.find("@Review(required,human)") != std::string::npos, + "expected human review annotation"); + CHECK(!s.reviewReason.empty(), "expected non-empty review reason"); + PASS(); +} + +void test_high_confidence_annotation_has_no_review_tag() { + TEST(high_confidence_annotation_has_no_review_tag); + auto s = TranspilationScorer::score("sort_values", "sort loop", "std::sort(v.begin(), v.end())", "cpp", "cpp", true); + CHECK(s.annotation.find("@Confidence(") != std::string::npos, "expected confidence annotation"); + CHECK(s.annotation.find("@Review(required,human)") == std::string::npos, + "high confidence should not include review annotation"); + PASS(); +} + +void test_score_all_aggregates_counts() { + TEST(score_all_aggregates_counts); + std::vector>> fns = { + {"sort_values", {"sort loop", "items.sort()"}}, + {"sum_values", {"sum loop", "items.iter().fold(0, |a, x| a + x)"}}, + {"unknown", {"unknown", "TODO"}} + }; + auto report = TranspilationScorer::scoreAll(fns, "python", "rust", {true, true, false}); + CHECK(report.scores.size() == 3, "expected 3 scores"); + CHECK(report.countByCategory(TranslationCategory::DirectStdLib) == 1, "expected one stdlib score"); + CHECK(report.countReviewRequired() == 1, "expected one review-required score"); + PASS(); +} + +void test_average_score_computation() { + TEST(average_score_computation); + std::vector>> fns = { + {"a", {"sort", "items.sort()"}}, // 0.95 + {"b", {"sum", "items.iter().fold(0, |a, x| a + x)"}}, // 0.85 + {"c", {"u", "TODO"}} // 0.30 + }; + auto report = TranspilationScorer::scoreAll(fns, "python", "rust", {true, true, false}); + float avg = report.averageScore(); + CHECK(avg > 0.69f && avg < 0.71f, "expected average around 0.70"); + PASS(); +} + +void test_get_score_by_function_name() { + TEST(get_score_by_function_name); + std::vector>> fns = { + {"sort_values", {"sort", "items.sort()"}}, + {"mystery", {"u", "TODO"}} + }; + auto report = TranspilationScorer::scoreAll(fns, "python", "rust", {true, false}); + auto s = report.getScore("mystery"); + CHECK(s.functionName == "mystery", "expected returned function score"); + CHECK(s.reviewRequired, "mystery should require review"); + PASS(); +} + +void test_score_all_defaults_has_intent_false() { + TEST(score_all_defaults_has_intent_false); + std::vector>> fns = { + {"x", {"custom", "transform(data)"}} + }; + auto report = TranspilationScorer::scoreAll(fns, "python", "java"); + CHECK(report.scores.size() == 1, "expected one score"); + CHECK(report.scores[0].category == TranslationCategory::StructuralNoIntent, + "default intent should be false"); + PASS(); +} + +void test_category_score_lookup_values() { + TEST(category_score_lookup_values); + CHECK(TranspilationScorer::categoryScore(TranslationCategory::DirectStdLib) == 0.95f, + "expected stdlib category score"); + CHECK(TranspilationScorer::categoryScore(TranslationCategory::Unknown) == 0.30f, + "expected unknown category score"); + PASS(); +} + +int main() { + std::cout << "Step 456: Transpilation Confidence Scoring Tests\n"; + + test_direct_stdlib_scores_point_95(); // 1 + test_algorithm_pattern_scores_point_85(); // 2 + test_structural_with_intent_scores_point_70(); // 3 + test_structural_no_intent_scores_point_50_and_reviews(); // 4 + test_unknown_todo_scores_point_30_and_reviews(); // 5 + test_low_confidence_adds_review_annotation(); // 6 + test_high_confidence_annotation_has_no_review_tag(); // 7 + test_score_all_aggregates_counts(); // 8 + test_average_score_computation(); // 9 + test_get_score_by_function_name(); // 10 + test_score_all_defaults_has_intent_false(); // 11 + test_category_score_lookup_values(); // 12 + + std::cout << "\nResults: " << passed << "/" << (passed + failed) + << " passed\n"; + return failed == 0 ? 0 : 1; +} diff --git a/progress.md b/progress.md index c38b75d..be091e5 100644 --- a/progress.md +++ b/progress.md @@ -5881,3 +5881,32 @@ grading from confidence scores (verified/likely/unverified+review). **Architecture gate check:** - `editor/src/BehavioralEquivalence.h` within header-size limit (`141` <= `600`) - `editor/tests/step455_test.cpp` within test-file size guidance (`194` lines) + +### Step 456: Transpilation Confidence Scoring +**Status:** PASS (12/12 tests) + +Implements deterministic confidence scoring categories and review gating for +translated functions, including aggregated report statistics. + +**Files added:** +- `editor/src/TranspilationConfidence.h` — confidence scorer: + - category model (`DirectStdLib`, `AlgorithmPattern`, + `StructuralWithIntent`, `StructuralNoIntent`, `Unknown`) + - deterministic score mapping (0.95 / 0.85 / 0.70 / 0.50 / 0.30) + - low-confidence review gating (`@Review(required,human)` when `< 0.60`) + - per-function score output + batch report aggregation APIs +- `editor/tests/step456_test.cpp` — 12 tests covering: + - category classification and score assignment + - annotation/review behavior for high vs low confidence + - `scoreAll` aggregation, category/review counters, average score + - function-score lookup and default no-intent behavior +- `editor/CMakeLists.txt` — `step456_test` target + +**Verification run:** +- `cmake --build editor/build-native --target step456_test` — PASS +- `./editor/build-native/step456_test` — PASS (12/12) +- `./editor/build-native/step455_test` — PASS (12/12) regression coverage + +**Architecture gate check:** +- `editor/src/TranspilationConfidence.h` within header-size limit (`153` <= `600`) +- `editor/tests/step456_test.cpp` within test-file size guidance (`155` lines)