Complete step456 transpilation confidence scoring with tests
This commit is contained in:
Bill
@@ -2983,4 +2983,13 @@ target_link_libraries(step455_test PRIVATE
|
||||
tree_sitter_javascript tree_sitter_typescript
|
||||
tree_sitter_java tree_sitter_rust tree_sitter_go)
|
||||
|
||||
add_executable(step456_test tests/step456_test.cpp)
|
||||
target_include_directories(step456_test PRIVATE src)
|
||||
target_link_libraries(step456_test PRIVATE
|
||||
nlohmann_json::nlohmann_json
|
||||
unofficial::tree-sitter::tree-sitter
|
||||
tree_sitter_python tree_sitter_cpp tree_sitter_elisp
|
||||
tree_sitter_javascript tree_sitter_typescript
|
||||
tree_sitter_java tree_sitter_rust tree_sitter_go)
|
||||
|
||||
# Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies)
|
||||
|
||||
153
editor/src/TranspilationConfidence.h
Normal file
153
editor/src/TranspilationConfidence.h
Normal file
@@ -0,0 +1,153 @@
|
||||
#pragma once
|
||||
|
||||
// Step 456: Transpilation Confidence Scorer — scores translations 0.0-1.0
|
||||
// and auto-adds @Review(required,human) for low-confidence translations.
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
enum class TranslationCategory {
|
||||
DirectStdLib, // 0.95 — direct standard library mapping
|
||||
AlgorithmPattern, // 0.85 — recognized algorithm pattern
|
||||
StructuralWithIntent,// 0.70 — structural translation with @Intent
|
||||
StructuralNoIntent, // 0.50 — structural translation, no intent
|
||||
Unknown // 0.30 — unknown/unrecognized pattern
|
||||
};
|
||||
|
||||
struct ConfidenceScore {
|
||||
std::string functionName;
|
||||
std::string sourceLanguage;
|
||||
std::string targetLanguage;
|
||||
TranslationCategory category = TranslationCategory::Unknown;
|
||||
float score = 0.0f;
|
||||
bool reviewRequired = false;
|
||||
std::string annotation; // e.g. @Confidence(0.85)
|
||||
std::string reviewReason;
|
||||
};
|
||||
|
||||
struct ConfidenceReport {
|
||||
std::vector<ConfidenceScore> scores;
|
||||
std::string sourceLanguage;
|
||||
std::string targetLanguage;
|
||||
|
||||
float averageScore() const {
|
||||
if (scores.empty()) return 0.0f;
|
||||
float sum = 0.0f;
|
||||
for (const auto& s : scores) sum += s.score;
|
||||
return sum / (float)scores.size();
|
||||
}
|
||||
|
||||
int countReviewRequired() const {
|
||||
int n = 0;
|
||||
for (const auto& s : scores)
|
||||
if (s.reviewRequired) ++n;
|
||||
return n;
|
||||
}
|
||||
|
||||
int countByCategory(TranslationCategory cat) const {
|
||||
int n = 0;
|
||||
for (const auto& s : scores)
|
||||
if (s.category == cat) ++n;
|
||||
return n;
|
||||
}
|
||||
|
||||
ConfidenceScore getScore(const std::string& funcName) const {
|
||||
for (const auto& s : scores)
|
||||
if (s.functionName == funcName) return s;
|
||||
return {};
|
||||
}
|
||||
};
|
||||
|
||||
class TranspilationScorer {
|
||||
public:
|
||||
static ConfidenceScore score(const std::string& functionName,
|
||||
const std::string& sourceCode,
|
||||
const std::string& targetCode,
|
||||
const std::string& srcLang,
|
||||
const std::string& tgtLang,
|
||||
bool hasIntent = false) {
|
||||
ConfidenceScore cs;
|
||||
cs.functionName = functionName;
|
||||
cs.sourceLanguage = srcLang;
|
||||
cs.targetLanguage = tgtLang;
|
||||
cs.category = classify(sourceCode, targetCode, hasIntent);
|
||||
cs.score = categoryScore(cs.category);
|
||||
|
||||
if (cs.score < 0.60f) {
|
||||
cs.reviewRequired = true;
|
||||
cs.annotation = "@Confidence(" + std::to_string(cs.score).substr(0, 4) +
|
||||
"), @Review(required,human)";
|
||||
cs.reviewReason = "Low confidence translation — manual review needed";
|
||||
} else {
|
||||
cs.annotation = "@Confidence(" + std::to_string(cs.score).substr(0, 4) + ")";
|
||||
}
|
||||
|
||||
return cs;
|
||||
}
|
||||
|
||||
static ConfidenceReport scoreAll(
|
||||
const std::vector<std::pair<std::string, std::pair<std::string, std::string>>>& functions,
|
||||
const std::string& srcLang,
|
||||
const std::string& tgtLang,
|
||||
const std::vector<bool>& hasIntents = {}) {
|
||||
|
||||
ConfidenceReport report;
|
||||
report.sourceLanguage = srcLang;
|
||||
report.targetLanguage = tgtLang;
|
||||
|
||||
for (size_t i = 0; i < functions.size(); ++i) {
|
||||
bool intent = (i < hasIntents.size()) ? hasIntents[i] : false;
|
||||
auto cs = score(functions[i].first,
|
||||
functions[i].second.first,
|
||||
functions[i].second.second,
|
||||
srcLang, tgtLang, intent);
|
||||
report.scores.push_back(std::move(cs));
|
||||
}
|
||||
return report;
|
||||
}
|
||||
|
||||
static float categoryScore(TranslationCategory cat) {
|
||||
switch (cat) {
|
||||
case TranslationCategory::DirectStdLib: return 0.95f;
|
||||
case TranslationCategory::AlgorithmPattern: return 0.85f;
|
||||
case TranslationCategory::StructuralWithIntent: return 0.70f;
|
||||
case TranslationCategory::StructuralNoIntent: return 0.50f;
|
||||
case TranslationCategory::Unknown: return 0.30f;
|
||||
}
|
||||
return 0.30f;
|
||||
}
|
||||
|
||||
private:
|
||||
static TranslationCategory classify(const std::string& source,
|
||||
const std::string& target,
|
||||
bool hasIntent) {
|
||||
// Direct stdlib mapping
|
||||
if (target.find(".sort()") != std::string::npos ||
|
||||
target.find("sorted(") != std::string::npos ||
|
||||
target.find("Collections.sort") != std::string::npos ||
|
||||
target.find("std::sort") != std::string::npos ||
|
||||
target.find("binary_search") != std::string::npos ||
|
||||
target.find("read_to_string") != std::string::npos ||
|
||||
target.find("ReadFile") != std::string::npos)
|
||||
return TranslationCategory::DirectStdLib;
|
||||
|
||||
// Algorithm pattern: iterator chains, fold, filter, map
|
||||
if (target.find(".iter()") != std::string::npos ||
|
||||
target.find(".stream()") != std::string::npos ||
|
||||
target.find(".fold(") != std::string::npos ||
|
||||
target.find(".filter(") != std::string::npos ||
|
||||
target.find(".map(") != std::string::npos ||
|
||||
target.find("accumulate") != std::string::npos)
|
||||
return TranslationCategory::AlgorithmPattern;
|
||||
|
||||
// Structural with or without intent
|
||||
if (hasIntent)
|
||||
return TranslationCategory::StructuralWithIntent;
|
||||
|
||||
// Check if it looks like real code or just TODO
|
||||
if (target.find("TODO") != std::string::npos || target.empty())
|
||||
return TranslationCategory::Unknown;
|
||||
|
||||
return TranslationCategory::StructuralNoIntent;
|
||||
}
|
||||
};
|
||||
155
editor/tests/step456_test.cpp
Normal file
155
editor/tests/step456_test.cpp
Normal file
@@ -0,0 +1,155 @@
|
||||
// Step 456: Transpilation Confidence Scoring Tests (12 tests)
|
||||
|
||||
#include "TranspilationConfidence.h"
|
||||
|
||||
#include <iostream>
|
||||
|
||||
static int passed = 0, failed = 0;
|
||||
#define TEST(name) { std::cout << " " << #name << "... "; }
|
||||
#define PASS() { std::cout << "PASS\n"; ++passed; }
|
||||
#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; }
|
||||
#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {}
|
||||
|
||||
void test_direct_stdlib_scores_point_95() {
|
||||
TEST(direct_stdlib_scores_point_95);
|
||||
auto s = TranspilationScorer::score("sort_values", "sort loop", "items.sort()", "python", "rust", true);
|
||||
CHECK(s.category == TranslationCategory::DirectStdLib, "expected direct stdlib category");
|
||||
CHECK(s.score == 0.95f, "expected 0.95 score");
|
||||
CHECK(!s.reviewRequired, "high confidence should not require review");
|
||||
PASS();
|
||||
}
|
||||
|
||||
void test_algorithm_pattern_scores_point_85() {
|
||||
TEST(algorithm_pattern_scores_point_85);
|
||||
auto s = TranspilationScorer::score("sum_values", "sum loop", "items.iter().fold(0, |a, x| a + x)", "python", "rust", true);
|
||||
CHECK(s.category == TranslationCategory::AlgorithmPattern, "expected algorithm pattern");
|
||||
CHECK(s.score == 0.85f, "expected 0.85 score");
|
||||
PASS();
|
||||
}
|
||||
|
||||
void test_structural_with_intent_scores_point_70() {
|
||||
TEST(structural_with_intent_scores_point_70);
|
||||
auto s = TranspilationScorer::score("transform", "custom transform", "transform(data)", "python", "java", true);
|
||||
CHECK(s.category == TranslationCategory::StructuralWithIntent, "expected structural with intent");
|
||||
CHECK(s.score == 0.70f, "expected 0.70 score");
|
||||
CHECK(!s.reviewRequired, "0.70 should not require review");
|
||||
PASS();
|
||||
}
|
||||
|
||||
void test_structural_no_intent_scores_point_50_and_reviews() {
|
||||
TEST(structural_no_intent_scores_point_50_and_reviews);
|
||||
auto s = TranspilationScorer::score("transform", "custom transform", "transform(data)", "python", "java", false);
|
||||
CHECK(s.category == TranslationCategory::StructuralNoIntent, "expected structural no intent");
|
||||
CHECK(s.score == 0.50f, "expected 0.50 score");
|
||||
CHECK(s.reviewRequired, "0.50 should require review");
|
||||
PASS();
|
||||
}
|
||||
|
||||
void test_unknown_todo_scores_point_30_and_reviews() {
|
||||
TEST(unknown_todo_scores_point_30_and_reviews);
|
||||
auto s = TranspilationScorer::score("mystery", "unknown", "TODO: implement", "python", "rust", false);
|
||||
CHECK(s.category == TranslationCategory::Unknown, "expected unknown category");
|
||||
CHECK(s.score == 0.30f, "expected 0.30 score");
|
||||
CHECK(s.reviewRequired, "0.30 should require review");
|
||||
PASS();
|
||||
}
|
||||
|
||||
void test_low_confidence_adds_review_annotation() {
|
||||
TEST(low_confidence_adds_review_annotation);
|
||||
auto s = TranspilationScorer::score("mystery", "unknown", "TODO", "python", "rust", false);
|
||||
CHECK(s.annotation.find("@Review(required,human)") != std::string::npos,
|
||||
"expected human review annotation");
|
||||
CHECK(!s.reviewReason.empty(), "expected non-empty review reason");
|
||||
PASS();
|
||||
}
|
||||
|
||||
void test_high_confidence_annotation_has_no_review_tag() {
|
||||
TEST(high_confidence_annotation_has_no_review_tag);
|
||||
auto s = TranspilationScorer::score("sort_values", "sort loop", "std::sort(v.begin(), v.end())", "cpp", "cpp", true);
|
||||
CHECK(s.annotation.find("@Confidence(") != std::string::npos, "expected confidence annotation");
|
||||
CHECK(s.annotation.find("@Review(required,human)") == std::string::npos,
|
||||
"high confidence should not include review annotation");
|
||||
PASS();
|
||||
}
|
||||
|
||||
void test_score_all_aggregates_counts() {
|
||||
TEST(score_all_aggregates_counts);
|
||||
std::vector<std::pair<std::string, std::pair<std::string, std::string>>> fns = {
|
||||
{"sort_values", {"sort loop", "items.sort()"}},
|
||||
{"sum_values", {"sum loop", "items.iter().fold(0, |a, x| a + x)"}},
|
||||
{"unknown", {"unknown", "TODO"}}
|
||||
};
|
||||
auto report = TranspilationScorer::scoreAll(fns, "python", "rust", {true, true, false});
|
||||
CHECK(report.scores.size() == 3, "expected 3 scores");
|
||||
CHECK(report.countByCategory(TranslationCategory::DirectStdLib) == 1, "expected one stdlib score");
|
||||
CHECK(report.countReviewRequired() == 1, "expected one review-required score");
|
||||
PASS();
|
||||
}
|
||||
|
||||
void test_average_score_computation() {
|
||||
TEST(average_score_computation);
|
||||
std::vector<std::pair<std::string, std::pair<std::string, std::string>>> fns = {
|
||||
{"a", {"sort", "items.sort()"}}, // 0.95
|
||||
{"b", {"sum", "items.iter().fold(0, |a, x| a + x)"}}, // 0.85
|
||||
{"c", {"u", "TODO"}} // 0.30
|
||||
};
|
||||
auto report = TranspilationScorer::scoreAll(fns, "python", "rust", {true, true, false});
|
||||
float avg = report.averageScore();
|
||||
CHECK(avg > 0.69f && avg < 0.71f, "expected average around 0.70");
|
||||
PASS();
|
||||
}
|
||||
|
||||
void test_get_score_by_function_name() {
|
||||
TEST(get_score_by_function_name);
|
||||
std::vector<std::pair<std::string, std::pair<std::string, std::string>>> fns = {
|
||||
{"sort_values", {"sort", "items.sort()"}},
|
||||
{"mystery", {"u", "TODO"}}
|
||||
};
|
||||
auto report = TranspilationScorer::scoreAll(fns, "python", "rust", {true, false});
|
||||
auto s = report.getScore("mystery");
|
||||
CHECK(s.functionName == "mystery", "expected returned function score");
|
||||
CHECK(s.reviewRequired, "mystery should require review");
|
||||
PASS();
|
||||
}
|
||||
|
||||
void test_score_all_defaults_has_intent_false() {
|
||||
TEST(score_all_defaults_has_intent_false);
|
||||
std::vector<std::pair<std::string, std::pair<std::string, std::string>>> fns = {
|
||||
{"x", {"custom", "transform(data)"}}
|
||||
};
|
||||
auto report = TranspilationScorer::scoreAll(fns, "python", "java");
|
||||
CHECK(report.scores.size() == 1, "expected one score");
|
||||
CHECK(report.scores[0].category == TranslationCategory::StructuralNoIntent,
|
||||
"default intent should be false");
|
||||
PASS();
|
||||
}
|
||||
|
||||
void test_category_score_lookup_values() {
|
||||
TEST(category_score_lookup_values);
|
||||
CHECK(TranspilationScorer::categoryScore(TranslationCategory::DirectStdLib) == 0.95f,
|
||||
"expected stdlib category score");
|
||||
CHECK(TranspilationScorer::categoryScore(TranslationCategory::Unknown) == 0.30f,
|
||||
"expected unknown category score");
|
||||
PASS();
|
||||
}
|
||||
|
||||
int main() {
|
||||
std::cout << "Step 456: Transpilation Confidence Scoring Tests\n";
|
||||
|
||||
test_direct_stdlib_scores_point_95(); // 1
|
||||
test_algorithm_pattern_scores_point_85(); // 2
|
||||
test_structural_with_intent_scores_point_70(); // 3
|
||||
test_structural_no_intent_scores_point_50_and_reviews(); // 4
|
||||
test_unknown_todo_scores_point_30_and_reviews(); // 5
|
||||
test_low_confidence_adds_review_annotation(); // 6
|
||||
test_high_confidence_annotation_has_no_review_tag(); // 7
|
||||
test_score_all_aggregates_counts(); // 8
|
||||
test_average_score_computation(); // 9
|
||||
test_get_score_by_function_name(); // 10
|
||||
test_score_all_defaults_has_intent_false(); // 11
|
||||
test_category_score_lookup_values(); // 12
|
||||
|
||||
std::cout << "\nResults: " << passed << "/" << (passed + failed)
|
||||
<< " passed\n";
|
||||
return failed == 0 ? 0 : 1;
|
||||
}
|
||||
29
progress.md
29
progress.md
@@ -5881,3 +5881,32 @@ grading from confidence scores (verified/likely/unverified+review).
|
||||
**Architecture gate check:**
|
||||
- `editor/src/BehavioralEquivalence.h` within header-size limit (`141` <= `600`)
|
||||
- `editor/tests/step455_test.cpp` within test-file size guidance (`194` lines)
|
||||
|
||||
### Step 456: Transpilation Confidence Scoring
|
||||
**Status:** PASS (12/12 tests)
|
||||
|
||||
Implements deterministic confidence scoring categories and review gating for
|
||||
translated functions, including aggregated report statistics.
|
||||
|
||||
**Files added:**
|
||||
- `editor/src/TranspilationConfidence.h` — confidence scorer:
|
||||
- category model (`DirectStdLib`, `AlgorithmPattern`,
|
||||
`StructuralWithIntent`, `StructuralNoIntent`, `Unknown`)
|
||||
- deterministic score mapping (0.95 / 0.85 / 0.70 / 0.50 / 0.30)
|
||||
- low-confidence review gating (`@Review(required,human)` when `< 0.60`)
|
||||
- per-function score output + batch report aggregation APIs
|
||||
- `editor/tests/step456_test.cpp` — 12 tests covering:
|
||||
- category classification and score assignment
|
||||
- annotation/review behavior for high vs low confidence
|
||||
- `scoreAll` aggregation, category/review counters, average score
|
||||
- function-score lookup and default no-intent behavior
|
||||
- `editor/CMakeLists.txt` — `step456_test` target
|
||||
|
||||
**Verification run:**
|
||||
- `cmake --build editor/build-native --target step456_test` — PASS
|
||||
- `./editor/build-native/step456_test` — PASS (12/12)
|
||||
- `./editor/build-native/step455_test` — PASS (12/12) regression coverage
|
||||
|
||||
**Architecture gate check:**
|
||||
- `editor/src/TranspilationConfidence.h` within header-size limit (`153` <= `600`)
|
||||
- `editor/tests/step456_test.cpp` within test-file size guidance (`155` lines)
|
||||
|
||||
Reference in New Issue
Block a user