From 3f62fb1ce9e8b57cefde2224107909c96e54ae9d Mon Sep 17 00:00:00 2001 From: Bill Date: Tue, 17 Feb 2026 09:35:45 -0700 Subject: [PATCH] Step 537: add constrained execution telemetry summaries --- editor/CMakeLists.txt | 9 ++ editor/src/ConstrainedExecutionTelemetry.h | 75 +++++++++ editor/tests/step537_test.cpp | 175 +++++++++++++++++++++ progress.md | 37 +++++ 4 files changed, 296 insertions(+) create mode 100644 editor/src/ConstrainedExecutionTelemetry.h create mode 100644 editor/tests/step537_test.cpp diff --git a/editor/CMakeLists.txt b/editor/CMakeLists.txt index b53fe0f..65a2506 100644 --- a/editor/CMakeLists.txt +++ b/editor/CMakeLists.txt @@ -3712,4 +3712,13 @@ target_link_libraries(step536_test PRIVATE tree_sitter_javascript tree_sitter_typescript tree_sitter_java tree_sitter_rust tree_sitter_go) +add_executable(step537_test tests/step537_test.cpp) +target_include_directories(step537_test PRIVATE src) +target_link_libraries(step537_test PRIVATE + nlohmann_json::nlohmann_json + unofficial::tree-sitter::tree-sitter + tree_sitter_python tree_sitter_cpp tree_sitter_elisp + tree_sitter_javascript tree_sitter_typescript + tree_sitter_java tree_sitter_rust tree_sitter_go) + # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies) diff --git a/editor/src/ConstrainedExecutionTelemetry.h b/editor/src/ConstrainedExecutionTelemetry.h new file mode 100644 index 0000000..6664c3e --- /dev/null +++ b/editor/src/ConstrainedExecutionTelemetry.h @@ -0,0 +1,75 @@ +#pragma once +// Step 537: Constrained Execution Telemetry + +#include +#include +#include +#include + +struct TelemetryEvent { + std::string taskitemId; + int candidateOperationCount = 0; + int candidateSymbolCount = 0; + std::string selectedOperation; + std::vector rejectionReasons; + int constrainedTokenCount = 0; + int baselineTokenCount = 0; + bool success = false; +}; + +struct TelemetrySummary { + int events = 0; + int successCount = 0; + int failureCount = 0; + int avgCandidateOperationCount = 0; + int avgCandidateSymbolCount = 0; + int constrainedTokens = 0; + int baselineTokens = 0; + int tokenSavings = 0; + double tokenSavingsRatio = 0.0; + std::map rejectionHistogram; + std::map opSelectionHistogram; +}; + +class ConstrainedExecutionTelemetry { +public: + void record(const TelemetryEvent& event) { + events_.push_back(event); + } + + size_t size() const { return events_.size(); } + + TelemetrySummary summarize() const { + TelemetrySummary s; + s.events = (int)events_.size(); + if (events_.empty()) return s; + + int opTotal = 0; + int symTotal = 0; + for (const auto& event : events_) { + opTotal += event.candidateOperationCount; + symTotal += event.candidateSymbolCount; + s.constrainedTokens += event.constrainedTokenCount; + s.baselineTokens += event.baselineTokenCount; + if (event.success) ++s.successCount; else ++s.failureCount; + if (!event.selectedOperation.empty()) { + ++s.opSelectionHistogram[event.selectedOperation]; + } + for (const auto& reason : event.rejectionReasons) { + ++s.rejectionHistogram[reason]; + } + } + + s.avgCandidateOperationCount = opTotal / s.events; + s.avgCandidateSymbolCount = symTotal / s.events; + s.tokenSavings = s.baselineTokens - s.constrainedTokens; + if (s.baselineTokens > 0) { + s.tokenSavingsRatio = static_cast(s.tokenSavings) / + static_cast(s.baselineTokens); + } + return s; + } + +private: + std::vector events_; +}; diff --git a/editor/tests/step537_test.cpp b/editor/tests/step537_test.cpp new file mode 100644 index 0000000..5ee938b --- /dev/null +++ b/editor/tests/step537_test.cpp @@ -0,0 +1,175 @@ +// Step 537: Constrained Execution Telemetry (12 tests) + +#include "ConstrainedExecutionTelemetry.h" + +#include + +static int passed = 0, failed = 0; +#define TEST(name) { std::cout << " " << #name << "... "; } +#define PASS() { std::cout << "PASS\n"; ++passed; } +#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; } +#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {} + +static TelemetryEvent event(const std::string& id, + int opCount, + int symCount, + const std::string& op, + const std::vector& reasons, + int constrained, + int baseline, + bool success) { + TelemetryEvent e; + e.taskitemId = id; + e.candidateOperationCount = opCount; + e.candidateSymbolCount = symCount; + e.selectedOperation = op; + e.rejectionReasons = reasons; + e.constrainedTokenCount = constrained; + e.baselineTokenCount = baseline; + e.success = success; + return e; +} + +void test_empty_summary_is_zeroed() { + TEST(empty_summary_is_zeroed); + ConstrainedExecutionTelemetry t; + auto s = t.summarize(); + CHECK(s.events == 0, "no events expected"); + CHECK(s.constrainedTokens == 0 && s.baselineTokens == 0, "token counts should be zero"); + PASS(); +} + +void test_record_increments_event_count() { + TEST(record_increments_event_count); + ConstrainedExecutionTelemetry t; + t.record(event("a", 3, 5, "rename", {}, 20, 30, true)); + CHECK(t.size() == 1, "size should be one"); + PASS(); +} + +void test_summary_counts_success_and_failure() { + TEST(summary_counts_success_and_failure); + ConstrainedExecutionTelemetry t; + t.record(event("a", 3, 5, "rename", {}, 20, 30, true)); + t.record(event("b", 2, 4, "", {"out_of_scope_symbol"}, 22, 35, false)); + auto s = t.summarize(); + CHECK(s.successCount == 1, "success count mismatch"); + CHECK(s.failureCount == 1, "failure count mismatch"); + PASS(); +} + +void test_average_candidate_counts_are_computed() { + TEST(average_candidate_counts_are_computed); + ConstrainedExecutionTelemetry t; + t.record(event("a", 4, 10, "rename", {}, 10, 20, true)); + t.record(event("b", 2, 6, "update", {}, 10, 20, true)); + auto s = t.summarize(); + CHECK(s.avgCandidateOperationCount == 3, "avg op count mismatch"); + CHECK(s.avgCandidateSymbolCount == 8, "avg symbol count mismatch"); + PASS(); +} + +void test_token_savings_computation() { + TEST(token_savings_computation); + ConstrainedExecutionTelemetry t; + t.record(event("a", 4, 10, "rename", {}, 15, 30, true)); + t.record(event("b", 2, 6, "update", {}, 20, 40, true)); + auto s = t.summarize(); + CHECK(s.constrainedTokens == 35, "constrained total mismatch"); + CHECK(s.baselineTokens == 70, "baseline total mismatch"); + CHECK(s.tokenSavings == 35, "token savings mismatch"); + PASS(); +} + +void test_token_savings_ratio_computation() { + TEST(token_savings_ratio_computation); + ConstrainedExecutionTelemetry t; + t.record(event("a", 4, 10, "rename", {}, 25, 50, true)); + auto s = t.summarize(); + CHECK(s.tokenSavingsRatio > 0.49 && s.tokenSavingsRatio < 0.51, + "token ratio should be about 0.5"); + PASS(); +} + +void test_rejection_histogram_counts_reasons() { + TEST(rejection_histogram_counts_reasons); + ConstrainedExecutionTelemetry t; + t.record(event("a", 2, 5, "", {"out_of_scope_symbol", "illegal_operation"}, 20, 30, false)); + t.record(event("b", 2, 5, "", {"out_of_scope_symbol"}, 22, 35, false)); + auto s = t.summarize(); + CHECK(s.rejectionHistogram["out_of_scope_symbol"] == 2, "out_of_scope count mismatch"); + CHECK(s.rejectionHistogram["illegal_operation"] == 1, "illegal_operation count mismatch"); + PASS(); +} + +void test_operation_selection_histogram_counts_ops() { + TEST(operation_selection_histogram_counts_ops); + ConstrainedExecutionTelemetry t; + t.record(event("a", 3, 5, "rename", {}, 10, 20, true)); + t.record(event("b", 3, 5, "rename", {}, 10, 20, true)); + t.record(event("c", 3, 5, "update", {}, 10, 20, true)); + auto s = t.summarize(); + CHECK(s.opSelectionHistogram["rename"] == 2, "rename histogram mismatch"); + CHECK(s.opSelectionHistogram["update"] == 1, "update histogram mismatch"); + PASS(); +} + +void test_failed_events_with_empty_selected_op_do_not_count_selection() { + TEST(failed_events_with_empty_selected_op_do_not_count_selection); + ConstrainedExecutionTelemetry t; + t.record(event("a", 1, 1, "", {"unsupported_context"}, 5, 10, false)); + auto s = t.summarize(); + CHECK(s.opSelectionHistogram.empty(), "empty op should not be counted"); + PASS(); +} + +void test_baseline_zero_avoids_ratio_division_issue() { + TEST(baseline_zero_avoids_ratio_division_issue); + ConstrainedExecutionTelemetry t; + t.record(event("a", 1, 1, "rename", {}, 5, 0, true)); + auto s = t.summarize(); + CHECK(s.tokenSavingsRatio == 0.0, "ratio should be zero when baseline total is zero"); + PASS(); +} + +void test_negative_token_savings_supported() { + TEST(negative_token_savings_supported); + ConstrainedExecutionTelemetry t; + t.record(event("a", 1, 1, "rename", {}, 60, 50, true)); + auto s = t.summarize(); + CHECK(s.tokenSavings == -10, "negative savings should be represented"); + PASS(); +} + +void test_mixed_events_aggregate_all_metrics() { + TEST(mixed_events_aggregate_all_metrics); + ConstrainedExecutionTelemetry t; + t.record(event("a", 4, 7, "rename", {}, 20, 40, true)); + t.record(event("b", 2, 4, "", {"out_of_scope_symbol"}, 15, 30, false)); + t.record(event("c", 3, 8, "update", {}, 18, 36, true)); + auto s = t.summarize(); + CHECK(s.events == 3, "event count mismatch"); + CHECK(s.successCount == 2 && s.failureCount == 1, "success/failure mismatch"); + CHECK(s.rejectionHistogram["out_of_scope_symbol"] == 1, "rejection histogram mismatch"); + PASS(); +} + +int main() { + std::cout << "Step 537: Constrained Execution Telemetry\n"; + + test_empty_summary_is_zeroed(); // 1 + test_record_increments_event_count(); // 2 + test_summary_counts_success_and_failure(); // 3 + test_average_candidate_counts_are_computed(); // 4 + test_token_savings_computation(); // 5 + test_token_savings_ratio_computation(); // 6 + test_rejection_histogram_counts_reasons(); // 7 + test_operation_selection_histogram_counts_ops(); // 8 + test_failed_events_with_empty_selected_op_do_not_count_selection(); // 9 + test_baseline_zero_avoids_ratio_division_issue(); // 10 + test_negative_token_savings_supported(); // 11 + test_mixed_events_aggregate_all_metrics(); // 12 + + std::cout << "\nResults: " << passed << "/" << (passed + failed) << " passed\n"; + return failed == 0 ? 0 : 1; +} diff --git a/progress.md b/progress.md index 489aa3e..00dd239 100644 --- a/progress.md +++ b/progress.md @@ -9073,3 +9073,40 @@ contract legality for operation/symbol pairs. - `editor/src/ArgumentShapeValidator.h` within header-size limit (`113` <= `600`) - `editor/tests/step536_test.cpp` within test-file size guidance (`144` lines) - Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md` + +### Step 537: Constrained Execution Telemetry +**Status:** PASS (12/12 tests) + +Implements constrained execution telemetry to track candidate breadth, +selection paths, rejection reasons, and token-usage deltas versus unconstrained +baselines. + +**Files added:** +- `editor/src/ConstrainedExecutionTelemetry.h` - telemetry module: + - records per-taskitem constrained execution events + - aggregates success/failure counts and candidate breadth averages + - builds rejection-reason and selected-operation histograms + - computes constrained vs baseline token totals/savings/ratio +- `editor/tests/step537_test.cpp` - 12 tests covering: + - empty-summary behavior + - event recording/counting behavior + - success/failure aggregate behavior + - average candidate breadth computation + - token savings and ratio computation + - rejection and operation histogram behavior + - baseline-zero and negative-savings edge cases + - mixed-event aggregate behavior + +**Files modified:** +- `editor/CMakeLists.txt` - `step537_test` target + +**Verification run:** +- `cmake -S editor -B editor/build-native` - PASS +- `cmake --build editor/build-native --target step537_test step536_test` - PASS +- `./editor/build-native/step537_test` - PASS (12/12) +- `./editor/build-native/step536_test` - PASS (12/12) regression coverage + +**Architecture gate check:** +- `editor/src/ConstrainedExecutionTelemetry.h` within header-size limit (`75` <= `600`) +- `editor/tests/step537_test.cpp` within test-file size guidance (`175` lines) +- Header-only architecture and naming conventions remain aligned with `ARCHITECTURE.md`