Complete step471 problem description parser with tests

editor/CMakeLists.txt

@@ -3118,4 +3118,13 @@ target_link_libraries(step470_test PRIVATE
     tree_sitter_javascript tree_sitter_typescript
     tree_sitter_java tree_sitter_rust tree_sitter_go)
 
+add_executable(step471_test tests/step471_test.cpp)
+target_include_directories(step471_test PRIVATE src)
+target_link_libraries(step471_test PRIVATE
+    nlohmann_json::nlohmann_json
+    unofficial::tree-sitter::tree-sitter
+    tree_sitter_python tree_sitter_cpp tree_sitter_elisp
+    tree_sitter_javascript tree_sitter_typescript
+    tree_sitter_java tree_sitter_rust tree_sitter_go)
+
 # Step 12: Dear ImGui shell scaffolding created (main.cpp exists but not built due to dependencies)

editor/src/ArchitectProblemParser.h

@@ -0,0 +1,114 @@
+#pragma once
+
+// Step 471: Problem Description Parser
+// Heuristic structured requirement extraction from natural-language problem
+// descriptions with per-field confidence scores.
+
+#include <algorithm>
+#include <map>
+#include <string>
+#include <vector>
+
+struct ExtractedItem {
+    std::string text;
+    float confidence = 0.0f;
+};
+
+struct StructuredRequirements {
+    std::vector<ExtractedItem> functionalRequirements;
+    std::vector<ExtractedItem> nonFunctionalRequirements;
+    std::vector<ExtractedItem> platformConstraints;
+    std::vector<ExtractedItem> integrationRequirements;
+
+    bool hasFunctionalKeyword(const std::string& keyword) const {
+        for (const auto& r : functionalRequirements)
+            if (toLower(r.text).find(toLower(keyword)) != std::string::npos) return true;
+        return false;
+    }
+
+    bool hasPlatform(const std::string& keyword) const {
+        for (const auto& r : platformConstraints)
+            if (toLower(r.text).find(toLower(keyword)) != std::string::npos) return true;
+        return false;
+    }
+
+private:
+    static std::string toLower(std::string s) {
+        std::transform(s.begin(), s.end(), s.begin(),
+                       [](unsigned char c){ return static_cast<char>(std::tolower(c)); });
+        return s;
+    }
+};
+
+class ArchitectProblemParser {
+public:
+    static StructuredRequirements parse(const std::string& description) {
+        StructuredRequirements out;
+        auto text = toLower(description);
+
+        // Functional requirements
+        addIfPresent(text, {"api", "rest", "endpoint", "crud", "dashboard", "auth", "login",
+                            "signup", "search", "reporting", "notification", "upload"},
+                     out.functionalRequirements, 0.86f, 0.96f);
+
+        // Non-functional requirements
+        addIfPresent(text, {"performance", "fast", "latency", "scalable", "scale",
+                            "secure", "security", "reliable", "availability", "99.9",
+                            "throughput", "compliance"},
+                     out.nonFunctionalRequirements, 0.82f, 0.95f);
+
+        // Platform constraints
+        addIfPresent(text, {"web", "mobile", "ios", "android", "embedded", "server",
+                            "backend", "frontend", "cli", "desktop"},
+                     out.platformConstraints, 0.80f, 0.94f);
+
+        // Integration requirements
+        addIfPresent(text, {"postgres", "postgresql", "mysql", "sqlite", "redis", "kafka",
+                            "api", "stripe", "s3", "salesforce", "ldap", "oauth", "jwt"},
+                     out.integrationRequirements, 0.78f, 0.93f);
+
+        // Fallback generic functional intent if nothing extracted
+        if (out.functionalRequirements.empty()) {
+            out.functionalRequirements.push_back({"core application behavior", 0.55f});
+        }
+
+        return out;
+    }
+
+private:
+    static std::string toLower(std::string s) {
+        std::transform(s.begin(), s.end(), s.begin(),
+                       [](unsigned char c){ return static_cast<char>(std::tolower(c)); });
+        return s;
+    }
+
+    static void addIfPresent(const std::string& text,
+                             const std::vector<std::string>& keywords,
+                             std::vector<ExtractedItem>& bucket,
+                             float lowConf,
+                             float highConf) {
+        for (const auto& k : keywords) {
+            if (text.find(k) != std::string::npos) {
+                float conf = confidenceForKeyword(k, lowConf, highConf);
+                bucket.push_back({k, conf});
+            }
+        }
+        dedupe(bucket);
+    }
+
+    static float confidenceForKeyword(const std::string& k, float lowConf, float highConf) {
+        if (k.size() >= 7) return highConf;
+        if (k.size() >= 4) return (lowConf + highConf) * 0.5f;
+        return lowConf;
+    }
+
+    static void dedupe(std::vector<ExtractedItem>& items) {
+        std::map<std::string, float> best;
+        for (const auto& i : items) {
+            auto it = best.find(i.text);
+            if (it == best.end() || i.confidence > it->second) best[i.text] = i.confidence;
+        }
+        items.clear();
+        for (const auto& kv : best) items.push_back({kv.first, kv.second});
+    }
+};

editor/tests/step471_test.cpp

@@ -0,0 +1,157 @@
+// Step 471: Problem Description Parser Tests (12 tests)
+
+#include "ArchitectProblemParser.h"
+
+#include <iostream>
+
+static int passed = 0, failed = 0;
+#define TEST(name) { std::cout << "  " << #name << "... "; }
+#define PASS() { std::cout << "PASS\n"; ++passed; }
+#define FAIL(msg) { std::cout << "FAIL: " << msg << "\n"; ++failed; }
+#define CHECK(cond, msg) if (!(cond)) { FAIL(msg); return; } else {}
+
+void test_extracts_functional_requirements_api_auth() {
+    TEST(extracts_functional_requirements_api_auth);
+    auto r = ArchitectProblemParser::parse("Build a REST API with user auth and CRUD endpoints.");
+    CHECK(!r.functionalRequirements.empty(), "expected functional requirements");
+    CHECK(r.hasFunctionalKeyword("api"), "expected api functional requirement");
+    CHECK(r.hasFunctionalKeyword("auth"), "expected auth functional requirement");
+    PASS();
+}
+
+void test_extracts_nonfunctional_performance_security() {
+    TEST(extracts_nonfunctional_performance_security);
+    auto r = ArchitectProblemParser::parse("System must be fast, scalable, and secure.");
+    CHECK(!r.nonFunctionalRequirements.empty(), "expected non-functional requirements");
+    bool hasPerformance = false, hasSecurity = false;
+    for (const auto& it : r.nonFunctionalRequirements) {
+        if (it.text == "fast" || it.text == "performance" || it.text == "scalable") hasPerformance = true;
+        if (it.text == "secure" || it.text == "security") hasSecurity = true;
+    }
+    CHECK(hasPerformance, "missing performance/scalability extraction");
+    CHECK(hasSecurity, "missing security extraction");
+    PASS();
+}
+
+void test_extracts_platform_web_mobile() {
+    TEST(extracts_platform_web_mobile);
+    auto r = ArchitectProblemParser::parse("Need a web and mobile app for iOS and Android.");
+    CHECK(r.hasPlatform("web"), "missing web platform");
+    CHECK(r.hasPlatform("mobile"), "missing mobile platform");
+    CHECK(r.hasPlatform("ios"), "missing ios platform");
+    CHECK(r.hasPlatform("android"), "missing android platform");
+    PASS();
+}
+
+void test_extracts_integration_postgres_redis() {
+    TEST(extracts_integration_postgres_redis);
+    auto r = ArchitectProblemParser::parse("Integrate with PostgreSQL and Redis for data/cache.");
+    bool hasPg = false, hasRedis = false;
+    for (const auto& it : r.integrationRequirements) {
+        if (it.text == "postgres" || it.text == "postgresql") hasPg = true;
+        if (it.text == "redis") hasRedis = true;
+    }
+    CHECK(hasPg, "missing postgres integration");
+    CHECK(hasRedis, "missing redis integration");
+    PASS();
+}
+
+void test_confidence_scores_in_expected_range() {
+    TEST(confidence_scores_in_expected_range);
+    auto r = ArchitectProblemParser::parse("Build secure API with PostgreSQL.");
+    auto checkBucket = [](const std::vector<ExtractedItem>& items) {
+        for (const auto& i : items) {
+            if (i.confidence < 0.0f || i.confidence > 1.0f) return false;
+        }
+        return true;
+    };
+    CHECK(checkBucket(r.functionalRequirements), "functional confidence out of range");
+    CHECK(checkBucket(r.nonFunctionalRequirements), "non-functional confidence out of range");
+    CHECK(checkBucket(r.integrationRequirements), "integration confidence out of range");
+    PASS();
+}
+
+void test_deduplicates_keywords() {
+    TEST(deduplicates_keywords);
+    auto r = ArchitectProblemParser::parse("API API API with auth auth auth.");
+    int apiCount = 0;
+    for (const auto& it : r.functionalRequirements) if (it.text == "api") ++apiCount;
+    CHECK(apiCount == 1, "keyword should be deduplicated");
+    PASS();
+}
+
+void test_fallback_functional_requirement_when_none_found() {
+    TEST(fallback_functional_requirement_when_none_found);
+    auto r = ArchitectProblemParser::parse("We need something nice and maintainable.");
+    CHECK(!r.functionalRequirements.empty(), "expected fallback functional requirement");
+    CHECK(r.functionalRequirements[0].confidence <= 0.60f, "fallback should have low confidence");
+    PASS();
+}
+
+void test_detects_cli_platform_constraint() {
+    TEST(detects_cli_platform_constraint);
+    auto r = ArchitectProblemParser::parse("Build a CLI tool for log analysis.");
+    CHECK(r.hasPlatform("cli"), "missing CLI platform constraint");
+    PASS();
+}
+
+void test_detects_external_api_integration() {
+    TEST(detects_external_api_integration);
+    auto r = ArchitectProblemParser::parse("Service must call Stripe API and store results.");
+    bool hasStripe = false;
+    for (const auto& it : r.integrationRequirements) if (it.text == "stripe") hasStripe = true;
+    CHECK(hasStripe, "missing stripe integration extraction");
+    PASS();
+}
+
+void test_case_insensitive_extraction() {
+    TEST(case_insensitive_extraction);
+    auto r = ArchitectProblemParser::parse("Build REST Api with AUTH and PostgreSQL backend.");
+    CHECK(r.hasFunctionalKeyword("api"), "missing case-insensitive api extraction");
+    CHECK(r.hasFunctionalKeyword("auth"), "missing case-insensitive auth extraction");
+    PASS();
+}
+
+void test_extracts_multiple_categories_from_single_prompt() {
+    TEST(extracts_multiple_categories_from_single_prompt);
+    auto r = ArchitectProblemParser::parse(
+        "Create a web dashboard API, secure and scalable, with PostgreSQL integration.");
+    CHECK(!r.functionalRequirements.empty(), "missing functional extraction");
+    CHECK(!r.nonFunctionalRequirements.empty(), "missing non-functional extraction");
+    CHECK(!r.platformConstraints.empty(), "missing platform extraction");
+    CHECK(!r.integrationRequirements.empty(), "missing integration extraction");
+    PASS();
+}
+
+void test_longer_keywords_receive_higher_confidence() {
+    TEST(longer_keywords_receive_higher_confidence);
+    auto r = ArchitectProblemParser::parse("Need scalable performance.");
+    float scalableConf = 0.0f, fastConf = 0.0f;
+    for (const auto& it : r.nonFunctionalRequirements) {
+        if (it.text == "scalable") scalableConf = it.confidence;
+        if (it.text == "fast") fastConf = it.confidence;
+    }
+    CHECK(scalableConf >= fastConf, "longer keyword should not have lower confidence");
+    PASS();
+}
+
+int main() {
+    std::cout << "Step 471: Problem Description Parser Tests\n";
+
+    test_extracts_functional_requirements_api_auth();    // 1
+    test_extracts_nonfunctional_performance_security();  // 2
+    test_extracts_platform_web_mobile();                 // 3
+    test_extracts_integration_postgres_redis();          // 4
+    test_confidence_scores_in_expected_range();          // 5
+    test_deduplicates_keywords();                        // 6
+    test_fallback_functional_requirement_when_none_found(); // 7
+    test_detects_cli_platform_constraint();              // 8
+    test_detects_external_api_integration();             // 9
+    test_case_insensitive_extraction();                  // 10
+    test_extracts_multiple_categories_from_single_prompt(); // 11
+    test_longer_keywords_receive_higher_confidence();    // 12
+
+    std::cout << "\nResults: " << passed << "/" << (passed + failed)
+              << " passed\n";
+    return failed == 0 ? 0 : 1;
+}

progress.md

@@ -6440,3 +6440,32 @@ and self-hosting signal checks against `TransformEngineExtended.h`.
     - try/catch/throw/noexcept patterns
     - operator overloading + friend operators
     - initializer lists + STL/iterator pattern detection
+
+### Step 471: Problem Description Parser
+**Status:** PASS (12/12 tests)
+
+Starts Sprint 23 architect mode by extracting structured requirements from
+natural-language problem descriptions with per-item confidence scores.
+
+**Files added:**
+- `editor/src/ArchitectProblemParser.h` — structured requirement parser:
+  - output model: functional, non-functional, platform, integration buckets
+  - heuristic keyword extraction with confidence scoring
+  - deduplication behavior for repeated terms
+  - fallback functional requirement when extraction is sparse
+- `editor/tests/step471_test.cpp` — 12 tests covering:
+  - functional/non-functional/platform/integration extraction
+  - confidence range behavior
+  - case-insensitive matching and dedupe
+  - fallback behavior
+  - mixed multi-category prompt extraction
+- `editor/CMakeLists.txt` — `step471_test` target
+
+**Verification run:**
+- `cmake --build editor/build-native --target step471_test` — PASS
+- `./editor/build-native/step471_test` — PASS (12/12)
+- `./editor/build-native/step470_test` — PASS (8/8) regression coverage
+
+**Architecture gate check:**
+- `editor/src/ArchitectProblemParser.h` within header-size limit (`114` <= `600`)
+- `editor/tests/step471_test.cpp` within test-file size guidance (`157` lines)