From 95f88ee6e4ef4f4d50f38c558ef2af4978c29670 Mon Sep 17 00:00:00 2001 From: Bill Date: Fri, 21 Nov 2025 17:51:00 -0700 Subject: [PATCH] documents for new features and fixes added --- docs/build-cleanup-chrono.md | 79 ++++++++++++++++++++++++++ docs/marketing-business.md | 105 +++++++++++++++++++++++++++++++++++ docs/tech_fix.md | 83 +++++++++++++++++++++++++++ setup_db.sql | 1 + 4 files changed, 268 insertions(+) create mode 100644 docs/build-cleanup-chrono.md create mode 100644 docs/marketing-business.md create mode 100644 docs/tech_fix.md diff --git a/docs/build-cleanup-chrono.md b/docs/build-cleanup-chrono.md new file mode 100644 index 0000000..9793d19 --- /dev/null +++ b/docs/build-cleanup-chrono.md @@ -0,0 +1,79 @@ +Technical Task: Build Fixes & Profiler Cleanup +Date: 2025-11-21 Goal: Fix PyPI build failures and remove heavy profiling code from production wheels. + +1. Fix PyPI Build Failure (chrono-tz) + Issue: The build fails on Linux/Mac/Windows CI because chrono-tz requires complex C-bindings that are often missing in standard Python environments. Analysis: UncheckedIO returns Apache Arrow RecordBatch objects (where timestamps are raw i64). We do not need pyo3 to convert Rust chrono types directly to Python datetime objects. The chrono-tz feature in pyo3 is dead weight causing build failures. + +Action: Update Cargo.toml to remove the conflicting feature. + +Ini, TOML + +# Cargo.toml + +# OLD +# pyo3 = { version = "0.27.1", features = ["extension-module", "chrono-tz"] } + +# NEW (Remove "chrono-tz") +pyo3 = { version = "0.27.1", features = ["extension-module"] } +2. Productionize the Profiler (Remove Tracy) + Issue: The tracy-client and tracing-tracy crates add significant binary size and runtime overhead. They should not be present in the production library installed by users. Solution: Use Rust Feature Flags to make profiling "opt-in" for development only. + +Step A: Update Cargo.toml Define a profiling feature and make the heavy dependencies optional. + +Ini, TOML + +[features] +default = [] +# The "profiling" feature enables these two optional crates +profiling = ["dep:tracing-tracy", "dep:tracy-client"] + +[dependencies] +# Keep this (Lightweight facade) +tracing = "0.1.41" + +# Make these OPTIONAL (Heavy implementation) +tracing-tracy = { version = "0.11.2", optional = true } +tracy-client = { version = "0.17.6", optional = true } + +# ... other dependencies ... +Step B: Update src/lib.rs Wrap the profiler initialization logic so it only compiles when the flag is active. + +Rust + +// src/lib.rs + +#[pymodule] +fn unchecked_io(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + + // --- START PROFILER LOGIC --- + // This block now ONLY compiles if you run with `--features profiling` + #[cfg(feature = "profiling")] + { + use tracing_subscriber::layer::SubscriberExt; + use tracing_subscriber::util::SubscriberInitExt; + + // Initialize the Tracy layer + let _ = tracing_subscriber::registry() + .with(tracing_tracy::TracyLayer::default()) + .try_init(); + + println!("--- UncheckedIO: Profiling Enabled (Tracy) ---"); + } + // --- END PROFILER LOGIC --- + + m.add_function(wrap_pyfunction!(load_data_from_config, m)?)?; + m.add_function(wrap_pyfunction!(to_pandas_dataframe, m)?)?; + m.add_function(wrap_pyfunction!(profile_data, m)?)?; + Ok(()) +} +3. How to Build + For Development (With Profiler): Use this when you want to see the tracks in the Tracy GUI. + +Bash + +maturin develop --features profiling +For Production / PyPI (Zero Overhead): This builds the "clean" version for users. The profiler code is stripped out entirely. + +Bash + +maturin publish diff --git a/docs/marketing-business.md b/docs/marketing-business.md new file mode 100644 index 0000000..edbbd5e --- /dev/null +++ b/docs/marketing-business.md @@ -0,0 +1,105 @@ +# UncheckedIO: Business Strategy & Technical Roadmap + +**Date:** 2025-11-21 +**Status:** Strategic Planning +**Core Philosophy:** "The Configured Opinion" — We trade flexibility for raw, unchecked speed. + +--- + +## 1. Target Audience & Value Propositions + +We are not selling "a faster Pandas." We are selling **infrastructure efficiency**. + +### Primary Target: Data Platform & MLOps Engineers +* **Who they are:** The people managing Kubernetes clusters, Airflow DAGs, and AWS Fargate costs. They care about stability and resource usage. +* **The Pitch:** "A Zero-Copy Transport Layer for Postgres." +* **Key Selling Points:** + * **Cut Compute Costs:** Reduces CPU starvation by saturating network bandwidth. 10x faster serialization means 10x less time paying for vCPUs. + * **Memory Safety:** Written in Rust, ensuring no segfaults in your Python pipeline despite the speed. + * **Predictable Throughput:** Unlike generic drivers that choke on large datasets, `UncheckedIO` uses partitioned streaming to maintain constant memory usage regardless of data size. + +### Secondary Target: High-Frequency Trading (HFT) / Quant Developers +* **Who they are:** Developers who need to backtest models on massive tick-data history stored in Postgres. +* **The Pitch:** "The fastest way to get data from Disk to Arrow Memory." +* **Key Selling Points:** + * **Zero-Copy to Polars:** Output directly to Apache Arrow memory layouts, ready for immediate analysis or SIMD vectorization. + * **Bypass the overhead:** Skips the "Safety Checks" (schema validation) that cost milliseconds per query. You promise the data is clean; we promise to load it instantly. + +### Tertiary Target: Data Scientists (The Users, Not Buyers) +* **Who they are:** End users writing Jupyter Notebooks. +* **The Pitch:** "Stop waiting for `read_sql`." +* **Key Selling Points:** + * **Installation Simplicity:** `pip install unchecked_io`. No complex C dependencies or drivers to configure. + * **Drop-in Speed:** Replace a 5-minute load time with a 30-second load time. + +--- + +## 2. Optimization Strategy: The "Win" Plan + +We need to stop fighting "theoretical" bottlenecks and fix the "physical" ones. + +### 🔴 Focus On (High ROI) +1. **Infrastructure-Aware Tuning (The "Blast Radius"):** + * *Why:* The "astronomical" delay was caused by thrashing. + * *Action:* Implement the `num_cpus` detection to auto-scale the connection pool and partition count. Do not let the user guess wrong. +2. **Memory Management (The "String Killer"):** + * *Why:* 15M heap allocations for strings creates allocator contention. + * *Action:* Refactor the parser to use a reusable buffer or "Zero-Copy" string slicing from the raw stream before writing to Arrow. +3. **Database Interaction (The Index):** + * *Why:* As discovered, a missing index turns a parallel fetch into a DDoS attack. + * *Action:* Documentation MUST strictly warn users: *"If you use `blast_radius`, you MUST index your partition column."* + +### 🛑 Leave Alone (Diminishing Returns) +1. **Complex "Safe Mode" Logic:** + * *Why:* We are `UncheckedIO`. If we add too many safety checks/fallbacks for dirty data, we become just another slow connector (like ConnectorX). + * *Strategy:* Let it fail fast. If the data violates the config schema, panic. That is the contract. +2. **Micro-Optimizing the TCP Handshake:** + * *Why:* Connection pooling already solves 95% of this. Saving another 1ms here is irrelevant compared to the 500ms of Query Planning time. + +--- + +## 3. Configuration Matrix + +To be a "System-Level" tool, we need to expose the knobs that Platform Engineers expect. + +### Currently Configurable (The "MVP" Set) +These features are already present in the code or `config.yaml`: +1. **`connection_string`:** Standard Postgres URI. +2. **`query`:** The exact `COPY ... TO STDOUT (FORMAT binary)` command. Allows total control over the SQL execution plan. +3. **`schema`:** Explicit definition of Arrow Types (`Int64`, `Utf8`, etc.). Skips runtime type inference. +4. **`blast_radius` (Argument):** Controls the row-count per partition. Passed from Python to Rust. + +### Must-Add Configurations (To Delight Consumers) +These options solve specific "Enterprise" pain points: +1. **`worker_threads`:** + * *Value:* Allows limiting the library to use only 4 cores on a shared 64-core server. + * *Implementation:* Pass to `tokio::runtime::Builder::worker_threads()`. +2. **`pool_size`:** + * *Value:* Overrides the auto-detected defaults. Essential for databases with strict connection limits (e.g., AWS RDS limits). + * *Implementation:* Pass to `deadpool_postgres::Pool::builder().max_size()`. +3. **`batch_size` (Arrow):** + * *Value:* Controls the size of the Arrow RecordBatches. Critical for streaming data into ML models (e.g., "Give me 10k rows at a time"). + * *Implementation:* Currently aggregated at the end; needs to be exposed in the streaming loop. +4. **`danger_mode` (Explicit Toggle):** + * *Value:* A flag to strictly enforce "Panic on Error" vs. "Try to Recover." + * *Implementation:* As described in Sprint 4. + +--- + +## 4. The "Reputation Builder" Roadmap (Future Optimizations) + +These features serve two purposes: Extreme performance for the 1% of users who need it, and "Street Cred" for the author. + +1. **`io_uring` Network Layer:** + * *The Tech:* Replace `tokio-postgres` (epoll) with a custom `io_uring` implementation for Linux. + * *The Flex:* "True Asynchronous syscall batching." Reduces context switches during massive data transfers. + * *Status:* High effort, high reputation, low immediate business ROI (unless on 100Gbps networks). +2. **SIMD Parser (AVX-512 / NEON):** + * *The Tech:* Use `std::simd` or `portable-simd` to parse 16 integers at once from the binary stream. + * *The Flex:* "Saturating the memory bandwidth of modern CPUs." +3. **Custom Allocator Support (`mimalloc` / `jemalloc`):** + * *The Tech:* Allow users to swap the global allocator via a feature flag. + * *The Flex:* "Optimized for high-fragmentation environments." (Note: `mimalloc` is already configured for non-MSVC targets). +4. **Kernel-Bypass Networking (DPDK):** + * *The Tech:* Bypassing the OS network stack entirely. + * *The Flex:* The absolute theoretical limit of data transfer. (Overkill, but legendary). \ No newline at end of file diff --git a/docs/tech_fix.md b/docs/tech_fix.md new file mode 100644 index 0000000..06d2099 --- /dev/null +++ b/docs/tech_fix.md @@ -0,0 +1,83 @@ +Here is the technical summary of our debugging session, formatted as a Markdown file for your documentation. + +Markdown + +# Technical Post-Mortem: The "Thrashing" & Performance Optimization Log + +**Date:** 2025-11-21 +**Project:** UncheckedIO +**Topic:** Debugging "Astronomical" Task Switching Delays & Laptop vs. Cloud Discrepancies + +## 1. The Symptoms +* **The "Astronomical" Delay:** Increasing partition count (e.g., from 16 to 32) caused disproportionate slowdowns. Switching between tasks took seconds, not nanoseconds. +* **The "Tracy" Paradox:** The profiler showed "CPU Data Starved" and frequently crashed the runtime. When it crashed/detached, the library actually ran faster. +* **Platform Discrepancy:** The library performed significantly better on Google Colab (Server CPU) than on a high-spec Laptop (Consumer CPU) relative to competitors, despite the laptop having faster single-core speed. + +## 2. Root Cause Analysis + +### A. The Database Villain: "The Missing Index" +* **The Issue:** The `setup_db.sql` script created the table but **failed to index the `id` column**. +* **The Consequence:** The "Parallel Coordinator" splits the job into partitions (e.g., `WHERE id BETWEEN 0 AND 125000`). Without an index, Postgres performed a **Sequential Scan** (Full Table Scan) for *every single partition*. +* **The Impact:** Running 16 parallel workers triggered **16 simultaneous Full Table Scans**. This saturated memory bandwidth instantly, explaining the "CPU Data Starved" metric in Tracy. + +### B. The Concurrency Villain: "Pool Contention & Thrashing" +* **The Issue:** The connection pool was hardcoded to `max_size(20)` in `src/parser.rs`. +* **The "Alive" Misconception:** We assumed keeping connections "alive" allowed infinite parallelism. However, a connection is a single-lane bridge; it can only transport one query at a time. +* **The Bottleneck:** When spawning 800 tasks (partitions) against 20 connections: + * **20 Tasks** ran immediately. + * **780 Tasks** sat frozen in a RAM queue. +* **The "Thrashing":** The CPU spent ~90% of its time managing this queue—saving/restoring task states (Context Switching) and flushing L1/L2 caches—rather than parsing data. This overhead destroyed performance on the laptop (which has smaller caches than the Colab server). + +### C. The Red Herring: `io_uring` +* **Hypothesis:** We initially thought switching to `io_uring` (Linux asynchronous I/O) would solve the "waiting." +* **Verdict:** **Incorrect.** The bottleneck was not syscall overhead (which `io_uring` solves); it was **Logical Contention** (waiting for a DB connection) and **Physical Latency** (waiting for RAM/Network). + +--- + +## 3. The Solutions + +### Fix #1: Create the Index (Database Side) +Change the operation from a Sequential Scan (reading 20M rows x 16 times) to an **Index Scan** (jumping directly to the data). + +**File:** `setup_db.sql` +```sql +-- OLD +ANALYZE benchmark_table; + +-- NEW +CREATE INDEX idx_benchmark_id ON benchmark_table(id); +ANALYZE benchmark_table; +Fix #2: Dynamic Pool Sizing (Rust Side) +Remove the artificial speed limit. Match the pool size to the hardware capabilities so tasks never wait in a queue. + +File: src/parser.rs + +Rust + +// OLD +.max_size(20) + +// NEW +use num_cpus; // Add to Cargo.toml +let num_threads = num_cpus::get(); +.max_size(num_threads) // 1:1 mapping of Threads to Connections +Fix #3: Auto-Tuned "Blast Radius" +Instead of a hardcoded partition size or user guess, calculate the optimal partition count to balance +load without causing overhead. + +Strategy: 4 * CPU Cores partitions. + +Why: Enough chunks to allow "Work Stealing" (fast threads take more work), but few enough to prevent +connection setup overhead. + +Rust + +pub fn calculate_optimal_blast_radius(min_id: i64, max_id: i64) -> i64 { + let total_rows = max_id - min_id; + let target_partitions = num_cpus::get() * 4; + total_rows / (target_partitions as i64) +} +4. Conclusion +The library's "Schema-on-Read" optimization (trusting the config) was working perfectly. +The performance loss was due to infrastructure bottlenecks (DB Index and Connection Pool limits) +that forced the CPU to manage queues instead of parsing data. \ No newline at end of file diff --git a/setup_db.sql b/setup_db.sql index 99b4251..f6a8e91 100644 --- a/setup_db.sql +++ b/setup_db.sql @@ -39,5 +39,6 @@ SELECT FROM generate_series(1, 20000000) s(i); -- 4. Analyze the table for better query planning (good practice) +CREATE INDEX idx_benchmark_id ON benchmark_table(id); ANALYZE benchmark_table;