Fix all four Phase 1 correctness bugs (dynamic buffer, DLPack ownership, thread safety, padding)

- 1.1: Remove hardcoded 4096*512 buffer from PinnedBatcher; allocate host+device per-call
- 1.2: Wrap CudaSlice in DLPackContext owned by DLManagedTensor.manager_ctx; deleter frees
  device memory when PyTorch releases tensor — no more silent overwrite across calls
- 1.3: batch_encode_to_gpu now takes &self (no mutable shared state); TokenizerEngine wraps
  Arc<PinnedBatcher> and encode_batch takes &self — safe for concurrent Python threads
- 1.4: seq_len = max across all encodings; host buffer prefilled with pad_id before token write

Also: switch cudarc gpu feature from cuda-version-from-build-system to cuda-12050 to fix
build on machines with CUDA 13.x (cudarc 0.11.9 only knows up to 12.5)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

.github/workflows/publish.yml

@@ -1,75 +1,76 @@
 name: Publish to PyPI
 
-# This workflow is triggered when you push a new version tag (e.g., v0.1.0, v1.2.3)
 on:
   push:
     tags:
-      - 'v[0-9]+.[0-9]+.[0-9]+*' # Matches v0.1.0 or v1.0.0-rc1, etc.
+      - 'v[0-9]+.[0-9]+.[0-9]+*'
-  # Allows manual triggering from the GitHub Actions UI
   workflow_dispatch:
 
 jobs:
-  build_wheels:
+  build_and_publish:
-    name: Build wheels on ${{ matrix.os }}
+    name: Build and Publish All Wheels
     runs-on: ${{ matrix.os }}
-    # Define a matrix to build for common operating systems and architectures
     strategy:
       matrix:
-        include:
+        os: [ubuntu-latest, macos-latest, windows-latest]
-          - os: ubuntu-latest
+        target: [x86_64]
-            target: x86_64
-          - os: macos-latest
-            target: x86_64
-          - os: windows-latest
-            target: x86_64
     
     steps:
       - uses: actions/checkout@v4
       
+      # Only setup python for non-linux, or let maturin handle linux via docker
       - name: Set up Python
+        if: runner.os != 'Linux'
         uses: actions/setup-python@v5
         with:
-          python-version: '3.12' # Target a modern Python version
+          python-version: '3.12' 
+          # Note: On Windows/Mac, to build for multiple python versions, 
+          # you usually need multiple setup-python steps or a matrix of python versions.
+          # But for now, let's at least get the artifacts downloading correctly.
+          # Ideally, you remove this and let maturin find python, but GitHub runners 
+          # might only have one default.
           
       - name: Install Rust
         uses: dtolnay/rust-toolchain@stable
         with:
           toolchain: stable
 
-      - name: Build wheels with Maturin
+      - name: Build wheel
         uses: PyO3/maturin-action@v1
         with:
-          # Use the target from the matrix (x86_64)
           target: ${{ matrix.target }}
-          # Build a manylinux-compatible wheel for Linux/Colab users
+          manylinux: auto # This triggers the Docker container on Linux for multi-python builds
-          manylinux: auto
           command: build
           args: --release --out dist --find-interpreter
 
-      - name: Upload built wheels as Artifact
+      - name: Upload wheel artifact
         uses: actions/upload-artifact@v4
         with:
-          name: wheels
+          name: wheel-${{ matrix.os }}-${{ matrix.target }}
-          path: dist
+          path: dist/*.whl
 
   publish:
-    name: Publish to PyPI
+    name: Publish All Wheels
-    needs: [build_wheels]
+    needs: [build_and_publish]
     runs-on: ubuntu-latest
-    # This step will only run if the 'build_wheels' job completed successfully
-    if: startsWith(github.ref, 'refs/tags/')
 
     steps:
-      - uses: actions/download-artifact@v4
+    - name: Install maturin
-        with:
+      run: pip install maturin
-          name: wheels
-          path: dist
 
-      - name: Publish to PyPI
+    # FIX: Use merge-multiple to flatten everything into 'dist' automatically
-        uses: PyO3/maturin-action@v1
+    - name: Download ALL wheels
-        with:
+      uses: actions/download-artifact@v4
-          # Use the secret we configured on GitHub
+      with:
-          pypi-token: ${{ secrets.PYPI_API_TOKEN }}
+        path: dist
-          # Command to upload all wheels in the 'dist' directory
+        pattern: wheel-*
-          command: upload
+        merge-multiple: true
-          args: --skip-existing --non-interactive
+
+    - name: List Files (Debug)
+      run: ls -la dist
+
+    - name: Upload to PyPI
+      env:
+        MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}
+      run: |
+        maturin upload --skip-existing --non-interactive dist/*

.github/workflows/publish2.yml

@@ -0,0 +1,80 @@
+name: Publish to PyPI
+
+on:
+  push:
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+*'
+  workflow_dispatch:
+
+jobs:
+  build_and_publish:
+    name: Build and Publish All Wheels
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        target: [x86_64]
+    
+    steps:
+      - uses: actions/checkout@v4
+      
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: stable
+
+      - name: Build wheel
+        uses: PyO3/maturin-action@v1
+        with:
+          target: ${{ matrix.target }}
+          command: build
+          args: --release --out dist --find-interpreter
+
+      - name: Upload wheel artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: wheel-${{ matrix.os }}-${{ matrix.target }}
+          path: dist/*.whl
+
+  publish:
+    name: Publish All Wheels
+    needs: [build_and_publish]
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Install maturin
+      run: pip install maturin
+
+    - name: Download ALL wheels
+      uses: actions/download-artifact@v4
+      with:
+        path: dist
+
+    - name: Flatten and upload
+      env:
+        MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }}  # ← CORRECT TOKEN VAR
+      run: |
+        mkdir -p dist/flattened
+        rm -rf dist/flattened/* 
+        
+        # Copy EVERY wheel
+        find dist -name "*.whl" | while read wheel; do
+          cp "$wheel" dist/flattened/
+        done
+        
+        echo "=== TOTAL WHEELS ==="
+        ls -la dist/flattened/*.whl | wc -l
+        
+        cd dist/flattened
+        echo "=== UPLOADING $(ls *.whl | wc -l) WHEELS ==="
+        ls -la *.whl | head -5
+        
+        # TWINE fallback (double protection)
+        export TWINE_USERNAME=__token__
+        export TWINE_PASSWORD=${{ secrets.PYPI_API_TOKEN }}
+        maturin upload --skip-existing --non-interactive *.whl

Cargo.lock

@@ -118,7 +118,7 @@ dependencies = [
  "arrow-schema",
  "arrow-select",
  "atoi",
- "base64",
+ "base64 0.22.1",
  "chrono",
  "comfy-table",
  "half",
@@ -297,6 +297,12 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
 
+[[package]]
+name = "base64"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8"
+
 [[package]]
 name = "base64"
 version = "0.22.1"
@@ -395,6 +401,19 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "console"
+version = "0.15.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8"
+dependencies = [
+ "encode_unicode",
+ "libc",
+ "once_cell",
+ "unicode-width",
+ "windows-sys 0.59.0",
+]
+
 [[package]]
 name = "const-random"
 version = "0.1.18"
@@ -430,6 +449,25 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "crossbeam-deque"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-epoch"
+version = "0.9.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+dependencies = [
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "crossbeam-utils"
 version = "0.8.21"
@@ -473,6 +511,68 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "cuda-config"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ee74643f7430213a1a78320f88649de309b20b80818325575e393f848f79f5d"
+dependencies = [
+ "glob",
+]
+
+[[package]]
+name = "cuda-driver-sys"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d4c552cc0de854877d80bcd1f11db75d42be32962d72a6799b88dcca88fffbd"
+dependencies = [
+ "cuda-config",
+]
+
+[[package]]
+name = "cudarc"
+version = "0.11.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a5bd4d1eee570c3b2ac64ed114125517dd1e541d88dd28fc259f1de4dba8d60"
+dependencies = [
+ "libloading",
+]
+
+[[package]]
+name = "darling"
+version = "0.20.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
+dependencies = [
+ "darling_core",
+ "darling_macro",
+]
+
+[[package]]
+name = "darling_core"
+version = "0.20.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e"
+dependencies = [
+ "fnv",
+ "ident_case",
+ "proc-macro2",
+ "quote",
+ "strsim",
+ "syn",
+]
+
+[[package]]
+name = "darling_macro"
+version = "0.20.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
+dependencies = [
+ "darling_core",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "deadpool"
 version = "0.12.3"
@@ -508,6 +608,37 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "derive_builder"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947"
+dependencies = [
+ "derive_builder_macro",
+]
+
+[[package]]
+name = "derive_builder_core"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8"
+dependencies = [
+ "darling",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "derive_builder_macro"
+version = "0.20.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c"
+dependencies = [
+ "derive_builder_core",
+ "syn",
+]
+
 [[package]]
 name = "digest"
 version = "0.10.7"
@@ -519,12 +650,33 @@ dependencies = [
  "subtle",
 ]
 
+[[package]]
+name = "either"
+version = "1.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
+
+[[package]]
+name = "encode_unicode"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
+
 [[package]]
 name = "equivalent"
 version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
 
+[[package]]
+name = "esaxx-rs"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6"
+dependencies = [
+ "cc",
+]
+
 [[package]]
 name = "event-listener"
 version = "5.4.1"
@@ -568,6 +720,12 @@ dependencies = [
  "rustc_version",
 ]
 
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
 [[package]]
 name = "futures-channel"
 version = "0.3.31"
@@ -671,6 +829,12 @@ dependencies = [
  "wasip2",
 ]
 
+[[package]]
+name = "glob"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
+
 [[package]]
 name = "half"
 version = "2.7.1"
@@ -734,6 +898,12 @@ dependencies = [
  "cc",
 ]
 
+[[package]]
+name = "ident_case"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
+
 [[package]]
 name = "indexmap"
 version = "2.12.0"
@@ -744,6 +914,19 @@ dependencies = [
  "hashbrown",
 ]
 
+[[package]]
+name = "indicatif"
+version = "0.17.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235"
+dependencies = [
+ "console",
+ "number_prefix",
+ "portable-atomic",
+ "unicode-width",
+ "web-time",
+]
+
 [[package]]
 name = "indoc"
 version = "2.0.7"
@@ -753,6 +936,24 @@ dependencies = [
  "rustversion",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57"
+dependencies = [
+ "either",
+]
+
+[[package]]
+name = "itertools"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.15"
@@ -838,6 +1039,16 @@ version = "0.2.177"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976"
 
+[[package]]
+name = "libloading"
+version = "0.8.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55"
+dependencies = [
+ "cfg-if",
+ "windows-link 0.2.1",
+]
+
 [[package]]
 name = "libm"
 version = "0.2.15"
@@ -893,6 +1104,22 @@ dependencies = [
  "tracing-subscriber",
 ]
 
+[[package]]
+name = "macro_rules_attribute"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520"
+dependencies = [
+ "macro_rules_attribute-proc_macro",
+ "paste",
+]
+
+[[package]]
+name = "macro_rules_attribute-proc_macro"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30"
+
 [[package]]
 name = "matchers"
 version = "0.2.0"
@@ -946,6 +1173,12 @@ dependencies = [
  "libmimalloc-sys",
 ]
 
+[[package]]
+name = "minimal-lexical"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
+
 [[package]]
 name = "mio"
 version = "1.1.0"
@@ -957,6 +1190,28 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
+[[package]]
+name = "monostate"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67"
+dependencies = [
+ "monostate-impl",
+ "serde",
+ "serde_core",
+]
+
+[[package]]
+name = "monostate-impl"
+version = "0.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "ndarray"
 version = "0.16.1"
@@ -972,6 +1227,16 @@ dependencies = [
  "rawpointer",
 ]
 
+[[package]]
+name = "nom"
+version = "7.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
+dependencies = [
+ "memchr",
+ "minimal-lexical",
+]
+
 [[package]]
 name = "nu-ansi-term"
 version = "0.50.3"
@@ -1029,6 +1294,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "number_prefix"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
+
 [[package]]
 name = "numpy"
 version = "0.27.0"
@@ -1052,6 +1323,28 @@ version = "1.21.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
 
+[[package]]
+name = "onig"
+version = "6.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "336b9c63443aceef14bea841b899035ae3abe89b7c486aaf4c5bd8aafedac3f0"
+dependencies = [
+ "bitflags",
+ "libc",
+ "once_cell",
+ "onig_sys",
+]
+
+[[package]]
+name = "onig_sys"
+version = "69.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7f86c6eef3d6df15f23bcfb6af487cbd2fed4e5581d58d5bf1f5f8b7f6727dc"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
+
 [[package]]
 name = "parking"
 version = "2.2.1"
@@ -1081,6 +1374,12 @@ dependencies = [
  "windows-link 0.2.1",
 ]
 
+[[package]]
+name = "paste"
+version = "1.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
+
 [[package]]
 name = "percent-encoding"
 version = "2.3.2"
@@ -1136,6 +1435,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
+[[package]]
+name = "pkg-config"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
+
 [[package]]
 name = "portable-atomic"
 version = "1.11.1"
@@ -1157,14 +1462,14 @@ version = "0.6.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fbef655056b916eb868048276cfd5d6a7dea4f81560dfd047f97c8c6fe3fcfd4"
 dependencies = [
- "base64",
+ "base64 0.22.1",
  "byteorder",
  "bytes",
  "fallible-iterator",
  "hmac",
  "md-5",
  "memchr",
- "rand",
+ "rand 0.9.2",
  "sha2",
  "stringprep",
 ]
@@ -1200,9 +1505,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3"
-version = "0.27.1"
+version = "0.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "37a6df7eab65fc7bee654a421404947e10a0f7085b6951bf2ea395f4659fb0cf"
+checksum = "fa8e48c12afdeb26aa4be4e5c49fb5e11c3efa0878db783a960eea2b9ac6dd19"
 dependencies = [
  "chrono",
  "chrono-tz",
@@ -1241,18 +1546,27 @@ dependencies = [
 
 [[package]]
 name = "pyo3-build-config"
-version = "0.27.1"
+version = "0.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f77d387774f6f6eec64a004eac0ed525aab7fa1966d94b42f743797b3e395afb"
+checksum = "bc1989dbf2b60852e0782c7487ebf0b4c7f43161ffe820849b56cf05f945cee1"
 dependencies = [
  "target-lexicon",
 ]
 
 [[package]]
-name = "pyo3-ffi"
+name = "pyo3-dlpack"
-version = "0.27.1"
+version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dd13844a4242793e02df3e2ec093f540d948299a6a77ea9ce7afd8623f542be"
+checksum = "e2f4a0157d48883374fc6b3bf5f5b06f8563abb59113ab347ba8f11b6d9b9721"
+dependencies = [
+ "pyo3",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c808286da7500385148930152e54fb6883452033085bf1f857d85d4e82ca905c"
 dependencies = [
  "libc",
  "pyo3-build-config",
@@ -1260,9 +1574,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros"
-version = "0.27.1"
+version = "0.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "eaf8f9f1108270b90d3676b8679586385430e5c0bb78bb5f043f95499c821a71"
+checksum = "83a0543c16be0d86cf0dbf2e2b636ece9fd38f20406bb43c255e0bc368095f92"
 dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
@@ -1272,9 +1586,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros-backend"
-version = "0.27.1"
+version = "0.27.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70a3b2274450ba5288bc9b8c1b69ff569d1d61189d4bff38f8d22e03d17f932b"
+checksum = "2a00da2ce064dcd582448ea24a5a26fa9527e0483103019b741ebcbe632dcd29"
 dependencies = [
  "heck",
  "proc-macro2",
@@ -1298,14 +1612,35 @@ version = "5.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
 
+[[package]]
+name = "rand"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
+dependencies = [
+ "libc",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.4",
+]
+
 [[package]]
 name = "rand"
 version = "0.9.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
 dependencies = [
- "rand_chacha",
+ "rand_chacha 0.9.0",
- "rand_core",
+ "rand_core 0.9.3",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.6.4",
 ]
 
 [[package]]
@@ -1315,7 +1650,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
 dependencies = [
  "ppv-lite86",
- "rand_core",
+ "rand_core 0.9.3",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
+dependencies = [
+ "getrandom 0.2.16",
 ]
 
 [[package]]
@@ -1333,6 +1677,37 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
 
+[[package]]
+name = "rayon"
+version = "1.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f"
+dependencies = [
+ "either",
+ "rayon-core",
+]
+
+[[package]]
+name = "rayon-cond"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "059f538b55efd2309c9794130bc149c6a553db90e9d99c2030785c82f0bd7df9"
+dependencies = [
+ "either",
+ "itertools 0.11.0",
+ "rayon",
+]
+
+[[package]]
+name = "rayon-core"
+version = "1.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
+dependencies = [
+ "crossbeam-deque",
+ "crossbeam-utils",
+]
+
 [[package]]
 name = "redox_syscall"
 version = "0.5.18"
@@ -1541,6 +1916,18 @@ dependencies = [
  "windows-sys 0.60.2",
 ]
 
+[[package]]
+name = "spm_precompiled"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326"
+dependencies = [
+ "base64 0.13.1",
+ "nom",
+ "serde",
+ "unicode-segmentation",
+]
+
 [[package]]
 name = "stringprep"
 version = "0.1.5"
@@ -1552,6 +1939,12 @@ dependencies = [
  "unicode-properties",
 ]
 
+[[package]]
+name = "strsim"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
+
 [[package]]
 name = "strum"
 version = "0.26.3"
@@ -1647,6 +2040,38 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
 
+[[package]]
+name = "tokenizers"
+version = "0.20.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3b08cc37428a476fc9e20ac850132a513a2e1ce32b6a31addf2b74fa7033b905"
+dependencies = [
+ "aho-corasick",
+ "derive_builder",
+ "esaxx-rs",
+ "getrandom 0.2.16",
+ "indicatif",
+ "itertools 0.12.1",
+ "lazy_static",
+ "log",
+ "macro_rules_attribute",
+ "monostate",
+ "onig",
+ "paste",
+ "rand 0.8.5",
+ "rayon",
+ "rayon-cond",
+ "regex",
+ "regex-syntax",
+ "serde",
+ "serde_json",
+ "spm_precompiled",
+ "thiserror",
+ "unicode-normalization-alignments",
+ "unicode-segmentation",
+ "unicode_categories",
+]
+
 [[package]]
 name = "tokio"
 version = "1.48.0"
@@ -1694,7 +2119,7 @@ dependencies = [
  "pin-project-lite",
  "postgres-protocol",
  "postgres-types",
- "rand",
+ "rand 0.9.2",
  "socket2",
  "tokio",
  "tokio-util",
@@ -1815,7 +2240,7 @@ checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb"
 
 [[package]]
 name = "unchecked-io"
-version = "0.1.0"
+version = "0.1.7"
 dependencies = [
  "anyhow",
  "arrow",
@@ -1824,14 +2249,19 @@ dependencies = [
  "bytes",
  "chrono",
  "chrono-tz",
+ "cuda-driver-sys",
+ "cudarc",
  "deadpool-postgres",
  "futures-util",
  "mimalloc",
  "num_cpus",
  "pyo3",
  "pyo3-arrow",
+ "pyo3-dlpack",
+ "rayon",
  "serde",
  "serde_yaml",
+ "tokenizers",
  "tokio",
  "tokio-postgres",
  "tracing",
@@ -1861,18 +2291,39 @@ dependencies = [
  "tinyvec",
 ]
 
+[[package]]
+name = "unicode-normalization-alignments"
+version = "0.1.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de"
+dependencies = [
+ "smallvec",
+]
+
 [[package]]
 name = "unicode-properties"
 version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"
 
+[[package]]
+name = "unicode-segmentation"
+version = "1.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493"
+
 [[package]]
 name = "unicode-width"
 version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
 
+[[package]]
+name = "unicode_categories"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
+
 [[package]]
 name = "unindent"
 version = "0.2.4"
@@ -1985,6 +2436,16 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "web-time"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
 [[package]]
 name = "whoami"
 version = "1.6.1"
@@ -2135,6 +2596,15 @@ dependencies = [
  "windows-link 0.2.1",
 ]
 
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
 [[package]]
 name = "windows-sys"
 version = "0.60.2"

Cargo.toml

@@ -1,66 +1,56 @@
 [package]
 name = "unchecked-io"
-version = "0.1.0"
+version = "0.1.7"
-authors = ["Billthemaker"] # Replace with your name or alias
+authors = ["Billthemaker"]
-license = "Apache-2.0" # Good practice for open-source
+license = "Apache-2.0"
 edition = "2024"
 
 [lib]
 name = "unchecked_io"
 crate-type = ["cdylib", "rlib"]
 
-[dependencies]
+[features]
-# 1. Python Bindings for FFI
+default = []
-pyo3 = { version = "0.27.1", features = ["extension-module"] }
+gpu = [
-chrono-tz = "0.10"
+    "tokenizers",
+    "cudarc",
+    "pyo3-dlpack",
+    "cudarc/cuda-12050",
+    "dep:cuda-driver-sys"
+]
+profiling = ["dep:tracing", "dep:tracing-subscriber", "dep:tracing-tracy"]
 
-# 2. Configuration Parsing (YAML)
+[dependencies]
+pyo3 = { version = "=0.27.0", features = ["extension-module", "chrono-tz", "chrono"] }
 serde = { version = "1.0", features = ["derive"] }
 serde_yaml = "0.9"
-
+arrow = { version = "57.0.0", features = ["prettyprint"] }
-# 3. Apache Arrow and Data Handling
-# FIX: Add the "compute" feature to get concat_batches
-arrow = "57.0.0"
-
-# 4. Asynchronous Runtime (Essential for I/O and Postgres)
 tokio = { version = "1.37", features = ["full"] }
-
-
-# 5. Database Connection (Postgres)
 tokio-postgres = "0.7"
 deadpool-postgres = "0.14"
-
-# 6. Error Handling Crate
 anyhow = "1.0"
-
-# 7. Futures Utilities
 futures-util = "0.3"
-
-# 8. Byte Buffer Management
 bytes = "1.6"
-
-# 9. Binary Data Reading (NEW)
 byteorder = "1.5"
-
-# 10. Timestamp Handling (NEW)
 chrono = "0.4"
-
+chrono-tz = "=0.10.4"
-# 11. UUID Handling (NEW)
 uuid = { version = "1.8", features = ["serde", "v4"] }
-
-# 12. Arrow <-> Python Bridge (NEW)
 pyo3-arrow = "0.15.0"
-
-# 13. System CPU Count (NEW)
 num_cpus = "1.16"
+async-channel = "2.5.0"
+rayon = "1.10"
+
+# Optional dependencies enabled by features
+# UPGRADED: 0.20 supports modern Llama tokenizer exports
+tokenizers = { version = "0.20", optional = true }
+cudarc = { version = "0.11", features = ["driver"], optional = true }
+pyo3-dlpack = { version = "0.1.0", optional = true }
+cuda-driver-sys = { version = "0.3.0", optional = true }
 
 tracing-subscriber = { version = "0.3", features = ["registry", "env-filter"], optional = true}
 tracing-tracy = { version = "=0.11.2", optional = true }
 tracing = {version ="0.1.41", optional = true }
 
-async-channel = "2.5.0"
-
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
 mimalloc = { version = "0.1.39" }
 
-async-channel = "2.3"

HANDOFF-2026-04-20.md

@@ -0,0 +1,67 @@
+# Handoff — 2026-04-20
+
+## What Was Decided This Session
+
+- **Commercialization path:** Licensing first, IP sale after traction. Wyoming LLC before any contract.
+- **Language strategy:** Rust is source of truth. C++ generated via Whetstone + manual library substitution as enterprise deliverable. Both versions is a deliberate differentiator.
+- **Primary buyers:** GPU cloud providers and LLM inference companies (CoreWeave, Lambda Labs, Together AI, Fireworks AI, Modal). Not PyTorch/Meta.
+- **Hugging Face** is a secondary buyer worth a conversation — they own the `tokenizers` crate this code already uses.
+- **NVIDIA contact:** Approach after H100 benchmark. Send Nsight trace + one paragraph. Not before.
+- **NM funding:** Apply for NMSBA (Sandia/LANL hardware access). Skip money grants for now.
+- **Outsourcing:** Friend's software business for non-core work. 60-70% of contract value for defined scope. IP assignment clause required in writing first.
+
+## Documents Created This Session
+
+| File | Purpose |
+|---|---|
+| `ROADMAP.md` | Full technical development plan, Phases 1–5 |
+| `docs/talk-10min.md` | 10-minute presentation structure for Wednesday |
+| `docs/commercialization-strategy.md` | Pricing, B2B sales process, structure, outsourcing |
+| `HANDOFF-2026-04-20.md` | This file |
+
+---
+
+## This Week — Time-Sensitive (in order)
+
+### Before Wednesday
+- [ ] Fix Phase 1.1 — dynamic buffer allocation (`src/llm.rs`, `PinnedBatcher::new`)
+- [ ] Fix Phase 1.2 — buffer isolation / DLPack overwrite bug
+- [ ] Fix Phase 1.3 — thread-safe engine pool (`TokenizerEngine` takes `&mut self`)
+- [ ] Fix Phase 1.4 — padding for variable-length sequences
+
+### Wednesday
+- [ ] Rent H100 on Lambda Labs (~$2 for 30 min)
+- [ ] Run `benchmark_gpu.py` under `nsys profile --stats=true python benchmark_gpu.py`
+- [ ] Capture Nsight `.nsys-rep` file and download it
+- [ ] Note the speedup number — this is the headline for the talk
+
+### Wednesday/Thursday
+- [ ] Update `docs/talk-10min.md` Slide 6 with the H100 result
+- [ ] Open Nsight Systems GUI locally, load the `.nsys-rep` file, screenshot the timeline showing both pipelines
+
+### Thursday (if someone gives you a contact after the talk)
+- [ ] Do not show GPU source code without NDA signed first
+- [ ] First call is discovery — ask about their infrastructure before pitching
+- [ ] Pricing range if pushed: $50K–$150K annually depending on scale
+
+---
+
+## Next Session Priorities (after this week)
+
+1. **H100 setup script** — a single paste-into-terminal script for the cloud instance (was going to write this, ran out of session). Needs: Rust install, maturin build with `--features gpu`, nsys profile run, output capture.
+2. **Phase 1 fixes** — if not completed before Wednesday, finish these
+3. **vLLM integration** — Phase 2.1, this is the demo that sells the concept
+4. **NM anonymous LLC** — straightforward to form locally, no member names in public filings, no annual report requirement
+5. **C++ version scoping** — after Phase 1 is solid, assess Whetstone transpilation of `llm.rs`
+
+---
+
+## Key Technical Context for Next Session
+
+- Repo is cloned at `/home/bill/Documents/unchecked-io/`, currently on `gpu-zero` branch
+- GPU feature is compiled with `maturin develop --release --features gpu`
+- The four Phase 1 bugs are documented in detail in `ROADMAP.md` Phase 1 section
+- `src/llm.rs` is the file to fix — `PinnedBatcher` struct, `batch_encode_to_gpu` method
+- `src/lib.rs` `TokenizerEngine` wraps `PinnedBatcher` and is the Python-facing class
+- chrono-tz is pinned at `=0.10.4` — do not change this, it was hard-won
+- `gpu-zero` branch is private; `main` branch is open source Apache 2.0

HANDOFF-2026-04-21.md

@@ -0,0 +1,84 @@
+# Handoff — 2026-04-21
+
+## What Was Done This Session
+
+Fixed all four Phase 1 correctness bugs. Code compiles clean on temp-ubuntu with CUDA 13.1.
+
+---
+
+## Phase 1 Bugs — ALL FIXED
+
+### 1.1 Dynamic buffer allocation — DONE
+**Old behavior:** `max_elements = 4096 * 512` hardcoded in `PinnedBatcher::new`. Any batch exceeding ~2M tokens returned an error.
+**Fix:** Removed `gpu_buffer`, `host_buffer`, and `max_elements` from the `PinnedBatcher` struct entirely. Both buffers are now allocated per-call based on actual `batch_size * seq_len`.
+
+### 1.2 Buffer isolation (DLPack overwrite) — DONE
+**Old behavior:** Every call overwrote the same `CudaSlice`. Any tensor from call N got corrupted by call N+1.
+**Fix:** Each call allocates its own device buffer and wraps it in a `DLPackContext` struct stored in `DLManagedTensor.manager_ctx`. The deleter frees both the device memory (by dropping `CudaSlice<i64>`) and the shape array when PyTorch releases the tensor. Each returned tensor now owns its allocation.
+
+### 1.3 Thread safety — DONE
+**Old behavior:** `batch_encode_to_gpu` took `&mut self`. `TokenizerEngine.encode_batch` took `&mut self`. Not usable from multiple Python threads.
+**Fix:** Since `PinnedBatcher` no longer has mutable shared state (buffers are per-call), `batch_encode_to_gpu` now takes `&self`. `TokenizerEngine` wraps `Arc<PinnedBatcher>` instead of owning it directly. `encode_batch` takes `&self`. Concurrent calls from Python threads are safe.
+
+### 1.4 Variable sequence length + padding — DONE
+**Old behavior:** `seq_len = encodings[0].len()` — assumed uniform length. Variable-length batches produced wrong tensor shape.
+**Fix:** `seq_len = encodings.iter().map(|e| e.len()).max()`. Host buffer prefilled with `pad_id` (from `tokenizer.get_padding()`, fallback to 0). Rayon fill loop only writes actual tokens; shorter sequences retain the pad value.
+
+---
+
+## Build Issue Discovered and Resolved
+
+`cudarc 0.11.9` panics at build time on CUDA 13.1 (only knows 12.5 and below). This machine has CUDA 13.1.
+
+**Fix:** Changed the `gpu` feature in `Cargo.toml` from `cudarc/cuda-version-from-build-system` to `cudarc/cuda-12050`. This hard-pins to CUDA 12.5 headers and skips the `nvcc` version check. CUDA 13.x is backward-compatible with 12.5 driver API at runtime. If you ever upgrade cudarc to 0.12+, revert this change and re-test.
+
+---
+
+## Files Changed
+
+| File | What Changed |
+|------|-------------|
+| `src/llm.rs` | Full rewrite of `PinnedBatcher` — removed static buffers, per-call allocation, `DLPackContext` for ownership, `&self` throughout |
+| `src/lib.rs` | `TokenizerEngine.inner` changed to `Arc<PinnedBatcher>`, `encode_batch` takes `&self`, added `use std::sync::Arc` |
+| `Cargo.toml` | `gpu` feature: `cuda-version-from-build-system` → `cuda-12050` |
+
+---
+
+## This Week — What's Left (in order)
+
+### Today / Tomorrow (Tuesday 2026-04-22)
+- [ ] Rent H100 on Lambda Labs (~$2 for 30 min)
+- [ ] Run `benchmark_gpu.py` under `nsys profile --stats=true python benchmark_gpu.py`
+- [ ] Capture Nsight `.nsys-rep` file and download it
+- [ ] Note the speedup number — this is the headline for Thursday's talk
+
+**Still need:** H100 setup script (a single paste-into-terminal script for the cloud instance). Needs: Rust install, maturin build with `--features gpu`, nsys profile run, output capture. This was not written last session.
+
+### Wednesday (2026-04-23)
+- [ ] Update `docs/talk-10min.md` Slide 6 with the H100 result
+- [ ] Open Nsight Systems GUI locally, load the `.nsys-rep` file, screenshot the timeline showing both pipelines
+
+### Thursday (2026-04-24)
+- [ ] Talk to IT/security group (10 min)
+- [ ] If someone gives you a contact: do not show GPU source code without NDA signed first; first call is discovery
+
+---
+
+## Next Session Priorities (after Thursday)
+
+1. **Write H100 setup script** — single paste-into-terminal script (see above)
+2. **vLLM integration** — Phase 2.1, this is the demo that sells the concept
+3. **NM anonymous LLC** — Wyoming, single-member, before any contract is signed
+4. **C++ version scoping** — after Phase 1 solid, assess Whetstone transpilation of `llm.rs`
+
+---
+
+## Key Technical Context
+
+- Repo: `/home/bill/Documents/unchecked-io/`, on `gpu-zero` branch
+- Build: `maturin develop --release --features gpu` (maturin not installed on temp-ubuntu — install with `pip install maturin` first)
+- `cargo check --features gpu` confirms clean compile
+- The four Phase 1 bugs are fixed and verified to compile
+- `src/llm.rs` is the GPU core. `src/lib.rs` is the Python-facing wrapper.
+- `chrono-tz` pinned at `=0.10.4` — do not change
+- `gpu-zero` branch is private; `main` is open source Apache 2.0

ROADMAP.md

@@ -0,0 +1,160 @@
+# UncheckedIO — Development Roadmap
+
+**Goal:** Production-grade, multi-GPU zero-copy ingestion pipeline for LLM providers.
+**Current state:** Working proof-of-concept on CUDA (gpu-zero branch). PostgreSQL→Arrow open source on main.
+
+---
+
+## Phase 1 — Fix Correctness (gpu-zero branch)
+
+The GPU feature works in benchmark conditions but has four bugs that would surface immediately in production. Fix these before anything else.
+
+### 1.1 Dynamic buffer allocation
+**Problem:** `max_elements = 4096 * 512` is hardcoded at init time. Modern LLMs have 32K–128K context windows. A single request at 32K tokens + any real batch size hits this limit.
+**Fix:** Allocate `gpu_buffer` and `host_buffer` per-call based on actual `batch_size * max_seq_len`, or use a growable strategy that reallocates when the current batch exceeds the existing buffer size.
+
+### 1.2 Buffer isolation (silent data corruption bug)
+**Problem:** Every call to `encode_batch` overwrites the same `CudaSlice`. The DLPack capsule hands PyTorch a pointer to that same memory. A caller holding a tensor from call N gets its data silently overwritten by call N+1.
+**Fix:** Double-buffer (ping-pong between two allocations) or allocate a fresh device buffer per call and transfer ownership to the DLPack capsule's deleter so CUDA frees it when PyTorch releases the tensor.
+
+### 1.3 Thread safety
+**Problem:** `encode_batch` takes `&mut self`. The `TokenizerEngine` Python class cannot be called from multiple threads simultaneously. Every production serving engine (vLLM, TGI) is multithreaded.
+**Fix:** Wrap `PinnedBatcher` in an `Arc<Mutex<>>` or, better, create a `EnginePool` that holds N instances and checks one out per call. Pool size = CPU core count is a reasonable default.
+
+### 1.4 Variable sequence length + padding
+**Problem:** `seq_len = encodings[0].len()` assumes all sequences in the batch have identical length. The `tokenizers` crate doesn't apply padding by default, so real variable-length inputs produce an incorrect 2D tensor shape.
+**Fix:** Apply padding to `max(encodings[i].len())` within the batch, fill with the tokenizer's `pad_id`, and use the actual padded length as `seq_len`.
+
+---
+
+## Phase 2 — Production Features
+
+### 2.1 vLLM integration
+vLLM is the dominant open-source LLM serving engine. Make `unchecked_io` a drop-in tokenization backend for it.
+- Implement the `TokenizerGroup` protocol vLLM expects
+- The continuous batching scheduler sends variable-length requests asynchronously — this is where Phase 1 fixes must be solid
+- Target: a 3-line config change in vLLM to switch to the UncheckedIO backend
+- Deliverable: `examples/vllm_integration.py` showing the swap
+
+### 2.2 Streaming / chunked output
+**Current:** The entire batch is tokenized and DMA'd in one blocking call.
+**Target:** Yield Arrow RecordBatches or DLPack tensors incrementally as tokens are ready, so the model can begin prefill before the full batch is staged.
+
+### 2.3 Attention mask export
+LLM inference requires both `input_ids` and `attention_mask`. Currently only token IDs are exported. Add a second DLPack capsule for the mask, or pack both into a named dict capsule.
+
+### 2.4 Configurable max context
+Replace the hardcoded constants with constructor arguments:
+```python
+engine = TokenizerEngine(
+    tokenizer_path="tokenizer.json",
+    max_seq_len=32768,
+    max_batch_size=64,
+    pool_size=8
+)
+```
+
+---
+
+## Phase 3 — Multi-GPU Support
+
+### 3.1 ROCm / HIP (AMD) — highest priority
+AMD's MI300X has 192GB memory vs the H100's 80GB and providers are actively evaluating it to escape NVIDIA pricing. HIP is the path of least resistance — function names are nearly 1:1 with CUDA (`hipMemHostRegister`, `hipMemcpyHtoD`, `hipMemHostUnregister`).
+
+- Add `rocm` feature flag in `Cargo.toml`
+- Create `src/backends/rocm.rs` implementing the same `GpuBackend` trait
+- Use AMD's `hip-sys` or `rocm-sys` Rust crate
+- The DLPack export layer stays identical — device_type `10` for ROCm vs `2` for CUDA
+
+**Refactor needed first:** Extract a `GpuBackend` trait from the current CUDA-specific code:
+```rust
+trait GpuBackend {
+    fn allocate_pinned(capacity: usize) -> Result<PinnedBuffer>;
+    fn copy_to_device(src: &[i64], dst: &mut DeviceBuffer) -> Result<()>;
+    fn device_ptr(buf: &DeviceBuffer) -> u64;
+    fn dlpack_device_type() -> i32;
+}
+```
+
+### 3.2 OpenCL (broad coverage)
+Covers AMD, Intel, older NVIDIA, Apple (via Metal-OpenCL bridge). Lower peak performance than native CUDA/HIP but maximum hardware coverage. Use the `opencl3` Rust crate.
+- Add `opencl` feature flag
+- Useful for providers running mixed-GPU clusters
+
+### 3.3 Intel Gaudi / Level Zero
+Intel's Gaudi 2/3 accelerators are gaining traction for training and are available on AWS and Azure. Intel Level Zero is the low-level API equivalent to CUDA driver API. Lower priority than ROCm but worth tracking.
+
+---
+
+## Phase 4 — Benchmark Suite
+
+The current `benchmark_gpu.py` is good for what it tests but synthetic. A buyer will run their own benchmarks — make it easy and cover real scenarios.
+
+### 4.1 Scenarios to cover
+- Short sequences: 128, 256 tokens (BERT-style)
+- Medium sequences: 2048, 4096 tokens (typical chat)
+- Long sequences: 16K, 32K tokens (document/RAG workloads)
+- Variable length batches (realistic distribution, not identical strings)
+- Concurrent throughput: simulate N async callers
+
+### 4.2 Tokenizers to cover
+- BERT (`bert-base-uncased`) — already done
+- LLaMA 3 tokenizer (BPE, more representative of modern LLMs)
+- GPT-2 tokenizer (common baseline)
+
+### 4.3 Hardware targets
+- Local RTX 3060 (done, 4.7x)
+- Cloud H100 SXM (~$2 on Lambda Labs, 30 min run) — **do this first, it changes the pitch**
+- AMD MI300X (after Phase 3.1)
+
+### 4.4 What to measure
+- Tokens/second throughput
+- Latency per batch (p50, p95, p99)
+- GPU utilization % (the thing providers care about)
+- Memory bandwidth saturation
+
+---
+
+## Phase 5 — Business Infrastructure
+
+### 5.1 Wyoming LLC formation
+- Single-member LLC, Wyoming (strongest privacy, no public member disclosure)
+- LLC owns all IP
+- Registered agent handles public address
+- Done before any contract is signed
+
+### 5.2 License structure
+- Open core: PostgreSQL→Arrow stays Apache 2.0
+- GPU features: commercial license
+- Pricing model: annual per-cluster license, priced against GPU utilization savings
+- License enforcement: honor-system enterprise contracts initially, not technical DRM
+
+### 5.3 Sales collateral
+- 1-page technical brief (problem / mechanism / numbers / what's included)
+- Self-contained benchmark the prospect can run on their own hardware
+- Reference architecture diagram showing where this sits in a vLLM serving stack
+
+---
+
+## Branch Strategy
+
+| Branch | Purpose |
+|--------|---------|
+| `main` | Open source PostgreSQL→Arrow, Apache 2.0, public |
+| `gpu-zero` | GPU feature development, private |
+| `rocm` | AMD port (Phase 3.1) |
+| `release/x.y` | Tagged releases for licensing |
+
+The `gpu` feature is compiled in only with `--features gpu`. The open source build never includes GPU code.
+
+---
+
+## Immediate Next Steps (in order)
+
+1. Fix 1.1 (dynamic buffer) — unblocks everything else
+2. Fix 1.2 (buffer isolation) — correctness before benchmarking
+3. Fix 1.3 (thread safety) — required for any real serving test
+4. Fix 1.4 (padding) — correctness
+5. Spend $2, run benchmark on cloud H100 — this one number changes the pitch
+6. Phase 2.1 vLLM integration — this is the demo that sells the concept
+7. Phase 3.1 ROCm — this is the differentiator

benchmark_gpu.py

@@ -0,0 +1,114 @@
+import time
+import torch
+import os
+import json
+import unchecked_io
+from transformers import AutoTokenizer
+
+# --- CONFIGURATION ---
+MODEL_ID = "bert-base-uncased"
+TOKENIZER_PATH = "tokenizer.json"
+BATCH_SIZE = 4096      # Massive batch to saturate PCIe
+NUM_BATCHES = 50       # Run enough to stabilize GPU clock
+SEQ_LEN = 128          # Typical sentence length
+WARMUP = 5
+
+# Synthetic Data
+SAMPLE_TEXT = "The quick brown fox jumps over the lazy dog. " * 5  # ~50 tokens
+DATA_BATCH = [SAMPLE_TEXT for _ in range(BATCH_SIZE)]
+
+print(f"--- GPU PIPELINE BENCHMARK (RTX 3060) ---")
+print(f"Batch Size: {BATCH_SIZE} | Batches: {NUM_BATCHES}")
+print(f"Total Strings processed: {BATCH_SIZE * NUM_BATCHES:,}")
+
+# --- 1. SETUP RESOURCES ---
+
+# A. Prepare Tokenizer File for Rust
+if not os.path.exists(TOKENIZER_PATH):
+    print(f"Downloading {MODEL_ID} tokenizer...")
+    # forcing use_fast=True ensures we get the JSON file Rust needs
+    hf_tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, use_fast=True)
+    hf_tokenizer.save_pretrained(".")
+    print("Saved tokenizer.json")
+else:
+    print("Found existing tokenizer.json")
+    hf_tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, use_fast=True)
+
+# B. Initialize Engines
+print("Initializing Engines...")
+
+# UncheckedIO (Rust + CUDA)
+try:
+    rust_engine = unchecked_io.TokenizerEngine(TOKENIZER_PATH)
+    print("✅ UncheckedIO Engine Loaded")
+except AttributeError:
+    print("❌ Error: UncheckedIO 'TokenizerEngine' not found.")
+    print("Did you compile with: maturin develop --features gpu")
+    exit(1)
+
+# --- 2. BENCHMARK LOOPS ---
+
+def benchmark_standard():
+    torch.cuda.synchronize()
+    start = time.time()
+
+    for _ in range(NUM_BATCHES):
+        # Step 1: Tokenize (CPU Python/Rust mix)
+        # return_tensors='pt' creates a CPU tensor
+        encodings = hf_tokenizer(
+            DATA_BATCH,
+            padding=True,
+            truncation=True,
+            max_length=SEQ_LEN,
+            return_tensors="pt"
+        )
+
+        # Step 2: Move to GPU (The Bottleneck)
+        input_ids = encodings["input_ids"].to("cuda", non_blocking=True)
+
+        # Force sync to measure actual completion
+        torch.cuda.synchronize()
+
+    return time.time() - start
+
+def benchmark_unchecked():
+    torch.cuda.synchronize()
+    start = time.time()
+
+    for _ in range(NUM_BATCHES):
+        # Step 1 & 2: Tokenize + DMA to GPU (All in Rust)
+        dlpack_capsule = rust_engine.encode_batch(DATA_BATCH)
+
+        # Step 3: Zero-Copy Wrap (Python)
+        # This is virtually instant
+        input_ids = torch.from_dlpack(dlpack_capsule)
+
+        # Force sync
+        torch.cuda.synchronize()
+
+    return time.time() - start
+
+# --- 3. RUN RACES ---
+
+print("\nWARMING UP GPU...")
+# Run a few dummy passes to wake up the 3060
+_ = hf_tokenizer(DATA_BATCH[:10], return_tensors="pt")["input_ids"].to("cuda")
+_ = torch.from_dlpack(rust_engine.encode_batch(DATA_BATCH[:10]))
+
+print("\n🚀 RUNNING STANDARD PIPELINE (HF -> CPU Tensor -> CUDA)...")
+time_std = benchmark_standard()
+fps_std = (BATCH_SIZE * NUM_BATCHES) / time_std
+print(f"Time: {time_std:.4f}s | Throughput: {fps_std:,.0f} samples/sec")
+
+print("\n🚀 RUNNING UNCHECKED PIPELINE (Rust -> Pinned -> CUDA)...")
+time_unchecked = benchmark_unchecked()
+fps_unchecked = (BATCH_SIZE * NUM_BATCHES) / time_unchecked
+print(f"Time: {time_unchecked:.4f}s | Throughput: {fps_unchecked:,.0f} samples/sec")
+
+# --- 4. RESULTS ---
+
+speedup = fps_unchecked / fps_std
+print(f"\n🏆 WINNER: {'UncheckedIO' if speedup > 1 else 'Standard'}")
+print(f"SPEEDUP FACTOR: {speedup:.2f}x")
+print(f"Standard Latency per Batch: {(time_std/NUM_BATCHES)*1000:.2f}ms")
+print(f"Unchecked Latency per Batch: {(time_unchecked/NUM_BATCHES)*1000:.2f}ms")

docs/commercialization-strategy.md

@@ -0,0 +1,206 @@
+# UncheckedIO — Commercialization Strategy
+**Session date:** 2026-04-20
+**Status:** Active planning
+
+---
+
+## Product Summary
+
+Two components with different commercial treatment:
+
+| Component | Language | License | Status |
+|---|---|---|---|
+| PostgreSQL → Arrow pipeline | Rust | Apache 2.0 (open source) | On PyPI, working |
+| GPU zero-copy ingestion pipeline | Rust + CUDA | Commercial | Proof-of-concept, needs Phase 1 fixes |
+
+The GPU pipeline is the commercial product. The Postgres piece is the credibility signal and open-source loss leader.
+
+---
+
+## The Problem Being Solved
+
+GPU starvation — LLM providers run 30-50% actual GPU utilization on inference. The bottleneck is the data ingestion pipeline: CPU tokenizes, creates a Python tensor in pageable memory, `.to("cuda")` performs two OS-mediated copies (pageable → kernel buffer → GPU).
+
+UncheckedIO replaces this with: Rust parallel tokenization → pinned memory (DMA-accessible) → single direct DMA → DLPack zero-copy handover to PyTorch.
+
+**Current benchmark:** 4.7x faster than standard HuggingFace pipeline on RTX 3060.
+**Pending:** H100 SXM benchmark (Wednesday, ~$2 on Lambda Labs). This number changes the pitch.
+
+---
+
+## Business Structure
+
+**Entity:** New Mexico single-member LLC
+- NM does not require member names in Articles of Organization
+- No annual report requirement — less administrative overhead than most states
+- LLC owns all IP and signs all contracts
+- Registered agent handles public address
+- Form this before any contract is signed
+- Note: if the IP is eventually sold, the buyer's anonymity is their own concern — the LLC structure only needs to protect the original author
+
+**Operating model:** Anonymous principal — LLC is the public face. The identity of the technical author is not disclosed to buyers or the public. Any investor or partner arrangement must include explicit confidentiality provisions.
+
+---
+
+## Go-to-Market Strategy
+
+### Phase 1: Licensing (current goal)
+License the GPU pipeline commercially. Annual per-cluster licenses. Open-source Postgres piece stays Apache 2.0 as a credibility and discovery channel.
+
+**Why licensing before IP sale:**
+A library generating $50K+/month in licenses is a business acquisition, not a proof-of-concept. The IP sale price increases significantly and the negotiating position improves. Licensing revenue demonstrates market validation.
+
+### Phase 2: IP Sale (after traction)
+Once licensing revenue establishes market proof, approach GPU cloud providers and LLM infrastructure companies for outright IP purchase. Source code + short-term handoff contract. Clean exit.
+
+---
+
+## Pricing
+
+The ROI math for a GPU provider: 100 H100s at $175K/year each = $17.5M in GPU costs. A 10% utilization improvement = $1.75M saved. A $50K license is a 35x ROI.
+
+**Do not anchor at $50K. That is the floor for small deployments.**
+
+| Deployment size | Annual license | One-time source sale |
+|---|---|---|
+| Small (50–200 GPUs) | $25K–$75K | $100K–$200K |
+| Medium (200–1,000 GPUs) | $75K–$200K | $300K–$600K |
+| Large (1,000+ GPUs) | $200K–$500K | $750K–$1.5M |
+
+**First client:** Accept below market rate in exchange for a private reference and permission to say "deployed at [company]" to the next prospect. That reference is worth more than the price difference.
+
+---
+
+## Target Buyers
+
+**Primary (write the check):**
+GPU cloud providers and LLM inference companies feel GPU starvation directly in their margins. They sell GPU utilization as a product. Any measurable improvement is direct revenue.
+
+- CoreWeave, Lambda Labs — GPU cloud, sell utilization by the hour
+- Together AI, Fireworks AI, Modal — LLM inference providers
+- Anyscale — Ray-based ML infrastructure
+
+**Secondary (may buy, slower process):**
+- Hugging Face — owns the `tokenizers` Rust library this code uses, runs TGI serving stack. More plausible buyer than PyTorch/Meta.
+- Databricks (MosaicML) — LLM training infrastructure
+
+**Do not pitch:**
+- PyTorch/Meta — open source project, they'd implement it themselves
+- Hyperscalers (AWS, GCP, Azure) — too slow, too much process
+
+**NVIDIA contact:**
+Wait until H100 benchmark is complete. Send the Nsight trace + one paragraph. They may be interested as an ecosystem/marketing play ("runs X% faster on H100"). Do not approach before having the measurement.
+
+---
+
+## B2B Sales Process
+
+### First contact (Zoom call)
+The first call is discovery, not pitch. Goal: understand their specific infrastructure before presenting a solution.
+
+**Ask first:**
+- What GPU hardware? (H100, A100, MI300X — how many?)
+- What serving stack? (vLLM, TGI, custom?)
+- Current GPU utilization rate, if known?
+- What caught their attention?
+
+**Then:** "Based on what you're describing, here's specifically where this helps you..."
+
+**On pricing:** Don't volunteer it first. If pressed: *"For a deployment your size I'd expect $50K–$150K annually. I want to understand your infrastructure better before giving you a specific number."*
+
+**End of first call:** Set up follow-up with their infrastructure lead. Do not sign anything on the first call.
+
+### NDA
+Get an NDA signed before showing the GPU source code on any call. The open-source Postgres piece is fine to show. The GPU pipeline (`src/llm.rs`, `gpu-zero` branch) is not.
+
+### Solo developer objection
+Enterprise buyers will ask: *"What happens if you're unavailable?"*
+
+**Answers:**
+1. Source code sale model answers it directly — they own the code
+2. Technical partner (friend's software business) provides continuity
+3. Source code escrow clause: code held by third party, released if unavailable for 90 days — this is standard and removes most of the objection
+
+---
+
+## Maintenance and Support
+
+**Reality for 1–2 enterprise clients:**
+
+| Phase | Time commitment |
+|---|---|
+| Active development (first 3–6 months) | 20–40 hrs/week |
+| Steady state | 5–10 hrs/week per client |
+| Major CUDA/PyTorch version update | 2–4 days, happens 2–3x/year |
+| Production incident | Drop everything, 1–3 days |
+
+**Scope the support in the contract:** Cover current PyTorch LTS + one prior release. Anything newer is a paid upgrade. This prevents unbounded compatibility obligations.
+
+One person can support 2–3 enterprise clients at steady state.
+
+---
+
+## Outsourcing
+
+A friend with a software business can handle work that is not the core differentiator:
+- Support ticket triage and first-response
+- Documentation and integration guides
+- Testing infrastructure (benchmark runs against new PyTorch/CUDA versions)
+- Non-GPU features (Postgres pipeline, Python API polish)
+
+**Do not outsource:** The GPU pipeline code, benchmarking methodology, CUDA/ROCm work — these are the differentiator.
+
+**Rate:** Pass 60–70% of contract value for defined scope. On a $100K contract where the friend does 30% of the work: $18K–$21K to them. You take the larger share because you carry the business development risk and hold the client relationship.
+
+**Critical:** Any work they do on the core library must be IP-assigned to the LLC in writing before the first engagement.
+
+---
+
+## Multi-Language Strategy (Rust + C++)
+
+**Why both:**
+- Rust is the development language — memory safety makes GPU pipeline bugs easier to catch
+- C++ is the enterprise deliverable — the entire ML infrastructure ecosystem (cuDNN, TensorRT, vLLM C++ backend) is C++
+- Having both is a differentiator from vibe-coded projects; it signals deep systems understanding
+
+**How:**
+- WhetstoneAI (separate project) can transpile language primitives automatically
+- Library dependencies require manual substitution (see below)
+- Rust is source of truth; C++ is a generated + manually doctored artifact per release
+
+**Library substitutions for C++ version:**
+
+| Rust dependency | C++ equivalent | Complexity |
+|---|---|---|
+| `cudarc` | Direct CUDA C++ calls (actually simpler) | Low |
+| `rayon` | OpenMP `#pragma omp parallel for` | Low |
+| `tokenizers` | SentencePiece (LLaMA) + tiktoken (GPT) | Medium — different API |
+| `pyo3` | pybind11 | Medium — parallel structure |
+| `tokio` | Not needed for GPU-only C++ version | N/A |
+
+`parser.rs` (Postgres pipeline) is not worth transpiling — stays Rust only.
+
+**Estimated work for C++ `llm.rs`:** 2–3 days after Whetstone handles primitives.
+
+---
+
+## NM State Funding Assessment
+
+**Worth pursuing:**
+- **NM Small Business Assistance Program (NMSBA)** — pairs small businesses with Sandia/LANL. The value is free access to HPC hardware (H100/A100 clusters), not the money. Solves the enterprise benchmark problem. Apply for this.
+
+**Not worth pursuing (for now):**
+- NMEDD money grants, angel tax credits — require registered business, matching funds, 3–6 month timelines. Time cost doesn't pencil out against a direct licensing deal.
+
+**Key point:** Most state grant programs want an ongoing business, not a one-time IP sale. If the goal shifts toward a licensing business, revisit.
+
+---
+
+## Immediate Priorities
+
+1. Fix Phase 1 correctness issues (see ROADMAP.md) — nothing else matters until these are done
+2. Wednesday: H100 benchmark on Lambda Labs (~$2), run under `nsys profile`
+3. Thursday/Friday: Incorporate H100 numbers into talk slides
+4. Talk: 10-minute presentation to IT/Cybersecurity group (see docs/talk-10min.md)
+5. Form Wyoming LLC before any serious sales conversation
+6. After first client: evaluate C++ version via Whetstone transpilation

docs/talk-10min.md

@@ -0,0 +1,110 @@
+# UncheckedIO — 10-Minute Technical Talk
+**Audience:** IT/Cybersecurity group with programmers and ML engineers
+**Goal:** Plant the seed, demonstrate the mechanism, invite connections to LLM infrastructure teams
+
+---
+
+## Slide 1 — Open with the hidden thing (0:00–0:45)
+
+Lead with an Nsight Systems timeline screenshot. Don't explain it yet.
+
+> "This is what happens inside your GPU when a language model processes your request.
+> The orange is waiting. The GPU costs $30,000 and it's waiting for data."
+
+Let the image do the work. This audience will immediately want to know what they're looking at.
+
+---
+
+## Slide 2 — The problem (0:45–2:00)
+
+**GPU starvation.** Define it in dollar terms.
+
+- Real LLM providers run at 30–50% actual GPU utilization on inference
+- An 8-GPU H100 cluster costs ~$20/hr to rent
+- At 40% utilization, $12/hr is wasted — the GPUs are idle waiting for data
+- LLM providers run hundreds of clusters
+- The waste is structural and nobody has fixed the root cause
+
+---
+
+## Slides 3–4 — The mechanism (2:00–4:00)
+
+Two diagrams, side by side.
+
+**Standard pipeline:**
+```
+CPU tokenizes → Python tensor (pageable memory) → .to("cuda") → kernel buffer → GPU
+                                                   ^--- two OS-mediated copies
+```
+
+**UncheckedIO pipeline:**
+```
+Rust tokenizes in parallel → pinned memory (DMA-accessible) → direct DMA → DLPack → GPU
+                                                               ^--- one copy, no kernel involvement
+```
+
+The second diagram has fewer boxes and fewer arrows. That is the entire pitch.
+
+Key terms to explain briefly:
+- **Pinned memory** — memory registered with the OS so the GPU's DMA engine can read it directly
+- **DMA** — the GPU pulls the data itself, no CPU involvement after the transfer starts
+- **DLPack** — a standard zero-copy tensor exchange protocol; PyTorch accepts it natively with `torch.from_dlpack()`
+
+---
+
+## Slide 5 — The Nsight demo (4:00–7:00)
+
+Show the recorded Nsight Systems timeline from the H100 benchmark run.
+
+Both pipelines visible side by side:
+- Point out the idle gaps (orange) in the standard pipeline
+- Point out the absence of idle gaps in the UncheckedIO pipeline
+- Show the DMA transfer lane — this is the "hidden thing" made visible
+
+This is the centerpiece of the talk. Expect questions here. That's good.
+
+---
+
+## Slide 6 — The number (7:00–8:00)
+
+One slide. No prose.
+
+| Hardware | Speedup |
+|----------|---------|
+| RTX 3060 (local) | 4.7x |
+| H100 SXM (cloud) | [Wednesday result] |
+
+---
+
+## Slide 7 — Market fit (8:00–9:00)
+
+- This is infrastructure every LLM provider needs
+- Built as an open-core library: PostgreSQL→Arrow is Apache 2.0 and already on PyPI
+- GPU ingestion pipeline is the commercial layer
+- AMD ROCm support (MI300X) is next — providers are actively evaluating MI300X to escape H100 pricing
+
+Don't pitch deal specifics. This slide plants the seed.
+
+---
+
+## Slide 8 — The close (9:00–10:00)
+
+> "I'm looking for connections to ML infrastructure teams and LLM providers.
+> If you know someone building at that layer, I'd appreciate an introduction."
+
+Direct. Not desperate. Leave room for questions.
+
+---
+
+## Notes
+
+**What NOT to do:**
+- Don't go deep on the Rust implementation — the visual is the hook, not the code
+- Don't mention the PostgreSQL→Arrow piece — it's a distraction in this context
+- Don't volunteer that it's unfinished — if asked, "it's in active development and the core mechanism is proven"
+
+**For the well-connected engineer (separate talk, two weeks out):**
+Don't reuse this version. Prep a deeper, tailored pitch after seeing what questions come up here.
+
+**For the NVIDIA contact:**
+Wait until after the H100 benchmark. Send the Nsight trace + one paragraph. Leading with a measurement is different from leading with a claim.

pyproject.toml

@@ -4,8 +4,33 @@ build-backend = "maturin"
 
 [project]
 name = "unchecked-io"
-version = "0.1.0"
+# Dynamic versioning allows maturin to read version from Cargo.toml
+dynamic = ["version"]
+description = "The world's fastest, most dangerous PostgreSQL-to-Arrow loader."
+readme = "README.md"
+requires-python = ">=3.9"
+license = {text = "BSL-1.1"}
+authors = [{name = "Billthemaker"}]
+keywords = ["rust", "postgresql", "arrow", "etl", "fast", "copy", "database"]
 classifiers = [
     "Programming Language :: Rust",
+    "Programming Language :: Python :: Implementation :: CPython",
+    "Programming Language :: Python :: Implementation :: PyPy",
     "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Database",
+    "Topic :: Scientific/Engineering :: Information Analysis",
+    "Operating System :: POSIX :: Linux",
+    "Operating System :: Microsoft :: Windows",
+    "Operating System :: MacOS :: MacOS X",
 ]
+dependencies = [
+    "arro3-core>=0.1.0",
+]
+
+[tool.maturin]
+features = ["pyo3/extension-module"]

server.py

@@ -0,0 +1,109 @@
+import os
+import time
+import torch
+import uvicorn
+from contextlib import asynccontextmanager
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+# Import your custom high-speed engine
+import unchecked_io
+
+# --- CONFIGURATION ---
+# Switching to GPT-2 for stability. It is small, fast, and has a standard tokenizer
+# that won't crash older/newer Rust crate versions.
+MODEL_ID = "gpt2"
+TOKENIZER_FILE = "tokenizer.json"
+
+# Global state container
+model_state = {
+    "model": None,
+    "rust_engine": None,
+    "py_tokenizer": None
+}
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    print(f"\n--- 🚀 UNCHECKED SERVER STARTUP ---")
+
+    # 1. Fetch Tokenizer (Python side)
+    print(f"1. Fetching Tokenizer Config from {MODEL_ID}...")
+    tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, use_fast=True)
+    tokenizer.save_pretrained(".")
+    model_state["py_tokenizer"] = tokenizer
+
+    # 2. Initialize UncheckedIO (Rust + CUDA)
+    print(f"2. Initializing UncheckedIO (Fuel Injector)...")
+    try:
+        model_state["rust_engine"] = unchecked_io.TokenizerEngine(TOKENIZER_FILE)
+        print("   ✅ Rust Engine Ready (Zero-Copy Pipeline Active)")
+    except Exception as e:
+        print(f"   ❌ Failed to load Rust Engine: {e}")
+        raise e
+
+    # 3. Load Model (PyTorch)
+    print(f"3. Loading Model Weights...")
+    try:
+        model_state["model"] = AutoModelForCausalLM.from_pretrained(
+            MODEL_ID,
+            device_map="cuda",
+            torch_dtype=torch.float16
+        )
+        print(f"   ✅ Model Loaded on {model_state['model'].device}")
+    except Exception as e:
+        print(f"   ❌ Failed to load model: {e}")
+        raise e
+
+    print("--- SERVER READY ---\n")
+    yield
+    print("\n--- SERVER SHUTDOWN ---")
+    # Resources are cleaned up here
+
+app = FastAPI(title="UncheckedIO High-Speed Server", lifespan=lifespan)
+
+class GenerateRequest(BaseModel):
+    prompt: str
+    max_tokens: int = 50
+    temperature: float = 0.7
+
+@app.post("/generate")
+async def generate_text(req: GenerateRequest):
+    model = model_state["model"]
+    engine = model_state["rust_engine"]
+    tokenizer = model_state["py_tokenizer"]
+
+    if not model or not engine:
+        raise HTTPException(status_code=503, detail="Server not ready")
+
+    try:
+        # --- PHASE 1: UNCHECKED INGESTION (Rust) ---
+        # 4.8x Faster than standard .to("cuda")
+        input_capsule = engine.encode_batch([req.prompt])
+        input_ids = torch.from_dlpack(input_capsule)
+
+        # --- PHASE 2: INFERENCE (PyTorch) ---
+        with torch.no_grad():
+            output_ids = model.generate(
+                input_ids,
+                max_new_tokens=req.max_tokens,
+                temperature=req.temperature,
+                do_sample=True,
+                pad_token_id=tokenizer.eos_token_id
+            )
+
+        # --- PHASE 3: DECODING ---
+        generated_text = tokenizer.decode(output_ids[0], skip_special_tokens=True)
+
+        return {
+            "response": generated_text,
+            "backend": "UncheckedIO + PyTorch",
+            "status": "success"
+        }
+
+    except Exception as e:
+        print(f"Error during generation: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8000)

src/lib.rs

@@ -1,18 +1,18 @@
-// --- Declare our new modules ---
 mod config;
 mod parser;
 
-// --- External Crates ---
+#[cfg(feature = "gpu")]
+mod llm;
+
 use pyo3::prelude::*;
+#[cfg(feature = "gpu")]
+use std::sync::Arc;
 use pyo3::exceptions::PyValueError;
 use tokio;
 use pyo3::types::{PyModule, PyAny};
 use pyo3::Bound;
-
-// FIX: Use the export path you confirmed works in your IDE
 use pyo3_arrow::export::Arro3RecordBatch;
 
-// Required for global allocator (mimalloc)
 #[cfg(not(target_env = "msvc"))]
 use mimalloc;
 
@@ -20,26 +20,16 @@ use mimalloc;
 #[global_allocator]
 static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
-
-// --- Internal Crates ---
 use crate::config::{load_and_validate_config, ConnectorConfig};
-// NOTE: run_profiler_logic added, requires definition in parser.rs
 use crate::parser::{run_db_logic, run_profiler_logic};
 
-// NEW: Import the tracing libraries only if feature is enabled
 #[cfg(feature = "profiling")]
 use tracing_subscriber::layer::SubscriberExt;
 #[cfg(feature = "profiling")]
 use tracing_subscriber::util::SubscriberInitExt;
 
-
-// --- THE PYTHON-CALLABLE ENTRY POINT (Load Data) ---
-
 #[pyfunction]
-#[pyo3(signature = (config_path, blast_radius=0))] // Default to 0 for auto-tuning
+#[pyo3(signature = (config_path, blast_radius=0))]
-#[allow(unsafe_code)]
-#[allow(unsafe_op_in_unsafe_fn)]
-#[allow(rust_2024_compatibility)]
 fn load_data_from_config<'py>(
     py: Python<'py>,
     config_path: String,
@@ -51,12 +41,6 @@ fn load_data_from_config<'py>(
         Err(e) => return Err(PyValueError::new_err(format!("Configuration Error: {:?}", e))),
     };
 
-    println!("--- UncheckedIO: Schema Accepted ---");
-    println!("Database: {}", config.connection_string);
-    println!("Columns (in order): {:?}", config.schema.iter().map(|c| &c.column_name).collect::<Vec<_>>());
-
-    // FIX: Handle GIL release properly to avoid deprecation warnings if possible,
-    // but primarily ensure the logic works with the new Arro3RecordBatch wrapper.
     let record_batch = py.allow_threads(|| {
         tokio::runtime::Builder::new_multi_thread()
             .enable_all()
@@ -67,25 +51,19 @@ fn load_data_from_config<'py>(
             })
     }).map_err(|e| PyValueError::new_err(format!("Database/Runtime Error: {:?}", e)))?;
 
-    // FIX: Use Arro3RecordBatch::from() instead of new()
-    // This uses the standard From trait conversion.
     let py_record_batch = Arro3RecordBatch::from(record_batch);
     py_record_batch.into_pyobject(py)
 }
 
-// --- NEW PYTHON-CALLABLE FUNCTION FOR PANDAS CONVERSION (FIXED SIGNATURE) ---
 #[pyfunction]
-#[pyo3(signature = (arrow_table))] // FIX: Removed 'py' from the signature macro
+#[pyo3(signature = (arrow_table))]
-fn to_pandas_dataframe<'py>(py: Python<'py>, arrow_table: Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
+fn to_pandas_dataframe<'py>(_py: Python<'py>, arrow_table: Bound<'py, PyAny>) -> PyResult<Bound<'py, PyAny>> {
-    // This calls the 'to_pandas' method on the PyArrow object.
     arrow_table.call_method0("to_pandas")
 }
 
-// --- NEW PYTHON-CALLABLE ENTRY POINT FOR SCHEMA PROFILING ---
 #[pyfunction]
 #[pyo3(signature = (config_path))]
 fn profile_data(config_path: String) -> PyResult<String> {
-    // Note: The profiler uses a small, current_thread tokio runtime since it's sequential I/O.
     let output = std::thread::spawn(move || {
         tokio::runtime::Builder::new_current_thread()
             .enable_all()
@@ -99,25 +77,48 @@ fn profile_data(config_path: String) -> PyResult<String> {
     Ok(output)
 }
 
+#[cfg(feature = "gpu")]
+#[pyclass(name = "TokenizerEngine")]
+struct TokenizerEngine {
+    // 1.3: Arc allows cheap cloning; encode_batch takes &self so concurrent calls are safe
+    inner: Arc<llm::PinnedBatcher>,
+}
+
+#[cfg(feature = "gpu")]
+#[pymethods]
+impl TokenizerEngine {
+    #[new]
+    fn new(model_path: String) -> PyResult<Self> {
+        Ok(TokenizerEngine {
+            inner: Arc::new(llm::PinnedBatcher::new(&model_path)?)
+        })
+    }
+
+    fn encode_batch(&self, py: Python, texts: Vec<String>) -> PyResult<Py<PyAny>> {
+        self.inner.batch_encode_to_gpu(py, texts)
+    }
+}
 
-// --- PYTHON MODULE EXPORT ---
 #[pymodule]
 fn unchecked_io(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
 
-    // NEW: Only initialize Tracy if the feature is enabled
     #[cfg(feature = "profiling")]
     {
-        // Use 'default()' instead of 'new()' to avoid argument mismatch errors
-        // The .try_init() prevents crashing on module reloads (e.g. Jupyter)
         let _ = tracing_subscriber::registry()
             .with(tracing_tracy::TracyLayer::default())
             .try_init();
-
         println!("UncheckedIO: Profiling Mode ENABLED 🚀");
     }
 
     m.add_function(wrap_pyfunction!(load_data_from_config, m)?)?;
     m.add_function(wrap_pyfunction!(to_pandas_dataframe, m)?)?;
     m.add_function(wrap_pyfunction!(profile_data, m)?)?;
+
+    #[cfg(feature = "gpu")]
+    {
+        m.add_class::<TokenizerEngine>()?;
+        println!("UncheckedIO: GPU Acceleration ENABLED ⚡");
+    }
+
     Ok(())
 }

src/llm.rs

@@ -0,0 +1,197 @@
+// src/llm.rs
+#![cfg(feature = "gpu")]
+
+use std::sync::Arc;
+use tokenizers::Tokenizer;
+use cudarc::driver::{CudaDevice, DeviceSlice, CudaSlice, DevicePtr};
+use pyo3::prelude::*;
+use pyo3::exceptions::PyValueError;
+use std::ffi::CStr;
+use rayon::prelude::*;
+
+use cuda_driver_sys::{
+    cuMemHostRegister_v2,
+    cuMemHostUnregister,
+    cuMemcpyHtoD_v2,
+    cudaError_enum
+};
+
+// --- DLPack v0.4 ABI Definitions ---
+#[repr(C)]
+struct DLDataType { code: u8, bits: u8, lanes: u16 }
+#[repr(C)]
+struct DLDevice { device_type: i32, device_id: i32 }
+#[repr(C)]
+struct DLTensor {
+    data: *mut std::ffi::c_void,
+    device: DLDevice,
+    ndim: i32,
+    dtype: DLDataType,
+    shape: *mut i64,
+    strides: *mut i64,
+    byte_offset: u64,
+}
+#[repr(C)]
+struct DLManagedTensor {
+    dl_tensor: DLTensor,
+    manager_ctx: *mut std::ffi::c_void,
+    deleter: Option<unsafe extern "C" fn(*mut DLManagedTensor)>,
+}
+
+const CAPSULE_NAME: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"dltensor\0") };
+
+struct PinnedHostBuffer {
+    data: Vec<i64>,
+}
+
+impl PinnedHostBuffer {
+    fn new(capacity: usize) -> PyResult<Self> {
+        let data = vec![0i64; capacity];
+        let ptr = data.as_ptr() as *mut std::ffi::c_void;
+        let bytes = capacity * std::mem::size_of::<i64>();
+        unsafe {
+            let result = cuMemHostRegister_v2(ptr, bytes, 0);
+            if result != cudaError_enum::CUDA_SUCCESS {
+                return Err(PyValueError::new_err(format!("Failed to pin memory: {:?}", result)));
+            }
+        }
+        Ok(Self { data })
+    }
+}
+
+impl Drop for PinnedHostBuffer {
+    fn drop(&mut self) {
+        unsafe {
+            let ptr = self.data.as_ptr() as *mut std::ffi::c_void;
+            cuMemHostUnregister(ptr);
+        }
+    }
+}
+
+// Owned by DLManagedTensor.manager_ctx — freed when PyTorch releases the tensor.
+struct DLPackContext {
+    _gpu_buffer: CudaSlice<i64>,
+    shape: Box<[i64]>,
+}
+
+pub struct PinnedBatcher {
+    tokenizer: Tokenizer,
+    device: Arc<CudaDevice>,
+}
+
+impl PinnedBatcher {
+    pub fn new(model_path: &str) -> PyResult<Self> {
+        let tokenizer = Tokenizer::from_file(model_path)
+            .map_err(|e| PyValueError::new_err(format!("Failed to load tokenizer: {}", e)))?;
+        let device = CudaDevice::new(0)
+            .map_err(|e| PyValueError::new_err(format!("No CUDA GPU found: {:?}", e)))?;
+        Ok(PinnedBatcher { tokenizer, device })
+    }
+
+    pub fn batch_encode_to_gpu(&self, py: Python, texts: Vec<String>) -> PyResult<Py<PyAny>> {
+        let encodings = self.tokenizer.encode_batch(texts, true)
+            .map_err(|e| PyValueError::new_err(format!("Tokenization failed: {}", e)))?;
+
+        if encodings.is_empty() {
+            return Err(PyValueError::new_err("Empty batch provided"));
+        }
+
+        let batch_size = encodings.len();
+
+        // 1.4: use max length across all encodings, not just encodings[0]
+        let seq_len = encodings.iter().map(|e| e.len()).max().unwrap_or(0);
+        if seq_len == 0 {
+            return Err(PyValueError::new_err("All sequences are empty"));
+        }
+        let total_elements = batch_size * seq_len;
+
+        // 1.4: pad shorter sequences with the tokenizer's pad token (fallback: 0)
+        let pad_id = self.tokenizer.get_padding()
+            .map(|p| p.pad_id as i64)
+            .unwrap_or(0);
+
+        // 1.1: allocate pinned host buffer sized to the actual batch — no hardcoded max
+        let mut host_buffer = PinnedHostBuffer::new(total_elements)?;
+        host_buffer.data.fill(pad_id);
+
+        // Parallel write into pinned memory; shorter sequences leave pad_id in remaining slots
+        host_buffer.data[..total_elements].par_chunks_mut(seq_len)
+            .zip(encodings.par_iter())
+            .for_each(|(dest, enc): (&mut [i64], &tokenizers::Encoding)| {
+                let ids = enc.get_ids();
+                let len = ids.len().min(dest.len());
+                for i in 0..len {
+                    dest[i] = ids[i] as i64;
+                }
+            });
+
+        // 1.1: allocate device buffer sized to this batch — no hardcoded max
+        let gpu_buffer = self.device.alloc_zeros::<i64>(total_elements)
+            .map_err(|e| PyValueError::new_err(format!("GPU alloc failed: {:?}", e)))?;
+
+        // Direct DMA copy (pinned host -> device)
+        unsafe {
+            let src_ptr = host_buffer.data.as_ptr() as *const std::ffi::c_void;
+            let dst_ptr = *gpu_buffer.device_ptr();
+            let bytes = total_elements * std::mem::size_of::<i64>();
+            let result = cuMemcpyHtoD_v2(dst_ptr, src_ptr, bytes);
+            if result != cudaError_enum::CUDA_SUCCESS {
+                return Err(PyValueError::new_err(format!("DMA copy failed: {:?}", result)));
+            }
+        }
+        // host_buffer drops here — unpins host memory
+
+        // 1.2: transfer gpu_buffer ownership into DLPackContext so each returned tensor owns its
+        // own device allocation. The deleter frees it when PyTorch releases the tensor.
+        let ctx = Box::new(DLPackContext {
+            _gpu_buffer: gpu_buffer,
+            shape: vec![batch_size as i64, seq_len as i64].into_boxed_slice(),
+        });
+
+        // Capture pointers before leaking the box
+        let shape_ptr = ctx.shape.as_ptr() as *mut i64;
+        let data_ptr = unsafe { *ctx._gpu_buffer.device_ptr() } as *mut std::ffi::c_void;
+        let manager_ctx = Box::into_raw(ctx) as *mut std::ffi::c_void;
+
+        let dl_tensor = DLTensor {
+            data: data_ptr,
+            device: DLDevice { device_type: 2, device_id: 0 },
+            ndim: 2,
+            dtype: DLDataType { code: 0, bits: 64, lanes: 1 },
+            shape: shape_ptr,
+            strides: std::ptr::null_mut(),
+            byte_offset: 0,
+        };
+
+        let managed_tensor = Box::new(DLManagedTensor {
+            dl_tensor,
+            manager_ctx,
+            deleter: Some(dlpack_deleter),
+        });
+
+        let managed_ptr = Box::into_raw(managed_tensor);
+
+        unsafe {
+            let capsule_ptr = pyo3::ffi::PyCapsule_New(
+                managed_ptr as *mut _,
+                CAPSULE_NAME.as_ptr(),
+                None,
+            );
+            if capsule_ptr.is_null() {
+                return Err(PyValueError::new_err("Failed to create PyCapsule"));
+            }
+            Ok(Bound::from_owned_ptr(py, capsule_ptr).into_any().unbind())
+        }
+    }
+}
+
+unsafe extern "C" fn dlpack_deleter(managed_ptr: *mut DLManagedTensor) {
+    if managed_ptr.is_null() { return; }
+    unsafe {
+        let managed = Box::from_raw(managed_ptr);
+        if !managed.manager_ctx.is_null() {
+            // Drops DLPackContext: frees CudaSlice (device memory) and shape array.
+            let _ = Box::from_raw(managed.manager_ctx as *mut DLPackContext);
+        }
+    }
+}