made zero-copy JSON - shared memory logic

Cargo.toml

@@ -1,67 +1,55 @@
 [package]
 name = "unchecked-io"
 version = "0.1.7"
-authors = ["Billthemaker"] # Replace with your name or alias
-license = "Apache-2.0" # Good practice for open-source
+authors = ["Billthemaker"]
+license = "Apache-2.0"
 edition = "2024"
 
-[lib] 
+[lib]
 name = "unchecked_io"
-crate-type = ["cdylib", "rlib"] 
+crate-type = ["cdylib", "rlib"]
+
+[features]
+default = []
+gpu = [
+    "tokenizers",
+    "cudarc",
+    "pyo3-dlpack",
+    "cudarc/cuda-version-from-build-system",
+    "dep:cuda-driver-sys"
+]
+profiling = ["dep:tracing", "dep:tracing-subscriber", "dep:tracing-tracy"]
 
 [dependencies]
-# 1. Python Bindings for FFI
 pyo3 = { version = "=0.27.0", features = ["extension-module", "chrono-tz", "chrono"] }
-
-
-# 2. Configuration Parsing (YAML)
 serde = { version = "1.0", features = ["derive"] }
 serde_yaml = "0.9"
-
-# 3. Apache Arrow and Data Handling
-# FIX: Add the "compute" feature to get concat_batches
-arrow = "57.0.0"
-
-# 4. Asynchronous Runtime (Essential for I/O and Postgres)
+arrow = { version = "57.0.0", features = ["prettyprint"] }
 tokio = { version = "1.37", features = ["full"] }
-
-
-# 5. Database Connection (Postgres)
 tokio-postgres = "0.7"
 deadpool-postgres = "0.14"
-
-# 6. Error Handling Crate
 anyhow = "1.0"
-
-# 7. Futures Utilities
 futures-util = "0.3"
-
-# 8. Byte Buffer Management
 bytes = "1.6"
-
-# 9. Binary Data Reading (NEW)
 byteorder = "1.5"
-
-# 10. Timestamp Handling (NEW)
 chrono = "0.4"
 chrono-tz = "=0.10.4"
-
-# 11. UUID Handling (NEW)
 uuid = { version = "1.8", features = ["serde", "v4"] }
-
-# 12. Arrow <-> Python Bridge (NEW)
 pyo3-arrow = "0.15.0"
-
-# 13. System CPU Count (NEW)
 num_cpus = "1.16"
+async-channel = "2.5.0"
+rayon = "1.10"
+
+# Optional dependencies enabled by features
+# UPGRADED: 0.20 supports modern Llama tokenizer exports
+tokenizers = { version = "0.20", optional = true }
+cudarc = { version = "0.11", features = ["driver"], optional = true }
+pyo3-dlpack = { version = "0.1.0", optional = true }
+cuda-driver-sys = { version = "0.3.0", optional = true }
 
 tracing-subscriber = { version = "0.3", features = ["registry", "env-filter"], optional = true}
 tracing-tracy = { version = "=0.11.2", optional = true }
 tracing = {version ="0.1.41", optional = true }
 
-async-channel = "2.5.0"
-
 [target.'cfg(not(target_env = "msvc"))'.dependencies]
-mimalloc = { version = "0.1.39" }
-
-async-channel = "2.3"
+mimalloc = { version = "0.1.39" }