Cargo.toml

[package]
name = "unchecked-io"
version = "0.1.7"
authors = ["Billthemaker"]
license = "Apache-2.0"
edition = "2024"

[lib]
name = "unchecked_io"
crate-type = ["cdylib", "rlib"]

[features]
default = []
gpu = [
    "tokenizers",
    "cudarc",
    "pyo3-dlpack",
    "cudarc/cuda-12050",
    "dep:cuda-driver-sys"
]
profiling = ["dep:tracing", "dep:tracing-subscriber", "dep:tracing-tracy"]

[dependencies]
pyo3 = { version = "=0.27.0", features = ["extension-module", "chrono-tz", "chrono"] }
serde = { version = "1.0", features = ["derive"] }
serde_yaml = "0.9"
arrow = { version = "57.0.0", features = ["prettyprint"] }
tokio = { version = "1.37", features = ["full"] }
tokio-postgres = "0.7"
deadpool-postgres = "0.14"
anyhow = "1.0"
futures-util = "0.3"
bytes = "1.6"
byteorder = "1.5"
chrono = "0.4"
chrono-tz = "=0.10.4"
uuid = { version = "1.8", features = ["serde", "v4"] }
pyo3-arrow = "0.15.0"
num_cpus = "1.16"
async-channel = "2.5.0"
rayon = "1.10"

# Optional dependencies enabled by features
# UPGRADED: 0.20 supports modern Llama tokenizer exports
tokenizers = { version = "0.20", optional = true }
cudarc = { version = "0.11", features = ["driver"], optional = true }
pyo3-dlpack = { version = "0.1.0", optional = true }
cuda-driver-sys = { version = "0.3.0", optional = true }

tracing-subscriber = { version = "0.3", features = ["registry", "env-filter"], optional = true}
tracing-tracy = { version = "=0.11.2", optional = true }
tracing = {version ="0.1.41", optional = true }

[target.'cfg(not(target_env = "msvc"))'.dependencies]
mimalloc = { version = "0.1.39" }
Initial commit 2025-11-17 14:23:03 -07:00			`[package]`
			`name = "unchecked-io"`
version update to publish to Pypi 7 2025-11-23 13:46:17 -07:00			`version = "0.1.7"`
made zero-copy JSON - shared memory logic 2026-02-11 06:29:08 +00:00			`authors = ["Billthemaker"]`
			`license = "Apache-2.0"`
optimizing hardware limits in Rust 2025-11-18 12:40:08 -07:00			`edition = "2024"`
Initial commit 2025-11-17 14:23:03 -07:00
made zero-copy JSON - shared memory logic 2026-02-11 06:29:08 +00:00			`[lib]`
Initial commit 2025-11-17 14:23:03 -07:00			`name = "unchecked_io"`
made zero-copy JSON - shared memory logic 2026-02-11 06:29:08 +00:00			`crate-type = ["cdylib", "rlib"]`

			`[features]`
			`default = []`
			`gpu = [`
			`"tokenizers",`
			`"cudarc",`
			`"pyo3-dlpack",`
Fix all four Phase 1 correctness bugs (dynamic buffer, DLPack ownership, thread safety, padding) - 1.1: Remove hardcoded 4096*512 buffer from PinnedBatcher; allocate host+device per-call - 1.2: Wrap CudaSlice in DLPackContext owned by DLManagedTensor.manager_ctx; deleter frees device memory when PyTorch releases tensor — no more silent overwrite across calls - 1.3: batch_encode_to_gpu now takes &self (no mutable shared state); TokenizerEngine wraps Arc<PinnedBatcher> and encode_batch takes &self — safe for concurrent Python threads - 1.4: seq_len = max across all encodings; host buffer prefilled with pad_id before token write Also: switch cudarc gpu feature from cuda-version-from-build-system to cuda-12050 to fix build on machines with CUDA 13.x (cudarc 0.11.9 only knows up to 12.5) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> 2026-04-21 14:10:31 -06:00			`"cudarc/cuda-12050",`
made zero-copy JSON - shared memory logic 2026-02-11 06:29:08 +00:00			`"dep:cuda-driver-sys"`
			`]`
			`profiling = ["dep:tracing", "dep:tracing-subscriber", "dep:tracing-tracy"]`
Initial commit 2025-11-17 14:23:03 -07:00
			`[dependencies]`
version update to publish to Pypi 5 2025-11-23 13:27:22 -07:00			`pyo3 = { version = "=0.27.0", features = ["extension-module", "chrono-tz", "chrono"] }`
Initial commit 2025-11-17 14:23:03 -07:00			`serde = { version = "1.0", features = ["derive"] }`
			`serde_yaml = "0.9"`
made zero-copy JSON - shared memory logic 2026-02-11 06:29:08 +00:00			`arrow = { version = "57.0.0", features = ["prettyprint"] }`
Update Cargo.toml via mobile 2025-11-17 22:58:09 -07:00			`tokio = { version = "1.37", features = ["full"] }`
			`tokio-postgres = "0.7"`
optimizing hardware limits in Rust 2025-11-18 12:40:08 -07:00			`deadpool-postgres = "0.14"`
Update Cargo.toml via mobile 2025-11-17 22:58:09 -07:00			`anyhow = "1.0"`
			`futures-util = "0.3"`
			`bytes = "1.6"`
			`byteorder = "1.5"`
Initial commit 2025-11-17 14:23:03 -07:00			`chrono = "0.4"`
version update to publish to Pypi 7 2025-11-23 13:46:17 -07:00			`chrono-tz = "=0.10.4"`
improving efficiency and writing to file 2025-11-17 16:22:54 -07:00			`uuid = { version = "1.8", features = ["serde", "v4"] }`
debugged parallelization logic 2025-11-18 09:59:49 -07:00			`pyo3-arrow = "0.15.0"`
optimizing hardware limits in Rust 2025-11-18 12:40:08 -07:00			`num_cpus = "1.16"`
made zero-copy JSON - shared memory logic 2026-02-11 06:29:08 +00:00			`async-channel = "2.5.0"`
			`rayon = "1.10"`

			`# Optional dependencies enabled by features`
			`# UPGRADED: 0.20 supports modern Llama tokenizer exports`
			`tokenizers = { version = "0.20", optional = true }`
			`cudarc = { version = "0.11", features = ["driver"], optional = true }`
			`pyo3-dlpack = { version = "0.1.0", optional = true }`
			`cuda-driver-sys = { version = "0.3.0", optional = true }`
optimizing hardware limits in Rust 2025-11-18 12:40:08 -07:00
make tracy optional, add readme.md, continue fixing thrashing for sequential reads for workers 2025-11-22 22:11:08 -07:00			`tracing-subscriber = { version = "0.3", features = ["registry", "env-filter"], optional = true}`
			`tracing-tracy = { version = "=0.11.2", optional = true }`
			`tracing = {version ="0.1.41", optional = true }`

freeze state, no good 2025-11-18 20:51:03 -07:00			`[target.'cfg(not(target_env = "msvc"))'.dependencies]`
Fix all four Phase 1 correctness bugs (dynamic buffer, DLPack ownership, thread safety, padding) - 1.1: Remove hardcoded 4096*512 buffer from PinnedBatcher; allocate host+device per-call - 1.2: Wrap CudaSlice in DLPackContext owned by DLManagedTensor.manager_ctx; deleter frees device memory when PyTorch releases tensor — no more silent overwrite across calls - 1.3: batch_encode_to_gpu now takes &self (no mutable shared state); TokenizerEngine wraps Arc<PinnedBatcher> and encode_batch takes &self — safe for concurrent Python threads - 1.4: seq_len = max across all encodings; host buffer prefilled with pad_id before token write Also: switch cudarc gpu feature from cuda-version-from-build-system to cuda-12050 to fix build on machines with CUDA 13.x (cudarc 0.11.9 only knows up to 12.5) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> 2026-04-21 14:10:31 -06:00			`mimalloc = { version = "0.1.39" }`