optimizing hardware limits in Rust

Cargo.lock

@@ -446,6 +446,41 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "deadpool"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b"
+dependencies = [
+ "deadpool-runtime",
+ "lazy_static",
+ "num_cpus",
+ "tokio",
+]
+
+[[package]]
+name = "deadpool-postgres"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d697d376cbfa018c23eb4caab1fd1883dd9c906a8c034e8d9a3cb06a7e0bef9"
+dependencies = [
+ "async-trait",
+ "deadpool",
+ "getrandom 0.2.16",
+ "tokio",
+ "tokio-postgres",
+ "tracing",
+]
+
+[[package]]
+name = "deadpool-runtime"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b"
+dependencies = [
+ "tokio",
+]
+
 [[package]]
 name = "digest"
 version = "0.10.7"
@@ -556,8 +591,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592"
 dependencies = [
  "cfg-if",
+ "js-sys",
  "libc",
  "wasi",
+ "wasm-bindgen",
 ]
 
 [[package]]
@@ -670,6 +707,12 @@ dependencies = [
  "wasm-bindgen",
 ]
 
+[[package]]
+name = "lazy_static"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
+
 [[package]]
 name = "lexical-core"
 version = "1.0.6"
@@ -739,6 +782,16 @@ version = "0.2.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de"
 
+[[package]]
+name = "libmimalloc-sys"
+version = "0.1.44"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870"
+dependencies = [
+ "cc",
+ "libc",
+]
+
 [[package]]
 name = "libredox"
 version = "0.1.10"
@@ -800,6 +853,15 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "mimalloc"
+version = "0.1.48"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e1ee66a4b64c74f4ef288bcbb9192ad9c3feaad75193129ac8509af543894fd8"
+dependencies = [
+ "libmimalloc-sys",
+]
+
 [[package]]
 name = "mio"
 version = "1.1.0"
@@ -1529,6 +1591,37 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tracing"
+version = "0.1.41"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
+dependencies = [
+ "pin-project-lite",
+ "tracing-attributes",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-attributes"
+version = "0.1.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678"
+dependencies = [
+ "once_cell",
+]
+
 [[package]]
 name = "typenum"
 version = "1.19.0"
@@ -1544,7 +1637,9 @@ dependencies = [
  "byteorder",
  "bytes",
  "chrono",
+ "deadpool-postgres",
  "futures-util",
+ "mimalloc",
  "num_cpus",
  "pyo3",
  "pyo3-arrow",

Cargo.toml

@@ -1,9 +1,9 @@
 [package]
 name = "unchecked-io"
 version = "0.1.0"
-authors = ["Your Name/Alias"] # Replace with your name or alias
-license = "MIT OR Apache-2.0" # Good practice for open-source
-edition = "2021"
+authors = ["Billthemaker"] # Replace with your name or alias
+license = "BSL-1" # Good practice for open-source
+edition = "2024"
 
 [lib] 
 name = "unchecked_io"
@@ -26,11 +26,12 @@ tokio = { version = "1.37", features = ["full"] }
 
 # 5. Database Connection (Postgres)
 tokio-postgres = "0.7"
+deadpool-postgres = "0.14"
 
-# 6. Error Handling Crate (The FIX)
+# 6. Error Handling Crate
 anyhow = "1.0"
 
-# 7. Futures Utilities (The FIX for StreamExt)
+# 7. Futures Utilities
 futures-util = "0.3"
 
 # 8. Byte Buffer Management
@@ -50,3 +51,6 @@ pyo3-arrow = "0.15.0"
 
 # 13. System CPU Count (NEW)
 num_cpus = "1.16"
+
+[target.'cfg(not(target_env = "msvc"))'.dependencies]
+mimalloc = { version = "0.1.39" }

setup_db.sql

@@ -36,7 +36,7 @@ SELECT
     (random() * 10 + 100)::INT AS course_id,
     (NOW() - (random() * '1000 days'::INTERVAL))::DATE AS start_date,
     (random() * 5)::FLOAT8 AS rating
-FROM generate_series(1, 5000000) s(i);
+FROM generate_series(1, 20000000) s(i);
 
 -- 4. Analyze the table for better query planning (good practice)
 ANALYZE benchmark_table;

src/lib.rs

@@ -10,6 +10,15 @@ use pyo3::types::{PyModule, PyAny};
 use pyo3::Bound;
 use pyo3_arrow::PyRecordBatch;
 
+// Use the high-performance mimalloc for better multi-threaded memory allocation.
+// We conditionally compile it to avoid issues on MSVC targets.
+#[cfg(not(target_env = "msvc"))]
+use mimalloc;
+
+#[cfg(not(target_env = "msvc"))]
+#[global_allocator]
+static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
+
 // --- Internal Crates ---
 use crate::config::{load_and_validate_config, ConnectorConfig};
 use crate::parser::run_db_logic;
@@ -17,9 +26,8 @@ use crate::parser::run_db_logic;
 
 // --- THE PYTHON-CALLABLE ENTRY POINT ---
 
-// Updated signature: Removed danger_mode as it's now permanently true (unchecked)
 #[pyfunction]
-#[pyo3(signature = (config_path, blast_radius=312500))] // Optimized default blast radius
+#[pyo3(signature = (config_path, blast_radius=312500))]
 #[allow(unsafe_code)]
 #[allow(unsafe_op_in_unsafe_fn)]
 #[allow(rust_2024_compatibility)]
@@ -46,7 +54,6 @@ fn load_data_from_config<'py>(
             .build()
             .unwrap()
             .block_on(async {
-                // Now only passes the two required args
                 run_db_logic(config, blast_radius).await
             })
     }).map_err(|e| PyValueError::new_err(format!("Database/Runtime Error: {:?}", e)))?;