diff --git a/Cargo.lock b/Cargo.lock index c5d1bf2..3a2dcd6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -446,6 +446,41 @@ dependencies = [ "memchr", ] +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-postgres" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d697d376cbfa018c23eb4caab1fd1883dd9c906a8c034e8d9a3cb06a7e0bef9" +dependencies = [ + "async-trait", + "deadpool", + "getrandom 0.2.16", + "tokio", + "tokio-postgres", + "tracing", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" +dependencies = [ + "tokio", +] + [[package]] name = "digest" version = "0.10.7" @@ -556,8 +591,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi", + "wasm-bindgen", ] [[package]] @@ -670,6 +707,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "lexical-core" version = "1.0.6" @@ -739,6 +782,16 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" +[[package]] +name = "libmimalloc-sys" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870" +dependencies = [ + "cc", + "libc", +] + [[package]] name = "libredox" version = "0.1.10" @@ -800,6 +853,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "mimalloc" +version = "0.1.48" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1ee66a4b64c74f4ef288bcbb9192ad9c3feaad75193129ac8509af543894fd8" +dependencies = [ + "libmimalloc-sys", +] + [[package]] name = "mio" version = "1.1.0" @@ -1529,6 +1591,37 @@ dependencies = [ "tokio", ] +[[package]] +name = "tracing" +version = "0.1.41" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tracing-core" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678" +dependencies = [ + "once_cell", +] + [[package]] name = "typenum" version = "1.19.0" @@ -1544,7 +1637,9 @@ dependencies = [ "byteorder", "bytes", "chrono", + "deadpool-postgres", "futures-util", + "mimalloc", "num_cpus", "pyo3", "pyo3-arrow", diff --git a/Cargo.toml b/Cargo.toml index fb7e40e..0bcfe93 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,9 +1,9 @@ [package] name = "unchecked-io" version = "0.1.0" -authors = ["Your Name/Alias"] # Replace with your name or alias -license = "MIT OR Apache-2.0" # Good practice for open-source -edition = "2021" +authors = ["Billthemaker"] # Replace with your name or alias +license = "BSL-1" # Good practice for open-source +edition = "2024" [lib] name = "unchecked_io" @@ -26,11 +26,12 @@ tokio = { version = "1.37", features = ["full"] } # 5. Database Connection (Postgres) tokio-postgres = "0.7" +deadpool-postgres = "0.14" -# 6. Error Handling Crate (The FIX) +# 6. Error Handling Crate anyhow = "1.0" -# 7. Futures Utilities (The FIX for StreamExt) +# 7. Futures Utilities futures-util = "0.3" # 8. Byte Buffer Management @@ -49,4 +50,7 @@ uuid = { version = "1.8", features = ["serde", "v4"] } pyo3-arrow = "0.15.0" # 13. System CPU Count (NEW) -num_cpus = "1.16" \ No newline at end of file +num_cpus = "1.16" + +[target.'cfg(not(target_env = "msvc"))'.dependencies] +mimalloc = { version = "0.1.39" } \ No newline at end of file diff --git a/setup_db.sql b/setup_db.sql index 8e38cf3..99b4251 100644 --- a/setup_db.sql +++ b/setup_db.sql @@ -36,7 +36,7 @@ SELECT (random() * 10 + 100)::INT AS course_id, (NOW() - (random() * '1000 days'::INTERVAL))::DATE AS start_date, (random() * 5)::FLOAT8 AS rating -FROM generate_series(1, 5000000) s(i); +FROM generate_series(1, 20000000) s(i); -- 4. Analyze the table for better query planning (good practice) ANALYZE benchmark_table; diff --git a/src/lib.rs b/src/lib.rs index dbf648a..d4b1843 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -10,6 +10,15 @@ use pyo3::types::{PyModule, PyAny}; use pyo3::Bound; use pyo3_arrow::PyRecordBatch; +// Use the high-performance mimalloc for better multi-threaded memory allocation. +// We conditionally compile it to avoid issues on MSVC targets. +#[cfg(not(target_env = "msvc"))] +use mimalloc; + +#[cfg(not(target_env = "msvc"))] +#[global_allocator] +static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; + // --- Internal Crates --- use crate::config::{load_and_validate_config, ConnectorConfig}; use crate::parser::run_db_logic; @@ -17,9 +26,8 @@ use crate::parser::run_db_logic; // --- THE PYTHON-CALLABLE ENTRY POINT --- -// Updated signature: Removed danger_mode as it's now permanently true (unchecked) #[pyfunction] -#[pyo3(signature = (config_path, blast_radius=312500))] // Optimized default blast radius +#[pyo3(signature = (config_path, blast_radius=312500))] #[allow(unsafe_code)] #[allow(unsafe_op_in_unsafe_fn)] #[allow(rust_2024_compatibility)] @@ -46,7 +54,6 @@ fn load_data_from_config<'py>( .build() .unwrap() .block_on(async { - // Now only passes the two required args run_db_logic(config, blast_radius).await }) }).map_err(|e| PyValueError::new_err(format!("Database/Runtime Error: {:?}", e)))?;