optimizing hardware limits in Rust
This commit is contained in:
Bill
95
Cargo.lock
generated
95
Cargo.lock
generated
@@ -446,6 +446,41 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deadpool"
|
||||
version = "0.12.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b"
|
||||
dependencies = [
|
||||
"deadpool-runtime",
|
||||
"lazy_static",
|
||||
"num_cpus",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deadpool-postgres"
|
||||
version = "0.14.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d697d376cbfa018c23eb4caab1fd1883dd9c906a8c034e8d9a3cb06a7e0bef9"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"deadpool",
|
||||
"getrandom 0.2.16",
|
||||
"tokio",
|
||||
"tokio-postgres",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "deadpool-runtime"
|
||||
version = "0.1.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b"
|
||||
dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "digest"
|
||||
version = "0.10.7"
|
||||
@@ -556,8 +591,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"js-sys",
|
||||
"libc",
|
||||
"wasi",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -670,6 +707,12 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lazy_static"
|
||||
version = "1.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
|
||||
|
||||
[[package]]
|
||||
name = "lexical-core"
|
||||
version = "1.0.6"
|
||||
@@ -739,6 +782,16 @@ version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de"
|
||||
|
||||
[[package]]
|
||||
name = "libmimalloc-sys"
|
||||
version = "0.1.44"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "667f4fec20f29dfc6bc7357c582d91796c169ad7e2fce709468aefeb2c099870"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libredox"
|
||||
version = "0.1.10"
|
||||
@@ -800,6 +853,15 @@ dependencies = [
|
||||
"autocfg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mimalloc"
|
||||
version = "0.1.48"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e1ee66a4b64c74f4ef288bcbb9192ad9c3feaad75193129ac8509af543894fd8"
|
||||
dependencies = [
|
||||
"libmimalloc-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mio"
|
||||
version = "1.1.0"
|
||||
@@ -1529,6 +1591,37 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing"
|
||||
version = "0.1.41"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
|
||||
dependencies = [
|
||||
"pin-project-lite",
|
||||
"tracing-attributes",
|
||||
"tracing-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-attributes"
|
||||
version = "0.1.30"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tracing-core"
|
||||
version = "0.1.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typenum"
|
||||
version = "1.19.0"
|
||||
@@ -1544,7 +1637,9 @@ dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"deadpool-postgres",
|
||||
"futures-util",
|
||||
"mimalloc",
|
||||
"num_cpus",
|
||||
"pyo3",
|
||||
"pyo3-arrow",
|
||||
|
||||
16
Cargo.toml
16
Cargo.toml
@@ -1,9 +1,9 @@
|
||||
[package]
|
||||
name = "unchecked-io"
|
||||
version = "0.1.0"
|
||||
authors = ["Your Name/Alias"] # Replace with your name or alias
|
||||
license = "MIT OR Apache-2.0" # Good practice for open-source
|
||||
edition = "2021"
|
||||
authors = ["Billthemaker"] # Replace with your name or alias
|
||||
license = "BSL-1" # Good practice for open-source
|
||||
edition = "2024"
|
||||
|
||||
[lib]
|
||||
name = "unchecked_io"
|
||||
@@ -26,11 +26,12 @@ tokio = { version = "1.37", features = ["full"] }
|
||||
|
||||
# 5. Database Connection (Postgres)
|
||||
tokio-postgres = "0.7"
|
||||
deadpool-postgres = "0.14"
|
||||
|
||||
# 6. Error Handling Crate (The FIX)
|
||||
# 6. Error Handling Crate
|
||||
anyhow = "1.0"
|
||||
|
||||
# 7. Futures Utilities (The FIX for StreamExt)
|
||||
# 7. Futures Utilities
|
||||
futures-util = "0.3"
|
||||
|
||||
# 8. Byte Buffer Management
|
||||
@@ -49,4 +50,7 @@ uuid = { version = "1.8", features = ["serde", "v4"] }
|
||||
pyo3-arrow = "0.15.0"
|
||||
|
||||
# 13. System CPU Count (NEW)
|
||||
num_cpus = "1.16"
|
||||
num_cpus = "1.16"
|
||||
|
||||
[target.'cfg(not(target_env = "msvc"))'.dependencies]
|
||||
mimalloc = { version = "0.1.39" }
|
||||
@@ -36,7 +36,7 @@ SELECT
|
||||
(random() * 10 + 100)::INT AS course_id,
|
||||
(NOW() - (random() * '1000 days'::INTERVAL))::DATE AS start_date,
|
||||
(random() * 5)::FLOAT8 AS rating
|
||||
FROM generate_series(1, 5000000) s(i);
|
||||
FROM generate_series(1, 20000000) s(i);
|
||||
|
||||
-- 4. Analyze the table for better query planning (good practice)
|
||||
ANALYZE benchmark_table;
|
||||
|
||||
13
src/lib.rs
13
src/lib.rs
@@ -10,6 +10,15 @@ use pyo3::types::{PyModule, PyAny};
|
||||
use pyo3::Bound;
|
||||
use pyo3_arrow::PyRecordBatch;
|
||||
|
||||
// Use the high-performance mimalloc for better multi-threaded memory allocation.
|
||||
// We conditionally compile it to avoid issues on MSVC targets.
|
||||
#[cfg(not(target_env = "msvc"))]
|
||||
use mimalloc;
|
||||
|
||||
#[cfg(not(target_env = "msvc"))]
|
||||
#[global_allocator]
|
||||
static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
|
||||
|
||||
// --- Internal Crates ---
|
||||
use crate::config::{load_and_validate_config, ConnectorConfig};
|
||||
use crate::parser::run_db_logic;
|
||||
@@ -17,9 +26,8 @@ use crate::parser::run_db_logic;
|
||||
|
||||
// --- THE PYTHON-CALLABLE ENTRY POINT ---
|
||||
|
||||
// Updated signature: Removed danger_mode as it's now permanently true (unchecked)
|
||||
#[pyfunction]
|
||||
#[pyo3(signature = (config_path, blast_radius=312500))] // Optimized default blast radius
|
||||
#[pyo3(signature = (config_path, blast_radius=312500))]
|
||||
#[allow(unsafe_code)]
|
||||
#[allow(unsafe_op_in_unsafe_fn)]
|
||||
#[allow(rust_2024_compatibility)]
|
||||
@@ -46,7 +54,6 @@ fn load_data_from_config<'py>(
|
||||
.build()
|
||||
.unwrap()
|
||||
.block_on(async {
|
||||
// Now only passes the two required args
|
||||
run_db_logic(config, blast_radius).await
|
||||
})
|
||||
}).map_err(|e| PyValueError::new_err(format!("Database/Runtime Error: {:?}", e)))?;
|
||||
|
||||
Reference in New Issue
Block a user