bill/unchecked-io
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improving efficiency and writing to file

Browse Source
This commit is contained in:
Bill
2025-11-17 18:49:57 -07:00
parent d84e52a9f3
commit 9acfc919b8
3 changed files with 81 additions and 190 deletions
Download Patch File Download Diff File Expand all files Collapse all files

239
Cargo.lock generated
View File

@@ -47,17 +47,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4df8bb5b0bd64c0b9bc61317fcc480bad0f00e56d3bc32c69a4c8dada4786bae" checksum = "4df8bb5b0bd64c0b9bc61317fcc480bad0f00e56d3bc32c69a4c8dada4786bae"
dependencies = [ dependencies = [
"arrow-arith", "arrow-arith",
"arrow-array 57.0.0", "arrow-array",
"arrow-buffer 57.0.0", "arrow-buffer",
"arrow-cast 57.0.0", "arrow-cast",
"arrow-csv", "arrow-csv",
"arrow-data 57.0.0", "arrow-data",
"arrow-ipc", "arrow-ipc",
"arrow-json", "arrow-json",
"arrow-ord", "arrow-ord",
"arrow-row", "arrow-row",
"arrow-schema 57.0.0", "arrow-schema",
"arrow-select 57.0.0", "arrow-select",
"arrow-string", "arrow-string",
] ]
@@ -67,31 +67,14 @@ version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1a640186d3bd30a24cb42264c2dafb30e236a6f50d510e56d40b708c9582491" checksum = "a1a640186d3bd30a24cb42264c2dafb30e236a6f50d510e56d40b708c9582491"
dependencies = [ dependencies = [
"arrow-array 57.0.0", "arrow-array",
"arrow-buffer 57.0.0", "arrow-buffer",
"arrow-data 57.0.0", "arrow-data",
"arrow-schema 57.0.0", "arrow-schema",
"chrono", "chrono",
"num-traits", "num-traits",
] ]
[[package]]
name = "arrow-array"
version = "56.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d"
dependencies = [
"ahash",
"arrow-buffer 56.2.0",
"arrow-data 56.2.0",
"arrow-schema 56.2.0",
"chrono",
"chrono-tz",
"half",
"hashbrown",
"num",
]
[[package]] [[package]]
name = "arrow-array" name = "arrow-array"
version = "57.0.0" version = "57.0.0"
@@ -99,10 +82,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "219fe420e6800979744c8393b687afb0252b3f8a89b91027d27887b72aa36d31" checksum = "219fe420e6800979744c8393b687afb0252b3f8a89b91027d27887b72aa36d31"
dependencies = [ dependencies = [
"ahash", "ahash",
"arrow-buffer 57.0.0", "arrow-buffer",
"arrow-data 57.0.0", "arrow-data",
"arrow-schema 57.0.0", "arrow-schema",
"chrono", "chrono",
"chrono-tz",
"half", "half",
"hashbrown", "hashbrown",
"num-complex", "num-complex",
@@ -110,17 +94,6 @@ dependencies = [
"num-traits", "num-traits",
] ]
[[package]]
name = "arrow-buffer"
version = "56.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc"
dependencies = [
"bytes",
"half",
"num",
]
[[package]] [[package]]
name = "arrow-buffer" name = "arrow-buffer"
version = "57.0.0" version = "57.0.0"
@@ -133,41 +106,21 @@ dependencies = [
"num-traits", "num-traits",
] ]
[[package]]
name = "arrow-cast"
version = "56.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023"
dependencies = [
"arrow-array 56.2.0",
"arrow-buffer 56.2.0",
"arrow-data 56.2.0",
"arrow-schema 56.2.0",
"arrow-select 56.2.0",
"atoi",
"base64",
"chrono",
"comfy-table",
"half",
"lexical-core",
"num",
"ryu",
]
[[package]] [[package]]
name = "arrow-cast" name = "arrow-cast"
version = "57.0.0" version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c9ebb4c987e6b3b236fb4a14b20b34835abfdd80acead3ccf1f9bf399e1f168" checksum = "9c9ebb4c987e6b3b236fb4a14b20b34835abfdd80acead3ccf1f9bf399e1f168"
dependencies = [ dependencies = [
"arrow-array 57.0.0", "arrow-array",
"arrow-buffer 57.0.0", "arrow-buffer",
"arrow-data 57.0.0", "arrow-data",
"arrow-schema 57.0.0", "arrow-schema",
"arrow-select 57.0.0", "arrow-select",
"atoi", "atoi",
"base64", "base64",
"chrono", "chrono",
"comfy-table",
"half", "half",
"lexical-core", "lexical-core",
"num-traits", "num-traits",
@@ -180,35 +133,23 @@ version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92386159c8d4bce96f8bd396b0642a0d544d471bdc2ef34d631aec80db40a09c" checksum = "92386159c8d4bce96f8bd396b0642a0d544d471bdc2ef34d631aec80db40a09c"
dependencies = [ dependencies = [
"arrow-array 57.0.0", "arrow-array",
"arrow-cast 57.0.0", "arrow-cast",
"arrow-schema 57.0.0", "arrow-schema",
"chrono", "chrono",
"csv", "csv",
"csv-core", "csv-core",
"regex", "regex",
] ]
[[package]]
name = "arrow-data"
version = "56.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0"
dependencies = [
"arrow-buffer 56.2.0",
"arrow-schema 56.2.0",
"half",
"num",
]
[[package]] [[package]]
name = "arrow-data" name = "arrow-data"
version = "57.0.0" version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "727681b95de313b600eddc2a37e736dcb21980a40f640314dcf360e2f36bc89b" checksum = "727681b95de313b600eddc2a37e736dcb21980a40f640314dcf360e2f36bc89b"
dependencies = [ dependencies = [
"arrow-buffer 57.0.0", "arrow-buffer",
"arrow-schema 57.0.0", "arrow-schema",
"half", "half",
"num-integer", "num-integer",
"num-traits", "num-traits",
@@ -220,11 +161,11 @@ version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da9ba92e3de170295c98a84e5af22e2b037f0c7b32449445e6c493b5fca27f27" checksum = "da9ba92e3de170295c98a84e5af22e2b037f0c7b32449445e6c493b5fca27f27"
dependencies = [ dependencies = [
"arrow-array 57.0.0", "arrow-array",
"arrow-buffer 57.0.0", "arrow-buffer",
"arrow-data 57.0.0", "arrow-data",
"arrow-schema 57.0.0", "arrow-schema",
"arrow-select 57.0.0", "arrow-select",
"flatbuffers", "flatbuffers",
] ]
@@ -234,11 +175,11 @@ version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b969b4a421ae83828591c6bf5450bd52e6d489584142845ad6a861f42fe35df8" checksum = "b969b4a421ae83828591c6bf5450bd52e6d489584142845ad6a861f42fe35df8"
dependencies = [ dependencies = [
"arrow-array 57.0.0", "arrow-array",
"arrow-buffer 57.0.0", "arrow-buffer",
"arrow-cast 57.0.0", "arrow-cast",
"arrow-data 57.0.0", "arrow-data",
"arrow-schema 57.0.0", "arrow-schema",
"chrono", "chrono",
"half", "half",
"indexmap", "indexmap",
@@ -258,11 +199,11 @@ version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "141c05298b21d03e88062317a1f1a73f5ba7b6eb041b350015b1cd6aabc0519b" checksum = "141c05298b21d03e88062317a1f1a73f5ba7b6eb041b350015b1cd6aabc0519b"
dependencies = [ dependencies = [
"arrow-array 57.0.0", "arrow-array",
"arrow-buffer 57.0.0", "arrow-buffer",
"arrow-data 57.0.0", "arrow-data",
"arrow-schema 57.0.0", "arrow-schema",
"arrow-select 57.0.0", "arrow-select",
] ]
[[package]] [[package]]
@@ -271,40 +212,20 @@ version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5f3c06a6abad6164508ed283c7a02151515cef3de4b4ff2cebbcaeb85533db2" checksum = "c5f3c06a6abad6164508ed283c7a02151515cef3de4b4ff2cebbcaeb85533db2"
dependencies = [ dependencies = [
"arrow-array 57.0.0", "arrow-array",
"arrow-buffer 57.0.0", "arrow-buffer",
"arrow-data 57.0.0", "arrow-data",
"arrow-schema 57.0.0", "arrow-schema",
"half", "half",
] ]
[[package]]
name = "arrow-schema"
version = "56.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe"
dependencies = [
"bitflags",
]
[[package]] [[package]]
name = "arrow-schema" name = "arrow-schema"
version = "57.0.0" version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cfa7a03d1eee2a4d061476e1840ad5c9867a544ca6c4c59256496af5d0a8be5" checksum = "9cfa7a03d1eee2a4d061476e1840ad5c9867a544ca6c4c59256496af5d0a8be5"
[[package]]
name = "arrow-select"
version = "56.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a"
dependencies = [ dependencies = [
"ahash", "bitflags",
"arrow-array 56.2.0",
"arrow-buffer 56.2.0",
"arrow-data 56.2.0",
"arrow-schema 56.2.0",
"num",
] ]
[[package]] [[package]]
@@ -314,10 +235,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bafa595babaad59f2455f4957d0f26448fb472722c186739f4fac0823a1bdb47" checksum = "bafa595babaad59f2455f4957d0f26448fb472722c186739f4fac0823a1bdb47"
dependencies = [ dependencies = [
"ahash", "ahash",
"arrow-array 57.0.0", "arrow-array",
"arrow-buffer 57.0.0", "arrow-buffer",
"arrow-data 57.0.0", "arrow-data",
"arrow-schema 57.0.0", "arrow-schema",
"num-traits", "num-traits",
] ]
@@ -327,11 +248,11 @@ version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32f46457dbbb99f2650ff3ac23e46a929e0ab81db809b02aa5511c258348bef2" checksum = "32f46457dbbb99f2650ff3ac23e46a929e0ab81db809b02aa5511c258348bef2"
dependencies = [ dependencies = [
"arrow-array 57.0.0", "arrow-array",
"arrow-buffer 57.0.0", "arrow-buffer",
"arrow-data 57.0.0", "arrow-data",
"arrow-schema 57.0.0", "arrow-schema",
"arrow-select 57.0.0", "arrow-select",
"memchr", "memchr",
"num-traits", "num-traits",
"regex", "regex",
@@ -899,20 +820,6 @@ dependencies = [
"rawpointer", "rawpointer",
] ]
[[package]]
name = "num"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23"
dependencies = [
"num-bigint",
"num-complex",
"num-integer",
"num-iter",
"num-rational",
"num-traits",
]
[[package]] [[package]]
name = "num-bigint" name = "num-bigint"
version = "0.4.6" version = "0.4.6"
@@ -941,28 +848,6 @@ dependencies = [
"num-traits", "num-traits",
] ]
[[package]]
name = "num-iter"
version = "0.1.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf"
dependencies = [
"autocfg",
"num-integer",
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
dependencies = [
"num-bigint",
"num-integer",
"num-traits",
]
[[package]] [[package]]
name = "num-traits" name = "num-traits"
version = "0.2.19" version = "0.2.19"
@@ -1158,16 +1043,16 @@ dependencies = [
[[package]] [[package]]
name = "pyo3-arrow" name = "pyo3-arrow"
version = "0.13.0" version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a048ed0635a5fd4b8b336ad2666ad8105b2d81bd3c358615137c44cd28955c3" checksum = "36b9f03cb749b0326951ebb30e39eda2f32b0b9205dce67e947e65779b8faffc"
dependencies = [ dependencies = [
"arrow-array 56.2.0", "arrow-array",
"arrow-buffer 56.2.0", "arrow-buffer",
"arrow-cast 56.2.0", "arrow-cast",
"arrow-data 56.2.0", "arrow-data",
"arrow-schema 56.2.0", "arrow-schema",
"arrow-select 56.2.0", "arrow-select",
"chrono", "chrono",
"chrono-tz", "chrono-tz",
"half", "half",

2
Cargo.toml
View File

@@ -33,4 +33,4 @@ bytes = "1.11.0"
byteorder = "1.5.0" byteorder = "1.5.0"
chrono = "0.4" chrono = "0.4"
uuid = { version = "1.8", features = ["serde", "v4"] } uuid = { version = "1.8", features = ["serde", "v4"] }
pyo3-arrow = "0.13" pyo3-arrow = "0.15.0"

30
src/lib.rs
View File

@@ -6,13 +6,17 @@ mod parser;
use pyo3::prelude::*; use pyo3::prelude::*;
use pyo3::exceptions::PyValueError; use pyo3::exceptions::PyValueError;
use tokio; use tokio;
use pyo3::types::PyModule;
use pyo3::Bound;
// FIX 1: Import PyRecordBatch.
use pyo3_arrow::PyRecordBatch;
// --- Internal Crates --- // --- Internal Crates ---
use crate::config::{load_and_validate_config, ConnectorConfig}; use crate::config::{load_and_validate_config, ConnectorConfig};
use crate::parser::run_db_logic; use crate::parser::run_db_logic;
// FIX: Import the Arrow bridge trait
use pyo3_arrow::PyArrowConvert;
// --- THE PYTHON-CALLABLE ENTRY POINT --- // --- THE PYTHON-CALLABLE ENTRY POINT ---
@@ -20,8 +24,7 @@ use pyo3_arrow::PyArrowConvert;
#[allow(unsafe_code)] #[allow(unsafe_code)]
#[allow(unsafe_op_in_unsafe_fn)] #[allow(unsafe_op_in_unsafe_fn)]
#[allow(rust_2024_compatibility)] #[allow(rust_2024_compatibility)]
// FIX: Change the return type from () to PyObject fn load_data_from_config<'py>(py: Python<'py>, config_path: String) -> PyResult<Bound<'py, PyAny>> {
fn load_data_from_config(py: Python, config_path: String) -> PyResult<PyObject> {
// --- Phase 1: Load and Validate Configuration --- // --- Phase 1: Load and Validate Configuration ---
let config: ConnectorConfig = match load_and_validate_config(&config_path) { let config: ConnectorConfig = match load_and_validate_config(&config_path) {
Ok(c) => c, Ok(c) => c,
@@ -33,7 +36,6 @@ fn load_data_from_config(py: Python, config_path: String) -> PyResult<PyObject>
println!("Columns (in order): {:?}", config.schema.iter().map(|c| &c.column_name).collect::<Vec<_>>()); println!("Columns (in order): {:?}", config.schema.iter().map(|c| &c.column_name).collect::<Vec<_>>());
// --- Phase 2: Run Core Logic --- // --- Phase 2: Run Core Logic ---
// We release the GIL and start our own Tokio runtime
let record_batch = py.allow_threads(|| { let record_batch = py.allow_threads(|| {
tokio::runtime::Builder::new_multi_thread() tokio::runtime::Builder::new_multi_thread()
.enable_all() .enable_all()
@@ -45,18 +47,22 @@ fn load_data_from_config(py: Python, config_path: String) -> PyResult<PyObject>
}).map_err(|e| PyValueError::new_err(format!("Database/Runtime Error: {:?}", e)))?; }).map_err(|e| PyValueError::new_err(format!("Database/Runtime Error: {:?}", e)))?;
// --- Phase 3: Return Data to Python --- // --- Phase 3: Return Data to Python ---
// FIX: Convert the Rust RecordBatch to a Python PyObject (a pyarrow.Table) // FIX 2: Create PyRecordBatch wrapper using the standard PyO3 __new__ convention.
// This is a ZERO-COPY operation. // let py_record_batch = PyRecordBatch::new(record_batch);
record_batch.to_pyarrow(py) // // Replace: let py_record_batch = PyRecordBatch::new(py, record_batch).map_err(|e| ...)?;
// // With:
let py_record_batch = PyRecordBatch::new(record_batch);
// Return the PyRecordBatch wrapper object as a generic PyObject reference.
py_record_batch.into_pyarrow(py)
} }
// --- PYTHON MODULE EXPORT --- // --- PYTHON MODULE EXPORT ---
#[pymodule] #[pymodule]
// FIX: Add the attribute to enable Arrow <-> Python conversion fn unchecked_io(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
#[pyo3(with_arrow)] // FIX: Use the two-argument version of wrap_pyfunction!
fn unchecked_io(_py: Python, m: &PyModule) -> PyResult<()> { // This resolves the E0308 type mismatch.
m.add_function(wrap_pyfunction!(load_data_from_config, m)?)?; m.add_function(wrap_pyfunction!(load_data_from_config, m)?)?;
Ok(()) Ok(())
} }