From 9acfc919b824057a8b91348f73a163417d340238 Mon Sep 17 00:00:00 2001 From: Bill Date: Mon, 17 Nov 2025 18:49:57 -0700 Subject: [PATCH] improving efficiency and writing to file --- Cargo.lock | 239 ++++++++++++++--------------------------------------- Cargo.toml | 2 +- src/lib.rs | 30 ++++--- 3 files changed, 81 insertions(+), 190 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 58d8a9d..1202595 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,17 +47,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4df8bb5b0bd64c0b9bc61317fcc480bad0f00e56d3bc32c69a4c8dada4786bae" dependencies = [ "arrow-arith", - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-cast 57.0.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", "arrow-csv", - "arrow-data 57.0.0", + "arrow-data", "arrow-ipc", "arrow-json", "arrow-ord", "arrow-row", - "arrow-schema 57.0.0", - "arrow-select 57.0.0", + "arrow-schema", + "arrow-select", "arrow-string", ] @@ -67,31 +67,14 @@ version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1a640186d3bd30a24cb42264c2dafb30e236a6f50d510e56d40b708c9582491" dependencies = [ - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-data 57.0.0", - "arrow-schema 57.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", "num-traits", ] -[[package]] -name = "arrow-array" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" -dependencies = [ - "ahash", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "chrono", - "chrono-tz", - "half", - "hashbrown", - "num", -] - [[package]] name = "arrow-array" version = "57.0.0" @@ -99,10 +82,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219fe420e6800979744c8393b687afb0252b3f8a89b91027d27887b72aa36d31" dependencies = [ "ahash", - "arrow-buffer 57.0.0", - "arrow-data 57.0.0", - "arrow-schema 57.0.0", + "arrow-buffer", + "arrow-data", + "arrow-schema", "chrono", + "chrono-tz", "half", "hashbrown", "num-complex", @@ -110,17 +94,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "arrow-buffer" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" -dependencies = [ - "bytes", - "half", - "num", -] - [[package]] name = "arrow-buffer" version = "57.0.0" @@ -133,41 +106,21 @@ dependencies = [ "num-traits", ] -[[package]] -name = "arrow-cast" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" -dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "arrow-select 56.2.0", - "atoi", - "base64", - "chrono", - "comfy-table", - "half", - "lexical-core", - "num", - "ryu", -] - [[package]] name = "arrow-cast" version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c9ebb4c987e6b3b236fb4a14b20b34835abfdd80acead3ccf1f9bf399e1f168" dependencies = [ - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-data 57.0.0", - "arrow-schema 57.0.0", - "arrow-select 57.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "atoi", "base64", "chrono", + "comfy-table", "half", "lexical-core", "num-traits", @@ -180,35 +133,23 @@ version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92386159c8d4bce96f8bd396b0642a0d544d471bdc2ef34d631aec80db40a09c" dependencies = [ - "arrow-array 57.0.0", - "arrow-cast 57.0.0", - "arrow-schema 57.0.0", + "arrow-array", + "arrow-cast", + "arrow-schema", "chrono", "csv", "csv-core", "regex", ] -[[package]] -name = "arrow-data" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" -dependencies = [ - "arrow-buffer 56.2.0", - "arrow-schema 56.2.0", - "half", - "num", -] - [[package]] name = "arrow-data" version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "727681b95de313b600eddc2a37e736dcb21980a40f640314dcf360e2f36bc89b" dependencies = [ - "arrow-buffer 57.0.0", - "arrow-schema 57.0.0", + "arrow-buffer", + "arrow-schema", "half", "num-integer", "num-traits", @@ -220,11 +161,11 @@ version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da9ba92e3de170295c98a84e5af22e2b037f0c7b32449445e6c493b5fca27f27" dependencies = [ - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-data 57.0.0", - "arrow-schema 57.0.0", - "arrow-select 57.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "flatbuffers", ] @@ -234,11 +175,11 @@ version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b969b4a421ae83828591c6bf5450bd52e6d489584142845ad6a861f42fe35df8" dependencies = [ - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-cast 57.0.0", - "arrow-data 57.0.0", - "arrow-schema 57.0.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", "chrono", "half", "indexmap", @@ -258,11 +199,11 @@ version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "141c05298b21d03e88062317a1f1a73f5ba7b6eb041b350015b1cd6aabc0519b" dependencies = [ - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-data 57.0.0", - "arrow-schema 57.0.0", - "arrow-select 57.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", ] [[package]] @@ -271,40 +212,20 @@ version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5f3c06a6abad6164508ed283c7a02151515cef3de4b4ff2cebbcaeb85533db2" dependencies = [ - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-data 57.0.0", - "arrow-schema 57.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "half", ] -[[package]] -name = "arrow-schema" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" -dependencies = [ - "bitflags", -] - [[package]] name = "arrow-schema" version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cfa7a03d1eee2a4d061476e1840ad5c9867a544ca6c4c59256496af5d0a8be5" - -[[package]] -name = "arrow-select" -version = "56.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" dependencies = [ - "ahash", - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "num", + "bitflags", ] [[package]] @@ -314,10 +235,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bafa595babaad59f2455f4957d0f26448fb472722c186739f4fac0823a1bdb47" dependencies = [ "ahash", - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-data 57.0.0", - "arrow-schema 57.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", "num-traits", ] @@ -327,11 +248,11 @@ version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32f46457dbbb99f2650ff3ac23e46a929e0ab81db809b02aa5511c258348bef2" dependencies = [ - "arrow-array 57.0.0", - "arrow-buffer 57.0.0", - "arrow-data 57.0.0", - "arrow-schema 57.0.0", - "arrow-select 57.0.0", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", "memchr", "num-traits", "regex", @@ -899,20 +820,6 @@ dependencies = [ "rawpointer", ] -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - [[package]] name = "num-bigint" version = "0.4.6" @@ -941,28 +848,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "num-iter" -version = "0.1.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" -dependencies = [ - "autocfg", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -1158,16 +1043,16 @@ dependencies = [ [[package]] name = "pyo3-arrow" -version = "0.13.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a048ed0635a5fd4b8b336ad2666ad8105b2d81bd3c358615137c44cd28955c3" +checksum = "36b9f03cb749b0326951ebb30e39eda2f32b0b9205dce67e947e65779b8faffc" dependencies = [ - "arrow-array 56.2.0", - "arrow-buffer 56.2.0", - "arrow-cast 56.2.0", - "arrow-data 56.2.0", - "arrow-schema 56.2.0", - "arrow-select 56.2.0", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-schema", + "arrow-select", "chrono", "chrono-tz", "half", diff --git a/Cargo.toml b/Cargo.toml index 8b83b24..770ebae 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,4 +33,4 @@ bytes = "1.11.0" byteorder = "1.5.0" chrono = "0.4" uuid = { version = "1.8", features = ["serde", "v4"] } -pyo3-arrow = "0.13" \ No newline at end of file +pyo3-arrow = "0.15.0" \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index a32c3f6..d305165 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -6,13 +6,17 @@ mod parser; use pyo3::prelude::*; use pyo3::exceptions::PyValueError; use tokio; +use pyo3::types::PyModule; +use pyo3::Bound; + +// FIX 1: Import PyRecordBatch. +use pyo3_arrow::PyRecordBatch; + // --- Internal Crates --- use crate::config::{load_and_validate_config, ConnectorConfig}; use crate::parser::run_db_logic; -// FIX: Import the Arrow bridge trait -use pyo3_arrow::PyArrowConvert; // --- THE PYTHON-CALLABLE ENTRY POINT --- @@ -20,8 +24,7 @@ use pyo3_arrow::PyArrowConvert; #[allow(unsafe_code)] #[allow(unsafe_op_in_unsafe_fn)] #[allow(rust_2024_compatibility)] -// FIX: Change the return type from () to PyObject -fn load_data_from_config(py: Python, config_path: String) -> PyResult { +fn load_data_from_config<'py>(py: Python<'py>, config_path: String) -> PyResult> { // --- Phase 1: Load and Validate Configuration --- let config: ConnectorConfig = match load_and_validate_config(&config_path) { Ok(c) => c, @@ -33,7 +36,6 @@ fn load_data_from_config(py: Python, config_path: String) -> PyResult println!("Columns (in order): {:?}", config.schema.iter().map(|c| &c.column_name).collect::>()); // --- Phase 2: Run Core Logic --- - // We release the GIL and start our own Tokio runtime let record_batch = py.allow_threads(|| { tokio::runtime::Builder::new_multi_thread() .enable_all() @@ -45,18 +47,22 @@ fn load_data_from_config(py: Python, config_path: String) -> PyResult }).map_err(|e| PyValueError::new_err(format!("Database/Runtime Error: {:?}", e)))?; // --- Phase 3: Return Data to Python --- - // FIX: Convert the Rust RecordBatch to a Python PyObject (a pyarrow.Table) - // This is a ZERO-COPY operation. - record_batch.to_pyarrow(py) + // FIX 2: Create PyRecordBatch wrapper using the standard PyO3 __new__ convention. + // let py_record_batch = PyRecordBatch::new(record_batch); + // // Replace: let py_record_batch = PyRecordBatch::new(py, record_batch).map_err(|e| ...)?; + // // With: + let py_record_batch = PyRecordBatch::new(record_batch); + // Return the PyRecordBatch wrapper object as a generic PyObject reference. + py_record_batch.into_pyarrow(py) } // --- PYTHON MODULE EXPORT --- - #[pymodule] -// FIX: Add the attribute to enable Arrow <-> Python conversion -#[pyo3(with_arrow)] -fn unchecked_io(_py: Python, m: &PyModule) -> PyResult<()> { +fn unchecked_io(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { + // FIX: Use the two-argument version of wrap_pyfunction! + // This resolves the E0308 type mismatch. m.add_function(wrap_pyfunction!(load_data_from_config, m)?)?; + Ok(()) } \ No newline at end of file