diff --git a/Cargo.lock b/Cargo.lock index ea35547..58d8a9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -47,17 +47,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4df8bb5b0bd64c0b9bc61317fcc480bad0f00e56d3bc32c69a4c8dada4786bae" dependencies = [ "arrow-arith", - "arrow-array", - "arrow-buffer", - "arrow-cast", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-cast 57.0.0", "arrow-csv", - "arrow-data", + "arrow-data 57.0.0", "arrow-ipc", "arrow-json", "arrow-ord", "arrow-row", - "arrow-schema", - "arrow-select", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", "arrow-string", ] @@ -67,14 +67,31 @@ version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1a640186d3bd30a24cb42264c2dafb30e236a6f50d510e56d40b708c9582491" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", "chrono", "num-traits", ] +[[package]] +name = "arrow-array" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" +dependencies = [ + "ahash", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "chrono", + "chrono-tz", + "half", + "hashbrown", + "num", +] + [[package]] name = "arrow-array" version = "57.0.0" @@ -82,9 +99,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "219fe420e6800979744c8393b687afb0252b3f8a89b91027d27887b72aa36d31" dependencies = [ "ahash", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", "chrono", "half", "hashbrown", @@ -93,6 +110,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-buffer" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" +dependencies = [ + "bytes", + "half", + "num", +] + [[package]] name = "arrow-buffer" version = "57.0.0" @@ -105,17 +133,38 @@ dependencies = [ "num-traits", ] +[[package]] +name = "arrow-cast" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "atoi", + "base64", + "chrono", + "comfy-table", + "half", + "lexical-core", + "num", + "ryu", +] + [[package]] name = "arrow-cast" version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c9ebb4c987e6b3b236fb4a14b20b34835abfdd80acead3ccf1f9bf399e1f168" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", "atoi", "base64", "chrono", @@ -131,23 +180,35 @@ version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92386159c8d4bce96f8bd396b0642a0d544d471bdc2ef34d631aec80db40a09c" dependencies = [ - "arrow-array", - "arrow-cast", - "arrow-schema", + "arrow-array 57.0.0", + "arrow-cast 57.0.0", + "arrow-schema 57.0.0", "chrono", "csv", "csv-core", "regex", ] +[[package]] +name = "arrow-data" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" +dependencies = [ + "arrow-buffer 56.2.0", + "arrow-schema 56.2.0", + "half", + "num", +] + [[package]] name = "arrow-data" version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "727681b95de313b600eddc2a37e736dcb21980a40f640314dcf360e2f36bc89b" dependencies = [ - "arrow-buffer", - "arrow-schema", + "arrow-buffer 57.0.0", + "arrow-schema 57.0.0", "half", "num-integer", "num-traits", @@ -159,11 +220,11 @@ version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da9ba92e3de170295c98a84e5af22e2b037f0c7b32449445e6c493b5fca27f27" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", "flatbuffers", ] @@ -173,11 +234,11 @@ version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b969b4a421ae83828591c6bf5450bd52e6d489584142845ad6a861f42fe35df8" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-cast", - "arrow-data", - "arrow-schema", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-cast 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", "chrono", "half", "indexmap", @@ -197,11 +258,11 @@ version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "141c05298b21d03e88062317a1f1a73f5ba7b6eb041b350015b1cd6aabc0519b" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", ] [[package]] @@ -210,19 +271,42 @@ version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5f3c06a6abad6164508ed283c7a02151515cef3de4b4ff2cebbcaeb85533db2" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", "half", ] +[[package]] +name = "arrow-schema" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" +dependencies = [ + "bitflags", +] + [[package]] name = "arrow-schema" version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cfa7a03d1eee2a4d061476e1840ad5c9867a544ca6c4c59256496af5d0a8be5" +[[package]] +name = "arrow-select" +version = "56.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" +dependencies = [ + "ahash", + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "num", +] + [[package]] name = "arrow-select" version = "57.0.0" @@ -230,10 +314,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bafa595babaad59f2455f4957d0f26448fb472722c186739f4fac0823a1bdb47" dependencies = [ "ahash", - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", "num-traits", ] @@ -243,11 +327,11 @@ version = "57.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32f46457dbbb99f2650ff3ac23e46a929e0ab81db809b02aa5511c258348bef2" dependencies = [ - "arrow-array", - "arrow-buffer", - "arrow-data", - "arrow-schema", - "arrow-select", + "arrow-array 57.0.0", + "arrow-buffer 57.0.0", + "arrow-data 57.0.0", + "arrow-schema 57.0.0", + "arrow-select 57.0.0", "memchr", "num-traits", "regex", @@ -348,6 +432,27 @@ dependencies = [ "windows-link", ] +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf 0.12.1", +] + +[[package]] +name = "comfy-table" +version = "7.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56" +dependencies = [ + "strum", + "strum_macros", + "unicode-width", +] + [[package]] name = "const-random" version = "0.1.18" @@ -566,9 +671,9 @@ checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" [[package]] name = "heck" -version = "0.4.1" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hmac" @@ -733,6 +838,16 @@ version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "md-5" version = "0.10.6" @@ -769,6 +884,35 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -797,6 +941,28 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -807,6 +973,23 @@ dependencies = [ "libm", ] +[[package]] +name = "numpy" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa24ffc88cf9d43f7269d6b6a0d0a00010924a8cc90604a21ef9c433b66998d" +dependencies = [ + "half", + "libc", + "ndarray", + "num-complex", + "num-integer", + "num-traits", + "pyo3", + "pyo3-build-config", + "rustc-hash", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -842,16 +1025,34 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared 0.12.1", +] + [[package]] name = "phf" version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ - "phf_shared", + "phf_shared 0.13.1", "serde", ] +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + [[package]] name = "phf_shared" version = "0.13.1" @@ -879,6 +1080,15 @@ version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + [[package]] name = "postgres-protocol" version = "0.6.9" @@ -928,15 +1138,17 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.20.3" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" +checksum = "37a6df7eab65fc7bee654a421404947e10a0f7085b6951bf2ea395f4659fb0cf" dependencies = [ - "cfg-if", + "chrono", + "chrono-tz", + "indexmap", "indoc", "libc", "memoffset", - "parking_lot", + "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", @@ -945,20 +1157,40 @@ dependencies = [ ] [[package]] -name = "pyo3-build-config" -version = "0.20.3" +name = "pyo3-arrow" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7" +checksum = "0a048ed0635a5fd4b8b336ad2666ad8105b2d81bd3c358615137c44cd28955c3" +dependencies = [ + "arrow-array 56.2.0", + "arrow-buffer 56.2.0", + "arrow-cast 56.2.0", + "arrow-data 56.2.0", + "arrow-schema 56.2.0", + "arrow-select 56.2.0", + "chrono", + "chrono-tz", + "half", + "indexmap", + "numpy", + "pyo3", + "thiserror", +] + +[[package]] +name = "pyo3-build-config" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f77d387774f6f6eec64a004eac0ed525aab7fa1966d94b42f743797b3e395afb" dependencies = [ - "once_cell", "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.20.3" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" +checksum = "2dd13844a4242793e02df3e2ec093f540d948299a6a77ea9ce7afd8623f542be" dependencies = [ "libc", "pyo3-build-config", @@ -966,9 +1198,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.20.3" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" +checksum = "eaf8f9f1108270b90d3676b8679586385430e5c0bb78bb5f043f95499c821a71" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -978,9 +1210,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.20.3" +version = "0.27.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" +checksum = "70a3b2274450ba5288bc9b8c1b69ff569d1d61189d4bff38f8d22e03d17f932b" dependencies = [ "heck", "proc-macro2", @@ -1033,6 +1265,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1071,6 +1309,12 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + [[package]] name = "rustc_version" version = "0.4.1" @@ -1231,6 +1475,25 @@ dependencies = [ "unicode-properties", ] +[[package]] +name = "strum" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" + +[[package]] +name = "strum_macros" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "rustversion", + "syn", +] + [[package]] name = "subtle" version = "2.6.1" @@ -1250,9 +1513,29 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.16" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" +checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c" + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] [[package]] name = "tiny-keccak" @@ -1321,7 +1604,7 @@ dependencies = [ "log", "parking_lot", "percent-encoding", - "phf", + "phf 0.13.1", "pin-project-lite", "postgres-protocol", "postgres-types", @@ -1362,6 +1645,7 @@ dependencies = [ "chrono", "futures-util", "pyo3", + "pyo3-arrow", "serde", "serde_yaml", "tokio", @@ -1396,6 +1680,12 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "unindent" version = "0.2.4" diff --git a/Cargo.toml b/Cargo.toml index 7c7ee6a..8b83b24 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ crate-type = ["cdylib", "rlib"] [dependencies] # 1. Python Bindings for FFI -pyo3 = { version = "0.20", features = ["extension-module"] } +pyo3 = { version = "0.27.1", features = ["extension-module"] } # 2. Configuration Parsing serde = { version = "1.0", features = ["derive"] } @@ -32,4 +32,5 @@ futures-util = "0.3.31" bytes = "1.11.0" byteorder = "1.5.0" chrono = "0.4" -uuid = { version = "1.8", features = ["serde", "v4"] } \ No newline at end of file +uuid = { version = "1.8", features = ["serde", "v4"] } +pyo3-arrow = "0.13" \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index a594a78..a32c3f6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,18 +1,18 @@ // --- Declare our new modules --- -// This tells Rust to look for src/config.rs and src/parser.rs mod config; mod parser; // --- External Crates --- use pyo3::prelude::*; use pyo3::exceptions::PyValueError; -use tokio; // We need the tokio runtime here +use tokio; // --- Internal Crates --- -// We 'use' the public functions from our new modules use crate::config::{load_and_validate_config, ConnectorConfig}; use crate::parser::run_db_logic; +// FIX: Import the Arrow bridge trait +use pyo3_arrow::PyArrowConvert; // --- THE PYTHON-CALLABLE ENTRY POINT --- @@ -20,9 +20,9 @@ use crate::parser::run_db_logic; #[allow(unsafe_code)] #[allow(unsafe_op_in_unsafe_fn)] #[allow(rust_2024_compatibility)] -fn load_data_from_config(py: Python, config_path: String) -> PyResult<()> { +// FIX: Change the return type from () to PyObject +fn load_data_from_config(py: Python, config_path: String) -> PyResult { // --- Phase 1: Load and Validate Configuration --- - // We call the function that now lives in src/config.rs let config: ConnectorConfig = match load_and_validate_config(&config_path) { Ok(c) => c, Err(e) => return Err(PyValueError::new_err(format!("Configuration Error: {:?}", e))), @@ -34,24 +34,28 @@ fn load_data_from_config(py: Python, config_path: String) -> PyResult<()> { // --- Phase 2: Run Core Logic --- // We release the GIL and start our own Tokio runtime - py.allow_threads(|| { + let record_batch = py.allow_threads(|| { tokio::runtime::Builder::new_multi_thread() .enable_all() .build() .unwrap() .block_on(async { - // We call the function that now lives in src/parser.rs run_db_logic(config).await }) }).map_err(|e| PyValueError::new_err(format!("Database/Runtime Error: {:?}", e)))?; - Ok(()) + // --- Phase 3: Return Data to Python --- + // FIX: Convert the Rust RecordBatch to a Python PyObject (a pyarrow.Table) + // This is a ZERO-COPY operation. + record_batch.to_pyarrow(py) } // --- PYTHON MODULE EXPORT --- #[pymodule] +// FIX: Add the attribute to enable Arrow <-> Python conversion +#[pyo3(with_arrow)] fn unchecked_io(_py: Python, m: &PyModule) -> PyResult<()> { m.add_function(wrap_pyfunction!(load_data_from_config, m)?)?; Ok(()) diff --git a/src/parser.rs b/src/parser.rs index 5185656..2178444 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -3,13 +3,11 @@ use std::pin::Pin; use std::sync::Arc; use tokio_postgres::{NoTls, CopyOutStream}; use anyhow::{Context, Result, anyhow}; -// FIX: Import new builders use arrow::array::{ ArrayBuilder, ArrayRef, Int64Builder, Float64Builder, Float32Builder, StringBuilder, BooleanBuilder, TimestampNanosecondBuilder, Date32Builder, Int32Builder }; -// FIX: Import new types use arrow::datatypes::{ DataType, Field, Schema, Float64Type, Float32Type, Int64Type, Int32Type, Utf8Type, BooleanType, TimestampNanosecondType, @@ -28,7 +26,8 @@ use crate::config::ConnectorConfig; // --- 1. CORE DATABASE LOGIC --- -pub async fn run_db_logic(config: ConnectorConfig) -> Result<()> { +// FIX: Change return type from Result<()> to Result +pub async fn run_db_logic(config: ConnectorConfig) -> Result { // 1. Establish the connection println!("UncheckedIO: Attempting connection..."); @@ -42,7 +41,6 @@ pub async fn run_db_logic(config: ConnectorConfig) -> Result<()> { }); // 2. Execute the COPY TO STDOUT command - // FIX: We are now using the query from the config file! let copy_query = &config.query; println!("UncheckedIO: Executing user-defined query..."); @@ -54,18 +52,17 @@ pub async fn run_db_logic(config: ConnectorConfig) -> Result<()> { // Build the Arrow Schema from the config let schema_fields: Vec = config.schema.iter().map(|col_cfg| { - // We now allow "nullable" to be controlled by the column name, a temporary "hack" let nullable = col_cfg.column_name == "notes"; let arrow_type = match col_cfg.arrow_type.as_str() { "Int64" => DataType::Int64, - "Int32" => DataType::Int32, // NEW - "Float64" => DataType::Float64, // NEW + "Int32" => DataType::Int32, + "Float64" => DataType::Float64, "Float32" => DataType::Float32, "Utf8" | "String" => DataType::Utf8, "Boolean" => DataType::Boolean, "Timestamp(Nanosecond, None)" => DataType::Timestamp(arrow::datatypes::TimeUnit::Nanosecond, None), - "Date32" => DataType::Date32, // NEW + "Date32" => DataType::Date32, _ => panic!("Unsupported type in config: {}", col_cfg.arrow_type), }; Field::new(&col_cfg.column_name, arrow_type, nullable) @@ -82,7 +79,9 @@ pub async fn run_db_logic(config: ConnectorConfig) -> Result<()> { record_batch.num_rows(), record_batch.num_columns()); println!("UncheckedIO: Data transfer complete. We lived."); - Ok(()) + + // FIX: Return the finished RecordBatch + Ok(record_batch) } @@ -91,13 +90,13 @@ pub async fn run_db_logic(config: ConnectorConfig) -> Result<()> { // This enum will hold our different builder types enum DynamicBuilder { Int64(Box), - Int32(Box), // NEW - Float64(Box), // NEW + Int32(Box), + Float64(Box), Float32(Box), String(Box), Boolean(Box), Timestamp(Box), - Date32(Box), // NEW + Date32(Box), } // Postgres Epoch for timestamps