improving efficiency and writing to file

Cargo.lock

@@ -47,17 +47,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "4df8bb5b0bd64c0b9bc61317fcc480bad0f00e56d3bc32c69a4c8dada4786bae"
 dependencies = [
  "arrow-arith",
- "arrow-array",
+ "arrow-array 57.0.0",
- "arrow-buffer",
+ "arrow-buffer 57.0.0",
- "arrow-cast",
+ "arrow-cast 57.0.0",
  "arrow-csv",
- "arrow-data",
+ "arrow-data 57.0.0",
  "arrow-ipc",
  "arrow-json",
  "arrow-ord",
  "arrow-row",
- "arrow-schema",
+ "arrow-schema 57.0.0",
- "arrow-select",
+ "arrow-select 57.0.0",
  "arrow-string",
 ]
 
@@ -67,14 +67,31 @@ version = "57.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a1a640186d3bd30a24cb42264c2dafb30e236a6f50d510e56d40b708c9582491"
 dependencies = [
- "arrow-array",
+ "arrow-array 57.0.0",
- "arrow-buffer",
+ "arrow-buffer 57.0.0",
- "arrow-data",
+ "arrow-data 57.0.0",
- "arrow-schema",
+ "arrow-schema 57.0.0",
  "chrono",
  "num-traits",
 ]
 
+[[package]]
+name = "arrow-array"
+version = "56.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d"
+dependencies = [
+ "ahash",
+ "arrow-buffer 56.2.0",
+ "arrow-data 56.2.0",
+ "arrow-schema 56.2.0",
+ "chrono",
+ "chrono-tz",
+ "half",
+ "hashbrown",
+ "num",
+]
+
 [[package]]
 name = "arrow-array"
 version = "57.0.0"
@@ -82,9 +99,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "219fe420e6800979744c8393b687afb0252b3f8a89b91027d27887b72aa36d31"
 dependencies = [
  "ahash",
- "arrow-buffer",
+ "arrow-buffer 57.0.0",
- "arrow-data",
+ "arrow-data 57.0.0",
- "arrow-schema",
+ "arrow-schema 57.0.0",
  "chrono",
  "half",
  "hashbrown",
@@ -93,6 +110,17 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "arrow-buffer"
+version = "56.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc"
+dependencies = [
+ "bytes",
+ "half",
+ "num",
+]
+
 [[package]]
 name = "arrow-buffer"
 version = "57.0.0"
@@ -105,17 +133,38 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "arrow-cast"
+version = "56.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023"
+dependencies = [
+ "arrow-array 56.2.0",
+ "arrow-buffer 56.2.0",
+ "arrow-data 56.2.0",
+ "arrow-schema 56.2.0",
+ "arrow-select 56.2.0",
+ "atoi",
+ "base64",
+ "chrono",
+ "comfy-table",
+ "half",
+ "lexical-core",
+ "num",
+ "ryu",
+]
+
 [[package]]
 name = "arrow-cast"
 version = "57.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9c9ebb4c987e6b3b236fb4a14b20b34835abfdd80acead3ccf1f9bf399e1f168"
 dependencies = [
- "arrow-array",
+ "arrow-array 57.0.0",
- "arrow-buffer",
+ "arrow-buffer 57.0.0",
- "arrow-data",
+ "arrow-data 57.0.0",
- "arrow-schema",
+ "arrow-schema 57.0.0",
- "arrow-select",
+ "arrow-select 57.0.0",
  "atoi",
  "base64",
  "chrono",
@@ -131,23 +180,35 @@ version = "57.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "92386159c8d4bce96f8bd396b0642a0d544d471bdc2ef34d631aec80db40a09c"
 dependencies = [
- "arrow-array",
+ "arrow-array 57.0.0",
- "arrow-cast",
+ "arrow-cast 57.0.0",
- "arrow-schema",
+ "arrow-schema 57.0.0",
  "chrono",
  "csv",
  "csv-core",
  "regex",
 ]
 
+[[package]]
+name = "arrow-data"
+version = "56.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0"
+dependencies = [
+ "arrow-buffer 56.2.0",
+ "arrow-schema 56.2.0",
+ "half",
+ "num",
+]
+
 [[package]]
 name = "arrow-data"
 version = "57.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "727681b95de313b600eddc2a37e736dcb21980a40f640314dcf360e2f36bc89b"
 dependencies = [
- "arrow-buffer",
+ "arrow-buffer 57.0.0",
- "arrow-schema",
+ "arrow-schema 57.0.0",
  "half",
  "num-integer",
  "num-traits",
@@ -159,11 +220,11 @@ version = "57.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "da9ba92e3de170295c98a84e5af22e2b037f0c7b32449445e6c493b5fca27f27"
 dependencies = [
- "arrow-array",
+ "arrow-array 57.0.0",
- "arrow-buffer",
+ "arrow-buffer 57.0.0",
- "arrow-data",
+ "arrow-data 57.0.0",
- "arrow-schema",
+ "arrow-schema 57.0.0",
- "arrow-select",
+ "arrow-select 57.0.0",
  "flatbuffers",
 ]
 
@@ -173,11 +234,11 @@ version = "57.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b969b4a421ae83828591c6bf5450bd52e6d489584142845ad6a861f42fe35df8"
 dependencies = [
- "arrow-array",
+ "arrow-array 57.0.0",
- "arrow-buffer",
+ "arrow-buffer 57.0.0",
- "arrow-cast",
+ "arrow-cast 57.0.0",
- "arrow-data",
+ "arrow-data 57.0.0",
- "arrow-schema",
+ "arrow-schema 57.0.0",
  "chrono",
  "half",
  "indexmap",
@@ -197,11 +258,11 @@ version = "57.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "141c05298b21d03e88062317a1f1a73f5ba7b6eb041b350015b1cd6aabc0519b"
 dependencies = [
- "arrow-array",
+ "arrow-array 57.0.0",
- "arrow-buffer",
+ "arrow-buffer 57.0.0",
- "arrow-data",
+ "arrow-data 57.0.0",
- "arrow-schema",
+ "arrow-schema 57.0.0",
- "arrow-select",
+ "arrow-select 57.0.0",
 ]
 
 [[package]]
@@ -210,19 +271,42 @@ version = "57.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c5f3c06a6abad6164508ed283c7a02151515cef3de4b4ff2cebbcaeb85533db2"
 dependencies = [
- "arrow-array",
+ "arrow-array 57.0.0",
- "arrow-buffer",
+ "arrow-buffer 57.0.0",
- "arrow-data",
+ "arrow-data 57.0.0",
- "arrow-schema",
+ "arrow-schema 57.0.0",
  "half",
 ]
 
+[[package]]
+name = "arrow-schema"
+version = "56.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe"
+dependencies = [
+ "bitflags",
+]
+
 [[package]]
 name = "arrow-schema"
 version = "57.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9cfa7a03d1eee2a4d061476e1840ad5c9867a544ca6c4c59256496af5d0a8be5"
 
+[[package]]
+name = "arrow-select"
+version = "56.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a"
+dependencies = [
+ "ahash",
+ "arrow-array 56.2.0",
+ "arrow-buffer 56.2.0",
+ "arrow-data 56.2.0",
+ "arrow-schema 56.2.0",
+ "num",
+]
+
 [[package]]
 name = "arrow-select"
 version = "57.0.0"
@@ -230,10 +314,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bafa595babaad59f2455f4957d0f26448fb472722c186739f4fac0823a1bdb47"
 dependencies = [
  "ahash",
- "arrow-array",
+ "arrow-array 57.0.0",
- "arrow-buffer",
+ "arrow-buffer 57.0.0",
- "arrow-data",
+ "arrow-data 57.0.0",
- "arrow-schema",
+ "arrow-schema 57.0.0",
  "num-traits",
 ]
 
@@ -243,11 +327,11 @@ version = "57.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32f46457dbbb99f2650ff3ac23e46a929e0ab81db809b02aa5511c258348bef2"
 dependencies = [
- "arrow-array",
+ "arrow-array 57.0.0",
- "arrow-buffer",
+ "arrow-buffer 57.0.0",
- "arrow-data",
+ "arrow-data 57.0.0",
- "arrow-schema",
+ "arrow-schema 57.0.0",
- "arrow-select",
+ "arrow-select 57.0.0",
  "memchr",
  "num-traits",
  "regex",
@@ -348,6 +432,27 @@ dependencies = [
  "windows-link",
 ]
 
+[[package]]
+name = "chrono-tz"
+version = "0.10.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3"
+dependencies = [
+ "chrono",
+ "phf 0.12.1",
+]
+
+[[package]]
+name = "comfy-table"
+version = "7.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56"
+dependencies = [
+ "strum",
+ "strum_macros",
+ "unicode-width",
+]
+
 [[package]]
 name = "const-random"
 version = "0.1.18"
@@ -566,9 +671,9 @@ checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d"
 
 [[package]]
 name = "heck"
-version = "0.4.1"
+version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
 [[package]]
 name = "hmac"
@@ -733,6 +838,16 @@ version = "0.4.28"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432"
 
+[[package]]
+name = "matrixmultiply"
+version = "0.3.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08"
+dependencies = [
+ "autocfg",
+ "rawpointer",
+]
+
 [[package]]
 name = "md-5"
 version = "0.10.6"
@@ -769,6 +884,35 @@ dependencies = [
  "windows-sys 0.61.2",
 ]
 
+[[package]]
+name = "ndarray"
+version = "0.16.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841"
+dependencies = [
+ "matrixmultiply",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "portable-atomic",
+ "portable-atomic-util",
+ "rawpointer",
+]
+
+[[package]]
+name = "num"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23"
+dependencies = [
+ "num-bigint",
+ "num-complex",
+ "num-integer",
+ "num-iter",
+ "num-rational",
+ "num-traits",
+]
+
 [[package]]
 name = "num-bigint"
 version = "0.4.6"
@@ -797,6 +941,28 @@ dependencies = [
  "num-traits",
 ]
 
+[[package]]
+name = "num-iter"
+version = "0.1.45"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf"
+dependencies = [
+ "autocfg",
+ "num-integer",
+ "num-traits",
+]
+
+[[package]]
+name = "num-rational"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
+dependencies = [
+ "num-bigint",
+ "num-integer",
+ "num-traits",
+]
+
 [[package]]
 name = "num-traits"
 version = "0.2.19"
@@ -807,6 +973,23 @@ dependencies = [
  "libm",
 ]
 
+[[package]]
+name = "numpy"
+version = "0.27.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fa24ffc88cf9d43f7269d6b6a0d0a00010924a8cc90604a21ef9c433b66998d"
+dependencies = [
+ "half",
+ "libc",
+ "ndarray",
+ "num-complex",
+ "num-integer",
+ "num-traits",
+ "pyo3",
+ "pyo3-build-config",
+ "rustc-hash",
+]
+
 [[package]]
 name = "once_cell"
 version = "1.21.3"
@@ -842,16 +1025,34 @@ version = "2.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
 
+[[package]]
+name = "phf"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7"
+dependencies = [
+ "phf_shared 0.12.1",
+]
+
 [[package]]
 name = "phf"
 version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
 dependencies = [
- "phf_shared",
+ "phf_shared 0.13.1",
  "serde",
 ]
 
+[[package]]
+name = "phf_shared"
+version = "0.12.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981"
+dependencies = [
+ "siphasher",
+]
+
 [[package]]
 name = "phf_shared"
 version = "0.13.1"
@@ -879,6 +1080,15 @@ version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
 
+[[package]]
+name = "portable-atomic-util"
+version = "0.2.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507"
+dependencies = [
+ "portable-atomic",
+]
+
 [[package]]
 name = "postgres-protocol"
 version = "0.6.9"
@@ -928,15 +1138,17 @@ dependencies = [
 
 [[package]]
 name = "pyo3"
-version = "0.20.3"
+version = "0.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233"
+checksum = "37a6df7eab65fc7bee654a421404947e10a0f7085b6951bf2ea395f4659fb0cf"
 dependencies = [
- "cfg-if",
+ "chrono",
+ "chrono-tz",
+ "indexmap",
  "indoc",
  "libc",
  "memoffset",
- "parking_lot",
+ "once_cell",
  "portable-atomic",
  "pyo3-build-config",
  "pyo3-ffi",
@@ -945,20 +1157,40 @@ dependencies = [
 ]
 
 [[package]]
-name = "pyo3-build-config"
+name = "pyo3-arrow"
-version = "0.20.3"
+version = "0.13.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7"
+checksum = "0a048ed0635a5fd4b8b336ad2666ad8105b2d81bd3c358615137c44cd28955c3"
+dependencies = [
+ "arrow-array 56.2.0",
+ "arrow-buffer 56.2.0",
+ "arrow-cast 56.2.0",
+ "arrow-data 56.2.0",
+ "arrow-schema 56.2.0",
+ "arrow-select 56.2.0",
+ "chrono",
+ "chrono-tz",
+ "half",
+ "indexmap",
+ "numpy",
+ "pyo3",
+ "thiserror",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.27.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f77d387774f6f6eec64a004eac0ed525aab7fa1966d94b42f743797b3e395afb"
 dependencies = [
- "once_cell",
  "target-lexicon",
 ]
 
 [[package]]
 name = "pyo3-ffi"
-version = "0.20.3"
+version = "0.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa"
+checksum = "2dd13844a4242793e02df3e2ec093f540d948299a6a77ea9ce7afd8623f542be"
 dependencies = [
  "libc",
  "pyo3-build-config",
@@ -966,9 +1198,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros"
-version = "0.20.3"
+version = "0.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158"
+checksum = "eaf8f9f1108270b90d3676b8679586385430e5c0bb78bb5f043f95499c821a71"
 dependencies = [
  "proc-macro2",
  "pyo3-macros-backend",
@@ -978,9 +1210,9 @@ dependencies = [
 
 [[package]]
 name = "pyo3-macros-backend"
-version = "0.20.3"
+version = "0.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185"
+checksum = "70a3b2274450ba5288bc9b8c1b69ff569d1d61189d4bff38f8d22e03d17f932b"
 dependencies = [
  "heck",
  "proc-macro2",
@@ -1033,6 +1265,12 @@ dependencies = [
  "getrandom 0.3.4",
 ]
 
+[[package]]
+name = "rawpointer"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
+
 [[package]]
 name = "redox_syscall"
 version = "0.5.18"
@@ -1071,6 +1309,12 @@ version = "0.8.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"
 
+[[package]]
+name = "rustc-hash"
+version = "2.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
+
 [[package]]
 name = "rustc_version"
 version = "0.4.1"
@@ -1231,6 +1475,25 @@ dependencies = [
  "unicode-properties",
 ]
 
+[[package]]
+name = "strum"
+version = "0.26.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
+
+[[package]]
+name = "strum_macros"
+version = "0.26.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "quote",
+ "rustversion",
+ "syn",
+]
+
 [[package]]
 name = "subtle"
 version = "2.6.1"
@@ -1250,9 +1513,29 @@ dependencies = [
 
 [[package]]
 name = "target-lexicon"
-version = "0.12.16"
+version = "0.13.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
+checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c"
+
+[[package]]
+name = "thiserror"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
 
 [[package]]
 name = "tiny-keccak"
@@ -1321,7 +1604,7 @@ dependencies = [
  "log",
  "parking_lot",
  "percent-encoding",
- "phf",
+ "phf 0.13.1",
  "pin-project-lite",
  "postgres-protocol",
  "postgres-types",
@@ -1362,6 +1645,7 @@ dependencies = [
  "chrono",
  "futures-util",
  "pyo3",
+ "pyo3-arrow",
  "serde",
  "serde_yaml",
  "tokio",
@@ -1396,6 +1680,12 @@ version = "0.1.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"
 
+[[package]]
+name = "unicode-width"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
+
 [[package]]
 name = "unindent"
 version = "0.2.4"

Cargo.toml

@@ -13,7 +13,7 @@ crate-type = ["cdylib", "rlib"]
 
 [dependencies]
 # 1. Python Bindings for FFI
-pyo3 = { version = "0.20", features = ["extension-module"] }
+pyo3 = { version = "0.27.1", features = ["extension-module"] }
 
 # 2. Configuration Parsing
 serde = { version = "1.0", features = ["derive"] }
@@ -33,3 +33,4 @@ bytes = "1.11.0"
 byteorder = "1.5.0"
 chrono = "0.4"
 uuid = { version = "1.8", features = ["serde", "v4"] }
+pyo3-arrow = "0.13"

src/lib.rs

@@ -1,18 +1,18 @@
 // --- Declare our new modules ---
-// This tells Rust to look for src/config.rs and src/parser.rs
 mod config;
 mod parser;
 
 // --- External Crates ---
 use pyo3::prelude::*;
 use pyo3::exceptions::PyValueError;
-use tokio; // We need the tokio runtime here
+use tokio;
 
 // --- Internal Crates ---
-// We 'use' the public functions from our new modules
 use crate::config::{load_and_validate_config, ConnectorConfig};
 use crate::parser::run_db_logic;
 
+// FIX: Import the Arrow bridge trait
+use pyo3_arrow::PyArrowConvert;
 
 // --- THE PYTHON-CALLABLE ENTRY POINT ---
 
@@ -20,9 +20,9 @@ use crate::parser::run_db_logic;
 #[allow(unsafe_code)]
 #[allow(unsafe_op_in_unsafe_fn)]
 #[allow(rust_2024_compatibility)]
-fn load_data_from_config(py: Python, config_path: String) -> PyResult<()> {
+// FIX: Change the return type from () to PyObject
+fn load_data_from_config(py: Python, config_path: String) -> PyResult<PyObject> {
     // --- Phase 1: Load and Validate Configuration ---
-    // We call the function that now lives in src/config.rs
     let config: ConnectorConfig = match load_and_validate_config(&config_path) {
         Ok(c) => c,
         Err(e) => return Err(PyValueError::new_err(format!("Configuration Error: {:?}", e))),
@@ -34,24 +34,28 @@ fn load_data_from_config(py: Python, config_path: String) -> PyResult<()> {
 
     // --- Phase 2: Run Core Logic ---
     // We release the GIL and start our own Tokio runtime
-    py.allow_threads(|| {
+    let record_batch = py.allow_threads(|| {
         tokio::runtime::Builder::new_multi_thread()
             .enable_all()
             .build()
             .unwrap()
             .block_on(async {
-                // We call the function that now lives in src/parser.rs
                 run_db_logic(config).await
             })
     }).map_err(|e| PyValueError::new_err(format!("Database/Runtime Error: {:?}", e)))?;
 
-    Ok(())
+    // --- Phase 3: Return Data to Python ---
+    // FIX: Convert the Rust RecordBatch to a Python PyObject (a pyarrow.Table)
+    // This is a ZERO-COPY operation.
+    record_batch.to_pyarrow(py)
 }
 
 
 // --- PYTHON MODULE EXPORT ---
 
 #[pymodule]
+// FIX: Add the attribute to enable Arrow <-> Python conversion
+#[pyo3(with_arrow)]
 fn unchecked_io(_py: Python, m: &PyModule) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(load_data_from_config, m)?)?;
     Ok(())

src/parser.rs

@@ -3,13 +3,11 @@ use std::pin::Pin;
 use std::sync::Arc;
 use tokio_postgres::{NoTls, CopyOutStream};
 use anyhow::{Context, Result, anyhow};
-// FIX: Import new builders
 use arrow::array::{
     ArrayBuilder, ArrayRef,
     Int64Builder, Float64Builder, Float32Builder, StringBuilder, BooleanBuilder,
     TimestampNanosecondBuilder, Date32Builder, Int32Builder
 };
-// FIX: Import new types
 use arrow::datatypes::{
     DataType, Field, Schema,
     Float64Type, Float32Type, Int64Type, Int32Type, Utf8Type, BooleanType, TimestampNanosecondType,
@@ -28,7 +26,8 @@ use crate::config::ConnectorConfig;
 
 
 // --- 1. CORE DATABASE LOGIC ---
-pub async fn run_db_logic(config: ConnectorConfig) -> Result<()> {
+// FIX: Change return type from Result<()> to Result<RecordBatch>
+pub async fn run_db_logic(config: ConnectorConfig) -> Result<RecordBatch> {
     // 1. Establish the connection
     println!("UncheckedIO: Attempting connection...");
 
@@ -42,7 +41,6 @@ pub async fn run_db_logic(config: ConnectorConfig) -> Result<()> {
     });
 
     // 2. Execute the COPY TO STDOUT command
-    // FIX: We are now using the query from the config file!
     let copy_query = &config.query;
     println!("UncheckedIO: Executing user-defined query...");
 
@@ -54,18 +52,17 @@ pub async fn run_db_logic(config: ConnectorConfig) -> Result<()> {
 
     // Build the Arrow Schema from the config
     let schema_fields: Vec<Field> = config.schema.iter().map(|col_cfg| {
-        // We now allow "nullable" to be controlled by the column name, a temporary "hack"
         let nullable = col_cfg.column_name == "notes";
 
         let arrow_type = match col_cfg.arrow_type.as_str() {
             "Int64" => DataType::Int64,
-            "Int32" => DataType::Int32, // NEW
+            "Int32" => DataType::Int32,
-            "Float64" => DataType::Float64, // NEW
+            "Float64" => DataType::Float64,
             "Float32" => DataType::Float32,
             "Utf8" | "String" => DataType::Utf8,
             "Boolean" => DataType::Boolean,
             "Timestamp(Nanosecond, None)" => DataType::Timestamp(arrow::datatypes::TimeUnit::Nanosecond, None),
-            "Date32" => DataType::Date32, // NEW
+            "Date32" => DataType::Date32,
             _ => panic!("Unsupported type in config: {}", col_cfg.arrow_type),
         };
         Field::new(&col_cfg.column_name, arrow_type, nullable)
@@ -82,7 +79,9 @@ pub async fn run_db_logic(config: ConnectorConfig) -> Result<()> {
              record_batch.num_rows(), record_batch.num_columns());
 
     println!("UncheckedIO: Data transfer complete. We lived.");
-    Ok(())
+
+    // FIX: Return the finished RecordBatch
+    Ok(record_batch)
 }
 
 
@@ -91,13 +90,13 @@ pub async fn run_db_logic(config: ConnectorConfig) -> Result<()> {
 // This enum will hold our different builder types
 enum DynamicBuilder {
     Int64(Box<Int64Builder>),
-    Int32(Box<Int32Builder>), // NEW
+    Int32(Box<Int32Builder>),
-    Float64(Box<Float64Builder>), // NEW
+    Float64(Box<Float64Builder>),
     Float32(Box<Float32Builder>),
     String(Box<StringBuilder>),
     Boolean(Box<BooleanBuilder>),
     Timestamp(Box<TimestampNanosecondBuilder>),
-    Date32(Box<Date32Builder>), // NEW
+    Date32(Box<Date32Builder>),
 }
 
 // Postgres Epoch for timestamps