bill/unchecked-io
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improving efficiency and writing to file

Browse Source
This commit is contained in:
Bill
2025-11-17 16:22:54 -07:00
parent da34dbda07
commit d84e52a9f3
4 changed files with 387 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

432
Cargo.lock generated
View File

@@ -47,17 +47,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4df8bb5b0bd64c0b9bc61317fcc480bad0f00e56d3bc32c69a4c8dada4786bae"
dependencies = [
"arrow-arith",
"arrow-array",
"arrow-buffer",
"arrow-cast",
"arrow-array 57.0.0",
"arrow-buffer 57.0.0",
"arrow-cast 57.0.0",
"arrow-csv",
"arrow-data",
"arrow-data 57.0.0",
"arrow-ipc",
"arrow-json",
"arrow-ord",
"arrow-row",
"arrow-schema",
"arrow-select",
"arrow-schema 57.0.0",
"arrow-select 57.0.0",
"arrow-string",
]
@@ -67,14 +67,31 @@ version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1a640186d3bd30a24cb42264c2dafb30e236a6f50d510e56d40b708c9582491"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-array 57.0.0",
"arrow-buffer 57.0.0",
"arrow-data 57.0.0",
"arrow-schema 57.0.0",
"chrono",
"num-traits",
]
[[package]]
name = "arrow-array"
version = "56.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d"
dependencies = [
"ahash",
"arrow-buffer 56.2.0",
"arrow-data 56.2.0",
"arrow-schema 56.2.0",
"chrono",
"chrono-tz",
"half",
"hashbrown",
"num",
]
[[package]]
name = "arrow-array"
version = "57.0.0"
@@ -82,9 +99,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "219fe420e6800979744c8393b687afb0252b3f8a89b91027d27887b72aa36d31"
dependencies = [
"ahash",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-buffer 57.0.0",
"arrow-data 57.0.0",
"arrow-schema 57.0.0",
"chrono",
"half",
"hashbrown",
@@ -93,6 +110,17 @@ dependencies = [
"num-traits",
]
[[package]]
name = "arrow-buffer"
version = "56.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc"
dependencies = [
"bytes",
"half",
"num",
]
[[package]]
name = "arrow-buffer"
version = "57.0.0"
@@ -105,17 +133,38 @@ dependencies = [
"num-traits",
]
[[package]]
name = "arrow-cast"
version = "56.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023"
dependencies = [
"arrow-array 56.2.0",
"arrow-buffer 56.2.0",
"arrow-data 56.2.0",
"arrow-schema 56.2.0",
"arrow-select 56.2.0",
"atoi",
"base64",
"chrono",
"comfy-table",
"half",
"lexical-core",
"num",
"ryu",
]
[[package]]
name = "arrow-cast"
version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9c9ebb4c987e6b3b236fb4a14b20b34835abfdd80acead3ccf1f9bf399e1f168"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
"arrow-array 57.0.0",
"arrow-buffer 57.0.0",
"arrow-data 57.0.0",
"arrow-schema 57.0.0",
"arrow-select 57.0.0",
"atoi",
"base64",
"chrono",
@@ -131,23 +180,35 @@ version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "92386159c8d4bce96f8bd396b0642a0d544d471bdc2ef34d631aec80db40a09c"
dependencies = [
"arrow-array",
"arrow-cast",
"arrow-schema",
"arrow-array 57.0.0",
"arrow-cast 57.0.0",
"arrow-schema 57.0.0",
"chrono",
"csv",
"csv-core",
"regex",
]
[[package]]
name = "arrow-data"
version = "56.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0"
dependencies = [
"arrow-buffer 56.2.0",
"arrow-schema 56.2.0",
"half",
"num",
]
[[package]]
name = "arrow-data"
version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "727681b95de313b600eddc2a37e736dcb21980a40f640314dcf360e2f36bc89b"
dependencies = [
"arrow-buffer",
"arrow-schema",
"arrow-buffer 57.0.0",
"arrow-schema 57.0.0",
"half",
"num-integer",
"num-traits",
@@ -159,11 +220,11 @@ version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "da9ba92e3de170295c98a84e5af22e2b037f0c7b32449445e6c493b5fca27f27"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
"arrow-array 57.0.0",
"arrow-buffer 57.0.0",
"arrow-data 57.0.0",
"arrow-schema 57.0.0",
"arrow-select 57.0.0",
"flatbuffers",
]
@@ -173,11 +234,11 @@ version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b969b4a421ae83828591c6bf5450bd52e6d489584142845ad6a861f42fe35df8"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-cast",
"arrow-data",
"arrow-schema",
"arrow-array 57.0.0",
"arrow-buffer 57.0.0",
"arrow-cast 57.0.0",
"arrow-data 57.0.0",
"arrow-schema 57.0.0",
"chrono",
"half",
"indexmap",
@@ -197,11 +258,11 @@ version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "141c05298b21d03e88062317a1f1a73f5ba7b6eb041b350015b1cd6aabc0519b"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
"arrow-array 57.0.0",
"arrow-buffer 57.0.0",
"arrow-data 57.0.0",
"arrow-schema 57.0.0",
"arrow-select 57.0.0",
]
[[package]]
@@ -210,19 +271,42 @@ version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5f3c06a6abad6164508ed283c7a02151515cef3de4b4ff2cebbcaeb85533db2"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-array 57.0.0",
"arrow-buffer 57.0.0",
"arrow-data 57.0.0",
"arrow-schema 57.0.0",
"half",
]
[[package]]
name = "arrow-schema"
version = "56.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe"
dependencies = [
"bitflags",
]
[[package]]
name = "arrow-schema"
version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9cfa7a03d1eee2a4d061476e1840ad5c9867a544ca6c4c59256496af5d0a8be5"
[[package]]
name = "arrow-select"
version = "56.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a"
dependencies = [
"ahash",
"arrow-array 56.2.0",
"arrow-buffer 56.2.0",
"arrow-data 56.2.0",
"arrow-schema 56.2.0",
"num",
]
[[package]]
name = "arrow-select"
version = "57.0.0"
@@ -230,10 +314,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bafa595babaad59f2455f4957d0f26448fb472722c186739f4fac0823a1bdb47"
dependencies = [
"ahash",
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-array 57.0.0",
"arrow-buffer 57.0.0",
"arrow-data 57.0.0",
"arrow-schema 57.0.0",
"num-traits",
]
@@ -243,11 +327,11 @@ version = "57.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32f46457dbbb99f2650ff3ac23e46a929e0ab81db809b02aa5511c258348bef2"
dependencies = [
"arrow-array",
"arrow-buffer",
"arrow-data",
"arrow-schema",
"arrow-select",
"arrow-array 57.0.0",
"arrow-buffer 57.0.0",
"arrow-data 57.0.0",
"arrow-schema 57.0.0",
"arrow-select 57.0.0",
"memchr",
"num-traits",
"regex",
@@ -348,6 +432,27 @@ dependencies = [
"windows-link",
]
[[package]]
name = "chrono-tz"
version = "0.10.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3"
dependencies = [
"chrono",
"phf 0.12.1",
]
[[package]]
name = "comfy-table"
version = "7.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e0d05af1e006a2407bedef5af410552494ce5be9090444dbbcb57258c1af3d56"
dependencies = [
"strum",
"strum_macros",
"unicode-width",
]
[[package]]
name = "const-random"
version = "0.1.18"
@@ -566,9 +671,9 @@ checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d"
[[package]]
name = "heck"
version = "0.4.1"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
[[package]]
name = "hmac"
@@ -733,6 +838,16 @@ version = "0.4.28"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432"
[[package]]
name = "matrixmultiply"
version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08"
dependencies = [
"autocfg",
"rawpointer",
]
[[package]]
name = "md-5"
version = "0.10.6"
@@ -769,6 +884,35 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "ndarray"
version = "0.16.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841"
dependencies = [
"matrixmultiply",
"num-complex",
"num-integer",
"num-traits",
"portable-atomic",
"portable-atomic-util",
"rawpointer",
]
[[package]]
name = "num"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23"
dependencies = [
"num-bigint",
"num-complex",
"num-integer",
"num-iter",
"num-rational",
"num-traits",
]
[[package]]
name = "num-bigint"
version = "0.4.6"
@@ -797,6 +941,28 @@ dependencies = [
"num-traits",
]
[[package]]
name = "num-iter"
version = "0.1.45"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf"
dependencies = [
"autocfg",
"num-integer",
"num-traits",
]
[[package]]
name = "num-rational"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824"
dependencies = [
"num-bigint",
"num-integer",
"num-traits",
]
[[package]]
name = "num-traits"
version = "0.2.19"
@@ -807,6 +973,23 @@ dependencies = [
"libm",
]
[[package]]
name = "numpy"
version = "0.27.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fa24ffc88cf9d43f7269d6b6a0d0a00010924a8cc90604a21ef9c433b66998d"
dependencies = [
"half",
"libc",
"ndarray",
"num-complex",
"num-integer",
"num-traits",
"pyo3",
"pyo3-build-config",
"rustc-hash",
]
[[package]]
name = "once_cell"
version = "1.21.3"
@@ -842,16 +1025,34 @@ version = "2.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
[[package]]
name = "phf"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7"
dependencies = [
"phf_shared 0.12.1",
]
[[package]]
name = "phf"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
dependencies = [
"phf_shared",
"phf_shared 0.13.1",
"serde",
]
[[package]]
name = "phf_shared"
version = "0.12.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981"
dependencies = [
"siphasher",
]
[[package]]
name = "phf_shared"
version = "0.13.1"
@@ -879,6 +1080,15 @@ version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
[[package]]
name = "portable-atomic-util"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507"
dependencies = [
"portable-atomic",
]
[[package]]
name = "postgres-protocol"
version = "0.6.9"
@@ -928,15 +1138,17 @@ dependencies = [
[[package]]
name = "pyo3"
version = "0.20.3"
version = "0.27.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233"
checksum = "37a6df7eab65fc7bee654a421404947e10a0f7085b6951bf2ea395f4659fb0cf"
dependencies = [
"cfg-if",
"chrono",
"chrono-tz",
"indexmap",
"indoc",
"libc",
"memoffset",
"parking_lot",
"once_cell",
"portable-atomic",
"pyo3-build-config",
"pyo3-ffi",
@@ -945,20 +1157,40 @@ dependencies = [
]
[[package]]
name = "pyo3-build-config"
version = "0.20.3"
name = "pyo3-arrow"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7"
checksum = "0a048ed0635a5fd4b8b336ad2666ad8105b2d81bd3c358615137c44cd28955c3"
dependencies = [
"arrow-array 56.2.0",
"arrow-buffer 56.2.0",
"arrow-cast 56.2.0",
"arrow-data 56.2.0",
"arrow-schema 56.2.0",
"arrow-select 56.2.0",
"chrono",
"chrono-tz",
"half",
"indexmap",
"numpy",
"pyo3",
"thiserror",
]
[[package]]
name = "pyo3-build-config"
version = "0.27.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f77d387774f6f6eec64a004eac0ed525aab7fa1966d94b42f743797b3e395afb"
dependencies = [
"once_cell",
"target-lexicon",
]
[[package]]
name = "pyo3-ffi"
version = "0.20.3"
version = "0.27.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa"
checksum = "2dd13844a4242793e02df3e2ec093f540d948299a6a77ea9ce7afd8623f542be"
dependencies = [
"libc",
"pyo3-build-config",
@@ -966,9 +1198,9 @@ dependencies = [
[[package]]
name = "pyo3-macros"
version = "0.20.3"
version = "0.27.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158"
checksum = "eaf8f9f1108270b90d3676b8679586385430e5c0bb78bb5f043f95499c821a71"
dependencies = [
"proc-macro2",
"pyo3-macros-backend",
@@ -978,9 +1210,9 @@ dependencies = [
[[package]]
name = "pyo3-macros-backend"
version = "0.20.3"
version = "0.27.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185"
checksum = "70a3b2274450ba5288bc9b8c1b69ff569d1d61189d4bff38f8d22e03d17f932b"
dependencies = [
"heck",
"proc-macro2",
@@ -1033,6 +1265,12 @@ dependencies = [
"getrandom 0.3.4",
]
[[package]]
name = "rawpointer"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3"
[[package]]
name = "redox_syscall"
version = "0.5.18"
@@ -1071,6 +1309,12 @@ version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"
[[package]]
name = "rustc-hash"
version = "2.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
[[package]]
name = "rustc_version"
version = "0.4.1"
@@ -1231,6 +1475,25 @@ dependencies = [
"unicode-properties",
]
[[package]]
name = "strum"
version = "0.26.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06"
[[package]]
name = "strum_macros"
version = "0.26.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be"
dependencies = [
"heck",
"proc-macro2",
"quote",
"rustversion",
"syn",
]
[[package]]
name = "subtle"
version = "2.6.1"
@@ -1250,9 +1513,29 @@ dependencies = [
[[package]]
name = "target-lexicon"
version = "0.12.16"
version = "0.13.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c"
[[package]]
name = "thiserror"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52"
dependencies = [
"thiserror-impl",
]
[[package]]
name = "thiserror-impl"
version = "1.0.69"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tiny-keccak"
@@ -1321,7 +1604,7 @@ dependencies = [
"log",
"parking_lot",
"percent-encoding",
"phf",
"phf 0.13.1",
"pin-project-lite",
"postgres-protocol",
"postgres-types",
@@ -1362,6 +1645,7 @@ dependencies = [
"chrono",
"futures-util",
"pyo3",
"pyo3-arrow",
"serde",
"serde_yaml",
"tokio",
@@ -1396,6 +1680,12 @@ version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d"
[[package]]
name = "unicode-width"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"
[[package]]
name = "unindent"
version = "0.2.4"