bill/unchecked-io
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

keep postgress connections open and switch tasks between them. Setup builds 20 million row data tables now

Browse Source
This commit is contained in:
Bill
2025-11-18 14:18:38 -07:00
parent 2888c0b5e7
commit 71d78f3fab
2 changed files with 68 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
benchmark.py
View File

@@ -16,7 +16,7 @@ DB_PORT = "5433" # <-- Your local Docker port
DB_NAME = "postgres"
# Global Configuration
BLAST_RADIUS = 125000 # Rows per parallel task (1M / 62500 = 16 partitions)
BLAST_RADIUS = 1250000 # Rows per parallel task (1M / 62500 = 16 partitions)
# SQLAlchemy connection string (for Pandas)
sqlalchemy_conn_str = f"postgresql://{DB_USER}:{DB_PASS}@{DB_HOST}:{DB_PORT}/{DB_NAME}"
@@ -57,8 +57,9 @@ print(f"UncheckedIO Config: {config_file} (dynamically created for local test)")
# --- 3. Define Benchmark Functions ---
def test_pandas():
df = pd.read_sql(sql_query, engine)
return df
# df = pd.read_sql(sql_query, engine)
# return df
pass
def test_connectorx():
df = cx.read_sql(connectorx_conn_str, sql_query, return_type="arrow")
@@ -70,16 +71,16 @@ def test_unchecked_io():
return arrow_table
# --- 4. Run Benchmarks ---
run_count = 3
print(f"Running benchmarks for 5,000,000 rows (average of {run_count} runs)...")
run_count = 1
print(f"Running benchmarks for 20,000,000 rows (average of {run_count} runs)...")
print(f"Blast Radius: {BLAST_RADIUS} rows per task")
# --- Pandas ---
print("\nRunning Pandas warmup...")
_ = test_pandas()
print("Timing pandas.read_sql...")
pandas_time = timeit.timeit(test_pandas, number=run_count) / run_count
print(f"Pandas Average Time: {pandas_time * 1000:.2f} ms")
# print("\nRunning Pandas warmup...")
# _ = test_pandas()
# print("Timing pandas.read_sql...")
# pandas_time = timeit.timeit(test_pandas, number=run_count) / run_count
# print(f"Pandas Average Time: {pandas_time * 1000:.2f} ms")
# --- ConnectorX ---
print("\nRunning ConnectorX warmup...")
@@ -97,15 +98,15 @@ print(f"UncheckedIO Average Time: {unchecked_io_time * 1000:.2f} ms")
# --- 5. Print Results ---
print("\n" + "---" * 10)
print("--- Benchmark Results (5,000,000 Rows) ---")
print(f"Pandas: {pandas_time * 1000:>10.2f} ms")
print("--- Benchmark Results (20,000,000 Rows) ---")
# print(f"Pandas: {pandas_time * 1000:>10.2f} ms")
print(f"ConnectorX: {connectorx_time * 1000:>10.2f} ms")
print(f"UncheckedIO: {unchecked_io_time * 1000:>10.2f} ms")
print("---" * 10)
print("\n--- Ratios ---")
if unchecked_io_time > 0:
print(f"UncheckedIO is {pandas_time / unchecked_io_time:.2f}x faster than Pandas")
# print(f"UncheckedIO is {pandas_time / unchecked_io_time:.2f}x faster than Pandas")
print(f"UncheckedIO is {connectorx_time / unchecked_io_time:.2f}x faster than ConnectorX")
else:
print("UncheckedIO was too fast to measure accurately!")