profiler set up! cpu starvation identified

Cargo.lock

@@ -1723,9 +1723,9 @@ dependencies = [
 
 [[package]]
 name = "tracing-tracy"
-version = "0.11.4"
+version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0eaa1852afa96e0fe9e44caa53dc0bd2d9d05e0f2611ce09f97f8677af56e4ba"
+checksum = "c6a90519f16f55e5c62ffd5976349f10744435a919ecff83d918300575dfb69b"
 dependencies = [
  "tracing-core",
  "tracing-subscriber",
@@ -1734,9 +1734,9 @@ dependencies = [
 
 [[package]]
 name = "tracy-client"
-version = "0.18.3"
+version = "0.17.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91d722a05fe49b31fef971c4732a7d4aa6a18283d9ba46abddab35f484872947"
+checksum = "73202d787346a5418f8222eddb5a00f29ea47caf3c7d38a8f2f69f8455fa7c7e"
 dependencies = [
  "loom",
  "once_cell",
@@ -1745,9 +1745,9 @@ dependencies = [
 
 [[package]]
 name = "tracy-client-sys"
-version = "0.27.0"
+version = "0.24.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2fb391ac70462b3097a755618fbf9c8f95ecc1eb379a414f7b46f202ed10db1f"
+checksum = "69fff37da548239c3bf9e64a12193d261e8b22b660991c6fd2df057c168f435f"
 dependencies = [
  "cc",
  "windows-targets 0.52.6",

Cargo.toml

@@ -55,7 +55,7 @@ pyo3-arrow = "0.15.0"
 num_cpus = "1.16"
 
 tracing-subscriber = { version = "0.3", features = ["registry", "env-filter"] }
-tracing-tracy = "0.11.4"
+tracing-tracy = "=0.11.2"
 tracing = "0.1.41"
 
 [target.'cfg(not(target_env = "msvc"))'.dependencies]

src/lib.rs

@@ -23,8 +23,11 @@ use crate::config::{load_and_validate_config, ConnectorConfig};
 // NOTE: run_profiler_logic added, requires definition in parser.rs
 use crate::parser::{run_db_logic, run_profiler_logic};
 
+// NEW: Import the tracing libraries we need for the initialization
 use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::util::SubscriberInitExt;
+
+
 // --- THE PYTHON-CALLABLE ENTRY POINT (Load Data) ---
 
 #[pyfunction]
@@ -91,13 +94,19 @@ fn profile_data(config_path: String) -> PyResult<String> {
 // --- PYTHON MODULE EXPORT ---
 #[pymodule]
 fn unchecked_io(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
+    // --- START NEW CODE ---
+    // This tries to initialize the "Tracy" layer.
+    // The '.try_init()' ensures that if you import the library twice (e.g. in a notebook reload),
+    // it doesn't crash by trying to initialize the global logger a second time.
 
+    // Fix: Use 'default()' instead of 'new()' to avoid argument mismatch errors
     let _ = tracing_subscriber::registry()
         .with(tracing_tracy::TracyLayer::default())
         .try_init();
+    // --- END NEW CODE ---
 
     m.add_function(wrap_pyfunction!(load_data_from_config, m)?)?;
     m.add_function(wrap_pyfunction!(to_pandas_dataframe, m)?)?;
-    m.add_function(wrap_pyfunction!(profile_data, m)?)?; // NEW: Schema profiler
+    m.add_function(wrap_pyfunction!(profile_data, m)?)?;
     Ok(())
 }

src/parser.rs

@@ -151,7 +151,14 @@ pub async fn run_db_logic(config: ConnectorConfig, blast_radius: i64) -> Result<
                 results[index] = Some(batch.1);
             }
             Err(e) => {
-                eprintln!("UncheckedIO: Partition {} failed! Error: {}. Falling back to NULLs (Self-Healing logic required here).", index, e);
+                // ERROR LOGGING
+                // We use tracing::error! here so it shows up brightly in the Tracy log view
+                tracing::error!(
+                    "Partition {} failed! Error: {}. Falling back to NULLs.",
+                    index, e
+                );
+                eprintln!("UncheckedIO: Partition {} failed! Error: {}. Falling back to NULLs.", index, e);
+
                 let null_batch = create_null_batch(arrow_schema.clone(), expected_rows)?;
                 results[index] = Some(null_batch);
             }
@@ -211,7 +218,9 @@ fn create_null_batch(schema: Arc<Schema>, num_rows: usize) -> Result<RecordBatch
 /// Helper function to build the Arrow Schema from the config
 fn build_arrow_schema(config: &ConnectorConfig) -> Result<Schema> {
     let schema_fields: Vec<Field> = config.schema.iter().map(|col_cfg| {
-        let nullable = col_cfg.column_name == "notes";
+        // FIX: Force all columns to be nullable.
+        // This allows our "Self-Healing" logic to insert NULLs if a partition fails.
+        let nullable = true;
 
         let arrow_type = match col_cfg.arrow_type.as_str() {
             "Int64" => DataType::Int64,