profiler set up! cpu starvation identified

src/lib.rs

@@ -23,8 +23,11 @@ use crate::config::{load_and_validate_config, ConnectorConfig};
 // NOTE: run_profiler_logic added, requires definition in parser.rs
 use crate::parser::{run_db_logic, run_profiler_logic};
 
+// NEW: Import the tracing libraries we need for the initialization
 use tracing_subscriber::layer::SubscriberExt;
 use tracing_subscriber::util::SubscriberInitExt;
+
+
 // --- THE PYTHON-CALLABLE ENTRY POINT (Load Data) ---
 
 #[pyfunction]
@@ -91,13 +94,19 @@ fn profile_data(config_path: String) -> PyResult<String> {
 // --- PYTHON MODULE EXPORT ---
 #[pymodule]
 fn unchecked_io(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
+    // --- START NEW CODE ---
+    // This tries to initialize the "Tracy" layer.
+    // The '.try_init()' ensures that if you import the library twice (e.g. in a notebook reload),
+    // it doesn't crash by trying to initialize the global logger a second time.
 
+    // Fix: Use 'default()' instead of 'new()' to avoid argument mismatch errors
     let _ = tracing_subscriber::registry()
         .with(tracing_tracy::TracyLayer::default())
         .try_init();
+    // --- END NEW CODE ---
 
     m.add_function(wrap_pyfunction!(load_data_from_config, m)?)?;
     m.add_function(wrap_pyfunction!(to_pandas_dataframe, m)?)?;
-    m.add_function(wrap_pyfunction!(profile_data, m)?)?; // NEW: Schema profiler
+    m.add_function(wrap_pyfunction!(profile_data, m)?)?;
     Ok(())
 }

src/parser.rs

@@ -151,7 +151,14 @@ pub async fn run_db_logic(config: ConnectorConfig, blast_radius: i64) -> Result<
                 results[index] = Some(batch.1);
             }
             Err(e) => {
-                eprintln!("UncheckedIO: Partition {} failed! Error: {}. Falling back to NULLs (Self-Healing logic required here).", index, e);
+                // ERROR LOGGING
+                // We use tracing::error! here so it shows up brightly in the Tracy log view
+                tracing::error!(
+                    "Partition {} failed! Error: {}. Falling back to NULLs.",
+                    index, e
+                );
+                eprintln!("UncheckedIO: Partition {} failed! Error: {}. Falling back to NULLs.", index, e);
+
                 let null_batch = create_null_batch(arrow_schema.clone(), expected_rows)?;
                 results[index] = Some(null_batch);
             }
@@ -211,7 +218,9 @@ fn create_null_batch(schema: Arc<Schema>, num_rows: usize) -> Result<RecordBatch
 /// Helper function to build the Arrow Schema from the config
 fn build_arrow_schema(config: &ConnectorConfig) -> Result<Schema> {
     let schema_fields: Vec<Field> = config.schema.iter().map(|col_cfg| {
-        let nullable = col_cfg.column_name == "notes";
+        // FIX: Force all columns to be nullable.
+        // This allows our "Self-Healing" logic to insert NULLs if a partition fails.
+        let nullable = true;
 
         let arrow_type = match col_cfg.arrow_type.as_str() {
             "Int64" => DataType::Int64,