Data saving againagain

app.py

@@ -7,6 +7,7 @@ from mini_agents import MasterAgentWrapper
 from utils import get_full_file_path
 from smolagents.memory import ActionStep, PlanningStep, TaskStep, SystemPromptStep, FinalAnswerStep
 from typing import Optional
+import numpy as np
 
 # (Keep Constants as is)
 # --- Constants ---
@@ -143,39 +144,68 @@ def save_dataset_to_hub(df: pd.DataFrame, dataset_name: str) -> tuple[bool, str]
         # Create a copy of the DataFrame to avoid modifying the original
         df_to_save = df.copy()
         
+        def is_none_or_nan(x):
+            """Safely check if a value is None or NaN"""
+            if x is None:
+                return True
+            if isinstance(x, (float, np.floating)) and np.isnan(x):
+                return True
+            if x == "None" or x == "nan" or x == "NaN":
+                return True
+            return False
+        
         def ensure_consistent_type(x, column_name):
             """Ensure consistent type within a column"""
-            if x is None or x == "None":
-                return None
-                
-            # Special handling for model_output_message and similar columns
-            if column_name in ['model_output_message', 'model_input_messages', 'tool_calls']:
+            try:
+                if is_none_or_nan(x):
+                    return None
+                    
+                # Special handling for model_input_messages and similar columns
+                if column_name in ['model_input_messages', 'model_output_message', 'tool_calls']:
+                    if isinstance(x, (list, tuple, np.ndarray)):
+                        # Convert each item in the array/list to string
+                        return str([str(item) if not is_none_or_nan(item) else None for item in x])
+                    if isinstance(x, dict):
+                        return str(x)
+                    if hasattr(x, 'dict'):
+                        return str(x.dict())
+                    if hasattr(x, '__dict__'):
+                        return str(x.__dict__)
+                    return str(x)
+                    
+                # For other columns, convert to string
+                if isinstance(x, (list, tuple, np.ndarray)):
+                    return str(x.tolist() if hasattr(x, 'tolist') else list(x))
                 if isinstance(x, dict):
-                    return str(x)  # Convert dict to string
+                    return str(x)
                 if hasattr(x, 'dict'):
-                    return str(x.dict())  # Convert object with dict() to string
+                    return str(x.dict())
                 if hasattr(x, '__dict__'):
-                    return str(x.__dict__)  # Convert object with __dict__ to string
-                return str(x) if pd.notna(x) else None
-                
-            # For other columns, convert to string
-            if isinstance(x, (list, tuple, dict)):
+                    return str(x.__dict__)
                 return str(x)
-            if hasattr(x, 'dict'):
-                return str(x.dict())
-            if hasattr(x, '__dict__'):
-                return str(x.__dict__)
-            return str(x) if pd.notna(x) else None
+            except Exception as e:
+                print(f"Warning: Error converting value in column {column_name}: {str(e)}")
+                return str(x) if not is_none_or_nan(x) else None
         
         # Convert all columns to consistent types
         for col in df_to_save.columns:
             print(f"Converting column: {col}")
-            df_to_save[col] = df_to_save[col].apply(lambda x: ensure_consistent_type(x, col))
-            
-            # Verify column type consistency
-            sample_values = df_to_save[col].dropna().head()
-            if not sample_values.empty:
-                print(f"Sample values for {col}: {sample_values.iloc[0]}")
+            try:
+                # Handle numpy arrays and pandas series
+                if isinstance(df_to_save[col], (np.ndarray, pd.Series)):
+                    # Convert None/NaN to None, everything else to string
+                    df_to_save[col] = df_to_save[col].apply(lambda x: None if is_none_or_nan(x) else str(x))
+                else:
+                    df_to_save[col] = df_to_save[col].apply(lambda x: ensure_consistent_type(x, col))
+                
+                # Verify column type consistency
+                sample_values = df_to_save[col].dropna().head()
+                if not sample_values.empty:
+                    print(f"Sample values for {col}: {sample_values.iloc[0]}")
+            except Exception as e:
+                print(f"Warning: Error processing column {col}: {str(e)}")
+                # If there's an error, try to convert the entire column to string
+                df_to_save[col] = df_to_save[col].apply(lambda x: None if is_none_or_nan(x) else str(x))
         
         # Convert to dataset
         dataset = datasets.Dataset.from_pandas(df_to_save)