Update app.py

app.py

@@ -3,15 +3,16 @@ import pandas as pd
 import numpy as np
 import os
 import traceback
-from typing import Tuple, Dict, Any, Optional
+from typing import Tuple, Dict, Any, Optional, List
 import tempfile
 import io
 import datetime
+import re
 
 class FeedbackTransformer:
     """
-    A class to transform feedback data with topic and sentiment columns
-    into a binary format where each topic is a separate column.
+    A class to transform feedback data with delimited topic and sentiment columns
+    into binary columns with prefixes T_, S_, and C_.
     """
 
     def __init__(self,
@@ -34,9 +35,14 @@ class FeedbackTransformer:
         self.sentiment_cols = []
         self.category_cols = []
         self.unique_topics = set()
+        self.unique_categories = set()
+        self.unique_sentiments = set()
+        self.topic_sentiment_mapping = {}  # Map topics to their sentiment values
         self.file_name = None
         self.original_filename = None
-        self.selected_columns = []  # Store columns selected for inclusion
+        self.selected_columns = []
+        self.verbatim_column = None  # Store the verbatim/text column
+        self.dynamic_topic_prefix = None  # Store dynamically extracted topic prefix
 
     def load_data(self, file_obj):
         """
@@ -69,6 +75,37 @@ class FeedbackTransformer:
 
         return len(self.data), len(self.data.columns)
 
+    def extract_topic_prefix_from_category(self):
+        """
+        Extract the topic prefix from a column containing "Category:"
+        by finding text between "Category:" and "("
+        """
+        # Look for columns containing "Category:"
+        category_pattern_cols = [col for col in self.data.columns if "Category:" in col]
+
+        if category_pattern_cols:
+            # Use the first matching column
+            category_col = category_pattern_cols[0]
+
+            # Try to extract from column name first
+            match = re.search(r'Category:\s*([^(]+)\s*\(', category_col)
+            if match:
+                extracted_prefix = match.group(1).strip() + ":"
+                self.dynamic_topic_prefix = extracted_prefix
+                return extracted_prefix
+
+            # If not found in column name, try to extract from column values
+            for value in self.data[category_col].dropna():
+                if isinstance(value, str):
+                    match = re.search(r'Category:\s*([^(]+)\s*\(', value)
+                    if match:
+                        extracted_prefix = match.group(1).strip() + ":"
+                        self.dynamic_topic_prefix = extracted_prefix
+                        return extracted_prefix
+
+        # If no match found, return None
+        return None
+
     def identify_columns(self):
         """
         Identify topic, category, and sentiment columns in the data.
@@ -76,11 +113,32 @@ class FeedbackTransformer:
         if self.data is None:
             raise ValueError("Data not loaded")
 
+        # First try to extract topic prefix dynamically
+        extracted_prefix = self.extract_topic_prefix_from_category()
+
+        # Use dynamic prefix if found, otherwise use the provided topic_prefix
+        topic_identifier = extracted_prefix if extracted_prefix else self.topic_prefix
+
+        # Log the prefix being used
+        print(f"Using topic prefix: '{topic_identifier}'")
+
         # Extract columns based on prefixes
-        self.topic_cols = [col for col in self.data.columns if self.topic_prefix in col]
+        # For topic columns, use the dynamic or provided prefix
+        if topic_identifier:
+            self.topic_cols = [col for col in self.data.columns if topic_identifier in col]
+        else:
+            self.topic_cols = [col for col in self.data.columns if "Topic:" in col]
+
         self.sentiment_cols = [col for col in self.data.columns if self.sentiment_prefix in col]
         self.category_cols = [col for col in self.data.columns if col.startswith(self.category_prefix)]
 
+        # Try to identify verbatim/text column
+        text_candidates = [col for col in self.data.columns if any(keyword in col.lower() for keyword in ['text', 'verbatim', 'comment', 'feedback'])]
+        if text_candidates:
+            self.verbatim_column = text_candidates[0]  # Use the first match
+        elif self.text_column in self.data.columns:
+            self.verbatim_column = self.text_column
+
         # If no columns found with specified prefixes, return all columns for manual selection
         all_cols = list(self.data.columns)
 
@@ -88,44 +146,57 @@ class FeedbackTransformer:
             'topic_cols': self.topic_cols,
             'sentiment_cols': self.sentiment_cols,
             'category_cols': self.category_cols,
-            'all_columns': all_cols
+            'all_columns': all_cols,
+            'verbatim_column': self.verbatim_column,
+            'dynamic_topic_prefix': self.dynamic_topic_prefix
         }
 
-    def extract_unique_topics(self):
+    def extract_unique_topics_and_categories(self):
         """
-        Extract all unique topics from the topic columns.
+        Extract all unique topics, categories, and sentiments from the respective columns.
         """
         self.unique_topics = set()
+        self.unique_categories = set()
+        self.unique_sentiments = set()
+        self.topic_sentiment_mapping = {}
 
-        # Extract from topic columns
+        # Extract from topic columns (delimited by |)
         for col in self.topic_cols:
-            self.unique_topics.update(self.data[col].dropna().unique())
+            for value in self.data[col].dropna():
+                if isinstance(value, str) and value.strip():
+                    # Split by | delimiter and clean each topic
+                    topics = [topic.strip() for topic in value.split('|') if topic.strip()]
+                    self.unique_topics.update(topics)
 
-        # Also extract from category columns if they exist
+        # Extract from category columns (delimited by |)
         for col in self.category_cols:
-            self.unique_topics.update(self.data[col].dropna().unique())
-
-        # Remove empty topics
-        self.unique_topics = {t for t in self.unique_topics if isinstance(t, str) and t.strip()}
-
-        return len(self.unique_topics)
-
-    @staticmethod
-    def create_column_name(topic):
-        """
-        Create a standardized column name from a topic string.
-        """
-        # Remove special characters and standardize
-        topic_clean = str(topic).strip()
-        # Remove brackets and special characters
-        topic_clean = topic_clean.replace('[', '').replace(']', '').replace('(', '').replace(')', '')
-        topic_clean = topic_clean.replace('**', '').replace('*', '')
-        topic_clean = topic_clean.replace('.', '_').replace(' ', '_').replace('&', 'and')
-        topic_clean = topic_clean.replace(':', '_').replace('-', '_').replace('/', '_')
-        # Remove multiple underscores
-        while '__' in topic_clean:
-            topic_clean = topic_clean.replace('__', '_')
-        return topic_clean.lower().strip('_')
+            for value in self.data[col].dropna():
+                if isinstance(value, str) and value.strip():
+                    # Split by | delimiter and clean each category
+                    categories = [cat.strip() for cat in value.split('|') if cat.strip()]
+                    self.unique_categories.update(categories)
+
+        # Extract sentiments from sentiment columns and build topic-sentiment mapping
+        for col in self.sentiment_cols:
+            for idx, value in enumerate(self.data[col].dropna()):
+                if isinstance(value, str) and value.strip():
+                    # Split by | delimiter to get individual topic::sentiment pairs
+                    pairs = [pair.strip() for pair in value.split('|') if pair.strip() and '::' in pair]
+                    for pair in pairs:
+                        if '::' in pair:
+                            topic_part, sentiment_part = pair.split('::', 1)
+                            topic = topic_part.strip()
+                            sentiment = sentiment_part.strip()
+                            if topic and sentiment:
+                                self.unique_topics.add(topic)  # Add topic from sentiment data
+                                self.unique_sentiments.add(sentiment)
+
+                                # Store the mapping for later use
+                                if idx not in self.topic_sentiment_mapping:
+                                    self.topic_sentiment_mapping[idx] = {}
+                                self.topic_sentiment_mapping[idx][topic] = sentiment
+
+        return len(self.unique_topics), len(self.unique_categories), len(self.unique_sentiments)
 
     def set_selected_columns(self, selected_columns):
         """
@@ -135,10 +206,10 @@ class FeedbackTransformer:
 
     def transform_data(self):
         """
-        Transform the data into binary topic columns with sentiment values.
+        Transform the data into binary columns with T_, S_, and C_ prefixes.
         """
-        if not self.unique_topics:
-            self.extract_unique_topics()
+        if not self.unique_topics and not self.unique_categories:
+            self.extract_unique_topics_and_categories()
 
         # Create output dataframe starting with feedback_id
         self.transformed_data = pd.DataFrame({'feedback_id': range(1, len(self.data) + 1)})
@@ -148,49 +219,99 @@ class FeedbackTransformer:
             if col in self.data.columns:
                 self.transformed_data[col] = self.data[col]
 
-        # Initialize all topic columns to 0
+        # Add Verbatim sentiment columns
+        self.transformed_data['Verbatim_Positive'] = 0
+        self.transformed_data['Verbatim_Neutral'] = 0
+        self.transformed_data['Verbatim_Negative'] = 0
+
+        # Create binary topic columns with T_ prefix
         for topic in sorted(self.unique_topics):
-            topic_col = self.create_column_name(topic)
-            self.transformed_data[topic_col] = 0
-            self.transformed_data[f'{topic_col}_sentiment'] = None
+            safe_topic_name = self._make_safe_column_name(topic)
+            col_name = f"T_{safe_topic_name}"
+            self.transformed_data[col_name] = 0
 
-        # Fill in the data from topic columns
+        # Create sentiment columns with S_ prefix (one per topic, containing actual sentiment values)
+        for topic in sorted(self.unique_topics):
+            safe_topic_name = self._make_safe_column_name(topic)
+            col_name = f"S_{safe_topic_name}"
+            self.transformed_data[col_name] = ""  # Initialize with empty strings
+
+        # Create binary category columns with C_ prefix
+        for category in sorted(self.unique_categories):
+            safe_category_name = self._make_safe_column_name(category)
+            col_name = f"C_{safe_category_name}"
+            self.transformed_data[col_name] = 0
+
+        # Fill in the data
         for idx, row in self.data.iterrows():
-            # Process topic columns with sentiments
-            for i, t_col in enumerate(self.topic_cols):
-                topic = row.get(t_col)
-
-                # Find corresponding sentiment column
-                if i < len(self.sentiment_cols):
-                    sentiment = row.get(self.sentiment_cols[i])
-                else:
-                    sentiment = None
-
-                if pd.notna(topic) and isinstance(topic, str) and topic.strip():
-                    topic_col = self.create_column_name(topic)
-                    if topic_col in self.transformed_data.columns:
-                        self.transformed_data.loc[idx, topic_col] = 1
-
-                        # Convert sentiment to numeric value
-                        if pd.notna(sentiment) and isinstance(sentiment, str):
-                            sentiment_lower = sentiment.lower()
-                            if 'positive' in sentiment_lower:
-                                self.transformed_data.loc[idx, f'{topic_col}_sentiment'] = 1
-                            elif 'negative' in sentiment_lower:
-                                self.transformed_data.loc[idx, f'{topic_col}_sentiment'] = 0
-                            elif 'neutral' in sentiment_lower:
-                                self.transformed_data.loc[idx, f'{topic_col}_sentiment'] = 0.5
-
-            # Process category columns (these typically don't have sentiments)
+            # Process sentiment columns to determine which topics exist in ABSA column
+            topics_in_absa = set()
+            all_sentiments_in_row = set()  # Track all sentiments for verbatim columns
+
+            for s_col in self.sentiment_cols:
+                sentiment_value = row.get(s_col)
+                if pd.notna(sentiment_value) and isinstance(sentiment_value, str) and sentiment_value.strip():
+                    pairs = [pair.strip() for pair in sentiment_value.split('|') if pair.strip()]
+                    for pair in pairs:
+                        if '::' in pair:
+                            topic_part, sentiment_part = pair.split('::', 1)
+                            topic = topic_part.strip()
+                            sentiment = sentiment_part.strip()
+
+                            if topic and sentiment:
+                                topics_in_absa.add(topic)
+                                all_sentiments_in_row.add(sentiment.lower())  # Store in lowercase for matching
+
+                                # Set the actual sentiment value (not 1/0)
+                                safe_topic_name = self._make_safe_column_name(topic)
+                                sentiment_col_name = f"S_{safe_topic_name}"
+                                if sentiment_col_name in self.transformed_data.columns:
+                                    self.transformed_data.loc[idx, sentiment_col_name] = sentiment
+
+            # Set Verbatim sentiment columns based on sentiments found in ABSA
+            if any(sentiment in all_sentiments_in_row for sentiment in ['positive', 'positiv']):
+                self.transformed_data.loc[idx, 'Verbatim_Positive'] = 1
+            if any(sentiment in all_sentiments_in_row for sentiment in ['neutral']):
+                self.transformed_data.loc[idx, 'Verbatim_Neutral'] = 1
+            if any(sentiment in all_sentiments_in_row for sentiment in ['negative', 'negativ']):
+                self.transformed_data.loc[idx, 'Verbatim_Negative'] = 1
+
+            # Set T_ columns to 1 if topic exists in ABSA column, 0 otherwise
+            for topic in topics_in_absa:
+                safe_topic_name = self._make_safe_column_name(topic)
+                topic_col_name = f"T_{safe_topic_name}"
+                if topic_col_name in self.transformed_data.columns:
+                    self.transformed_data.loc[idx, topic_col_name] = 1
+
+            # Process category columns
+            categories_in_row = set()
             for c_col in self.category_cols:
-                category = row.get(c_col)
-                if pd.notna(category) and isinstance(category, str) and category.strip():
-                    category_col = self.create_column_name(category)
-                    if category_col in self.transformed_data.columns:
-                        self.transformed_data.loc[idx, category_col] = 1
+                category_value = row.get(c_col)
+                if pd.notna(category_value) and isinstance(category_value, str) and category_value.strip():
+                    categories = [cat.strip() for cat in category_value.split('|') if cat.strip()]
+                    categories_in_row.update(categories)
+
+            # Set category binary values (always 1 if present in category column)
+            for category in categories_in_row:
+                safe_category_name = self._make_safe_column_name(category)
+                category_col_name = f"C_{safe_category_name}"
+                if category_col_name in self.transformed_data.columns:
+                    self.transformed_data.loc[idx, category_col_name] = 1
 
         return self.transformed_data.shape
 
+    def _make_safe_column_name(self, name):
+        """
+        Convert a name to a safe column name by removing/replacing problematic characters.
+        """
+        # Replace spaces and special characters with underscores
+        safe_name = re.sub(r'[^\w]', '_', str(name))
+        # Remove multiple consecutive underscores
+        safe_name = re.sub(r'_+', '_', safe_name)
+        # Remove leading/trailing underscores
+        safe_name = safe_name.strip('_')
+        return safe_name
+
     def analyze_data(self):
         """
         Analyze the transformed data to provide insights.
@@ -198,107 +319,117 @@ class FeedbackTransformer:
         if self.transformed_data is None:
             raise ValueError("No transformed data to analyze")
 
-        # Identify topic columns (exclude feedback_id, selected original columns, and sentiment columns)
-        excluded_cols = ['feedback_id'] + self.selected_columns
-        topic_cols = [col for col in self.transformed_data.columns
-                     if col not in excluded_cols and not col.endswith('_sentiment')]
-
-        # Count occurrences of each topic
-        topic_counts = {}
-        for topic in topic_cols:
-            topic_counts[topic] = self.transformed_data[topic].sum()
-
-        # Sort topics by frequency
-        sorted_topics = sorted(topic_counts.items(), key=lambda x: x[1], reverse=True)
+        # Count different types of columns
+        topic_cols = [col for col in self.transformed_data.columns if col.startswith('T_')]
+        sentiment_cols = [col for col in self.transformed_data.columns if col.startswith('S_')]
+        category_cols = [col for col in self.transformed_data.columns if col.startswith('C_')]
+        verbatim_cols = ['Verbatim_Positive', 'Verbatim_Neutral', 'Verbatim_Negative']
+
+        # Calculate statistics
+        topic_stats = {}
+        for col in topic_cols:
+            topic_stats[col] = self.transformed_data[col].sum()
+
+        # For sentiment columns, count non-empty values
+        sentiment_stats = {}
+        for col in sentiment_cols:
+            sentiment_stats[col] = (self.transformed_data[col] != "").sum()
+
+        category_stats = {}
+        for col in category_cols:
+            category_stats[col] = self.transformed_data[col].sum()
+
+        # Verbatim sentiment statistics
+        verbatim_stats = {}
+        for col in verbatim_cols:
+            if col in self.transformed_data.columns:
+                verbatim_stats[col] = self.transformed_data[col].sum()
+
+        # Sort by frequency
+        sorted_topics = sorted(topic_stats.items(), key=lambda x: x[1], reverse=True)
+        sorted_sentiments = sorted(sentiment_stats.items(), key=lambda x: x[1], reverse=True)
+        sorted_categories = sorted(category_stats.items(), key=lambda x: x[1], reverse=True)
+        sorted_verbatim = sorted(verbatim_stats.items(), key=lambda x: x[1], reverse=True)
 
         # Prepare analysis summary
         analysis_text = f"**Analysis Results**\n\n"
         analysis_text += f"Total feedbacks: {len(self.transformed_data)}\n"
         analysis_text += f"Selected original columns: {len(self.selected_columns)}\n"
-        analysis_text += f"Unique topics: {len(topic_cols)}\n\n"
+        analysis_text += f"Verbatim sentiment columns: 3 (Positive, Neutral, Negative)\n"
+        analysis_text += f"Topic columns (T_): {len(topic_cols)}\n"
+        analysis_text += f"Sentiment columns (S_): {len(sentiment_cols)}\n"
+        analysis_text += f"Category columns (C_): {len(category_cols)}\n"
+        analysis_text += f"Verbatim column used: {self.verbatim_column}\n"
+
+        # Add dynamic topic prefix info
+        if self.dynamic_topic_prefix:
+            analysis_text += f"Dynamic topic prefix extracted: '{self.dynamic_topic_prefix}'\n\n"
+        else:
+            analysis_text += f"Topic prefix used: '{self.topic_prefix}'\n\n"
 
         if self.selected_columns:
             analysis_text += f"**Included Original Columns:** {', '.join(self.selected_columns)}\n\n"
 
-        analysis_text += "**Top 10 Most Frequent Topics:**\n"
-        for topic, count in sorted_topics[:10]:
-            analysis_text += f"- {topic}: {count} occurrences\n"
-
-        # Calculate sentiment distributions for top topics
-        analysis_text += "\n**Sentiment Distributions for Top 5 Topics:**\n"
-        for topic, _ in sorted_topics[:5]:
-            sentiment_col = f"{topic}_sentiment"
-            if sentiment_col in self.transformed_data.columns:
-                # Filter rows where the topic is present
-                topic_rows = self.transformed_data[self.transformed_data[topic] == 1]
-
-                positive = (topic_rows[sentiment_col] == 1.0).sum()
-                negative = (topic_rows[sentiment_col] == 0.0).sum()
-                neutral = (topic_rows[sentiment_col] == 0.5).sum()
-
-                total = positive + negative + neutral
-
-                if total > 0:
-                    analysis_text += f"\n{topic} ({total} occurrences):\n"
-                    analysis_text += f"  - Positive: {positive} ({positive/total*100:.1f}%)\n"
-                    analysis_text += f"  - Negative: {negative} ({negative/total*100:.1f}%)\n"
-                    analysis_text += f"  - Neutral: {neutral} ({neutral/total*100:.1f}%)\n"
-
-        # Calculate number of topics per feedback
-        self.transformed_data['topic_count'] = self.transformed_data[topic_cols].sum(axis=1)
-        avg_topics = self.transformed_data['topic_count'].mean()
-        max_topics = self.transformed_data['topic_count'].max()
-
-        analysis_text += f"\n**Topics per Feedback:**\n"
-        analysis_text += f"- Average: {avg_topics:.2f}\n"
-        analysis_text += f"- Maximum: {max_topics}\n"
-
-        # Remove the temporary topic_count column
-        self.transformed_data.drop('topic_count', axis=1, inplace=True)
+        # Verbatim sentiment analysis
+        if sorted_verbatim:
+            analysis_text += "**Verbatim Sentiment Distribution:**\n"
+            for verbatim_col, count in sorted_verbatim:
+                percentage = (count / len(self.transformed_data)) * 100
+                analysis_text += f"- {verbatim_col}: {count} occurrences ({percentage:.1f}%)\n"
+
+        # Topic analysis
+        if sorted_topics:
+            analysis_text += "\n**Top 10 Most Frequent Topics (T_):**\n"
+            for topic_col, count in sorted_topics[:10]:
+                analysis_text += f"- {topic_col}: {count} occurrences\n"
+
+        # Category analysis
+        if sorted_categories:
+            analysis_text += "\n**Top 10 Most Frequent Categories (C_):**\n"
+            for category_col, count in sorted_categories[:10]:
+                analysis_text += f"- {category_col}: {count} occurrences\n"
+
+        # Sentiment analysis
+        if sorted_sentiments:
+            analysis_text += "\n**Top 10 Most Frequent Sentiments (S_):**\n"
+            for sentiment_col, count in sorted_sentiments[:10]:
+                analysis_text += f"- {sentiment_col}: {count} sentiment values\n"
 
         return analysis_text
 
     def save_transformed_data(self, output_format='xlsx'):
         """
         Save the transformed data and return the file path.
-        Modified to work properly with Hugging Face Spaces downloads.
         """
         if self.transformed_data is None:
             raise ValueError("No transformed data to save")
 
         # Create filename with original filename prefix and timestamp
         timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
-        
-        # Use original filename as prefix, or fallback to 'transformed_feedback' if not available
         prefix = self.original_filename if self.original_filename else 'transformed_feedback'
 
         if output_format == 'xlsx':
-            filename = f"{prefix}_transformed_{timestamp}.xlsx"
-            # Create temporary file that Gradio can handle
+            filename = f"{prefix}_transformed_topics_{timestamp}.xlsx"
             temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.xlsx')
             self.transformed_data.to_excel(temp_file.name, index=False)
             temp_file.close()
-            
-            # Rename the temporary file to have a meaningful name
+
             final_path = os.path.join(tempfile.gettempdir(), filename)
             if os.path.exists(final_path):
                 os.remove(final_path)
             os.rename(temp_file.name, final_path)
-            
+
         else:  # csv
-            filename = f"{prefix}_transformed_{timestamp}.csv"
-            # Create temporary file that Gradio can handle
+            filename = f"{prefix}_binary_matrix_{timestamp}.csv"
             temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.csv')
             self.transformed_data.to_csv(temp_file.name, index=False)
             temp_file.close()
-            
-            # Rename the temporary file to have a meaningful name
+
             final_path = os.path.join(tempfile.gettempdir(), filename)
             if os.path.exists(final_path):
                 os.remove(final_path)
             os.rename(temp_file.name, final_path)
 
-        # Verify file was created and is readable
         if not os.path.exists(final_path):
             raise ValueError(f"Failed to create output file: {final_path}")
 
@@ -334,17 +465,14 @@ def get_column_selector(file_obj):
             df = pd.read_csv(file_obj, sep='\t', nrows=5)
 
         columns = list(df.columns)
-        
-        # Create column display with indices for easier reference
         column_choices = [f"{i+1:2d}. {col}" for i, col in enumerate(columns)]
-        
-        # Return updated CheckboxGroup with numbered columns and individual rows
+
         return gr.CheckboxGroup(
             choices=column_choices,
-            value=[],  # No columns selected by default
+            value=[],
             label=f"📋 Select Columns to Include ({len(columns)} available)",
-            info="Choose which original columns to include in the transformed file (in addition to feedback_id). Columns are numbered for easy reference.",
-            elem_classes=["column-selector"]  # Add CSS class for styling
+            info="Choose which original columns to include in the transformed file (in addition to feedback_id).",
+            elem_classes=["column-selector"]
         )
 
     except Exception as e:
@@ -362,16 +490,15 @@ def extract_column_names(selected_display_names):
     """
     if not selected_display_names:
         return []
-    
+
     actual_names = []
     for display_name in selected_display_names:
-        # Remove the number prefix (e.g., "1. Column Name" -> "Column Name")
         if '. ' in display_name:
             actual_name = display_name.split('. ', 1)[1]
             actual_names.append(actual_name)
         else:
             actual_names.append(display_name)
-    
+
     return actual_names
 
 
@@ -383,7 +510,7 @@ def process_file(file_obj, topic_prefix, sentiment_prefix, category_prefix,
     try:
         # Extract actual column names from display format
         actual_column_names = extract_column_names(selected_columns)
-        
+
         # Initialize transformer
         transformer = FeedbackTransformer(
             topic_prefix=topic_prefix,
@@ -409,14 +536,26 @@ def process_file(file_obj, topic_prefix, sentiment_prefix, category_prefix,
         status_msg += f"- Topic columns: {len(col_info['topic_cols'])}\n"
         status_msg += f"- Sentiment columns: {len(col_info['sentiment_cols'])}\n"
         status_msg += f"- Category columns: {len(col_info['category_cols'])}\n"
+        status_msg += f"- Verbatim column: {col_info['verbatim_column']}\n"
+
+        # Add dynamic topic prefix info
+        if col_info.get('dynamic_topic_prefix'):
+            status_msg += f"- Dynamic topic prefix extracted: '{col_info['dynamic_topic_prefix']}'\n"
 
-        # Extract unique topics
-        num_topics = transformer.extract_unique_topics()
+        # Extract unique topics, categories, and sentiments
+        num_topics, num_categories, num_sentiments = transformer.extract_unique_topics_and_categories()
         status_msg += f"\n🎯 Found {num_topics} unique topics\n"
+        status_msg += f"🏷️ Found {num_categories} unique categories\n"
+        status_msg += f"💭 Found {num_sentiments} unique sentiments\n"
 
         # Transform data
         shape = transformer.transform_data()
         status_msg += f"\n✨ Transformed data shape: {shape[0]} rows × {shape[1]} columns\n"
+        status_msg += f"📊 Binary matrix created with T_, S_, C_ prefixes and Verbatim sentiment columns\n"
+        status_msg += f"🔧 T_ columns: 1 if topic present in ABSA column, 0 otherwise\n"
+        status_msg += f"🔧 S_ columns: contain actual sentiment values (not 1/0)\n"
+        status_msg += f"🔧 C_ columns: 1 if category assigned, 0 otherwise\n"
+        status_msg += f"🔧 Verbatim_Positive/Neutral/Negative: 1 if respective sentiment found in ABSA, 0 otherwise\n"
 
         # Analyze if requested
         analysis_result = ""
@@ -426,6 +565,7 @@ def process_file(file_obj, topic_prefix, sentiment_prefix, category_prefix,
         # Save transformed data
         output_file = transformer.save_transformed_data(output_format)
         status_msg += f"\n💾 File saved successfully: {os.path.basename(output_file)}\n"
+        #status_msg += f"📥 File download should start automatically\n"
 
         return status_msg, analysis_result, output_file
 
@@ -435,7 +575,7 @@ def process_file(file_obj, topic_prefix, sentiment_prefix, category_prefix,
 
 
 # Create Gradio interface
-with gr.Blocks(title="Feedback Topic & Sentiment Transformer", css="""
+with gr.Blocks(title="Binary Matrix Feedback Transformer", css="""
 .column-selector .form-check {
     display: block !important;
     margin-bottom: 8px !important;
@@ -445,15 +585,26 @@ with gr.Blocks(title="Feedback Topic & Sentiment Transformer", css="""
 }
 """) as demo:
     gr.Markdown("""
-    # 📊 Feedback Topic & Sentiment Transformer
-    Transform feedback data with topic and sentiment columns into a binary matrix format.
-    Each unique topic becomes a separate column with 0/1 values and associated sentiment scores.
-    ### 📋 Instructions:
-    1. Upload your Excel, CSV, or tab-delimited text file
-    2. Select which original columns to include in the output
-    3. Configure column prefixes (or use defaults)
-    4. Click "Transform Data" to process
-    5. Download the transformed file
+    # 📊 Binary Matrix Feedback Transformer
+    Transform feedback data with delimited topic and sentiment columns into binary matrix format.
+
+    ### 🔧 Processing Logic:
+    - **Automatic Topic Prefix Detection**: Extracts topic prefix from columns containing "Category:" by finding text between "Category:" and "("
+    - **Verbatim_Positive/Neutral/Negative**: Set to 1 if respective sentiment is found in ABSA column, 0 otherwise
+    - **T_ Columns**: Set to 1 if topic is present in ABSA column, 0 otherwise
+    - **S_ Columns**: One column per topic (e.g., S_Allgemeine_Zufriedenheit) containing actual sentiment values
+    - **C_ Columns**: Set to 1 if category is assigned, 0 otherwise
+
+    ### 📋 Data Format Requirements:
+    - **Topics**: Delimited by `|` (pipe) in columns identified by dynamic or manual prefix
+    - **Sentiments**: Format `Topic::Sentiment|Topic2::Sentiment2` in ABSA columns
+    - **Categories**: Delimited by `|` (pipe) in "Categories:" columns
+
+    ### 🆕 Key Features:
+    - **Dynamic Topic Prefix Extraction**: Automatically extracts topic prefix from "Category:" columns
+    - **Verbatim_** columns detect overall sentiment presence regardless of topic
+    - **T_** columns based on ABSA column presence (topics that have sentiment data)
+    - **S_** columns contain actual sentiment values (not binary 1/0)
     """)
 
     with gr.Row():
@@ -466,7 +617,7 @@ with gr.Blocks(title="Feedback Topic & Sentiment Transformer", css="""
                 type="filepath"
             )
 
-            # Combined column selector (replaces both preview and checkboxes)
+            # Combined column selector
             gr.Markdown("### 📋 2. Column Selection")
             column_selector = gr.CheckboxGroup(
                 choices=[],
@@ -477,18 +628,18 @@ with gr.Blocks(title="Feedback Topic & Sentiment Transformer", css="""
 
         with gr.Column(scale=1):
             # Configuration parameters
-            gr.Markdown("### ⚙️ 3. Configuration of column prefixes ")
+            gr.Markdown("### ⚙️ 3. Configuration")
 
             topic_prefix = gr.Textbox(
-                label="Topic Column Prefix",
-                value="[**WORKSHOP] SwissLife Taxonomy",
-                info="Prefix to identify topic columns"
+                label="Topic Column Identifier (Fallback)",
+                value="Topic:",
+                info="Fallback identifier if dynamic extraction from Category: column fails"
             )
 
             sentiment_prefix = gr.Textbox(
-                label="Sentiment Column Prefix",
+                label="Sentiment Column Prefix (ABSA)",
                 value="ABSA:",
-                info="Prefix to identify sentiment columns"
+                info="Prefix to identify sentiment columns (format: Topic::Sentiment)"
             )
 
             category_prefix = gr.Textbox(
@@ -498,9 +649,9 @@ with gr.Blocks(title="Feedback Topic & Sentiment Transformer", css="""
             )
 
             text_column = gr.Textbox(
-                label="Text Column Name",
+                label="Text/Verbatim Column Pattern",
                 value="TEXT",
-                info="Column containing original feedback text (for reference only)"
+                info="Pattern to identify verbatim text column (for reference only)"
             )
 
             recommendation_column = gr.Textbox(
@@ -521,14 +672,14 @@ with gr.Blocks(title="Feedback Topic & Sentiment Transformer", css="""
             )
 
     # Transform button
-    transform_btn = gr.Button("🔄 4. Transform Data", variant="primary", size="lg")
+    transform_btn = gr.Button("🔄 4. Transform to Binary Matrix & Download", variant="primary", size="lg")
 
     # Output sections
     with gr.Row():
         with gr.Column():
             status_output = gr.Textbox(
                 label="Processing Status",
-                lines=10,
+                lines=12,
                 interactive=False
             )
 
@@ -537,12 +688,13 @@ with gr.Blocks(title="Feedback Topic & Sentiment Transformer", css="""
                 label="Data Analysis"
             )
 
-    # Download section - Modified for better download functionality
+    # Download section
     with gr.Row():
         with gr.Column():
-            gr.Markdown("### 📥 5. Download Transformed File")
+            gr.Markdown("### 📥 Download Status")
+            gr.Markdown("Please click on the link inside the output file size value to download the transformed file (the number value on the right hand side below). You may need to right click and select Save Link As (or something similar)")
             output_file = gr.File(
-                label="Transformed File",
+                label="Transformed Binary Matrix (Auto-Download)",
                 interactive=False,
                 visible=True
             )
@@ -572,26 +724,38 @@ with gr.Blocks(title="Feedback Topic & Sentiment Transformer", css="""
 
     # Examples section
     gr.Markdown("""
-    ### 📝 Example Column Formats:
-    - **Topic columns**: `[**WORKSHOP] SwissLife Taxonomy(Kommentar) 1`, `[**WORKSHOP] SwissLife Taxonomy(Kommentar) 2`
-    - **Category columns**: `Categories:Topic1`, `Categories:Topic2`
-    - **Sentiment columns**: `ABSA:Sentiment1`, `ABSA:Sentiment2`
-    ### 🎯 Output Format:
-    - **feedback_id**: Unique identifier for each row
-    - **Selected original columns**: Any columns you selected from the original file
-    - **Topic columns**: Each unique topic becomes a column with values 0 (absent) or 1 (present)
-    - **Sentiment columns**: Each topic has an associated `_sentiment` column with values:
-      - 1.0 = Positive
-      - 0.5 = Neutral
-      - 0.0 = Negative
-    - **Output filename**: `[original_filename]_transformed_[timestamp].[format]`
-    ### 💡 Tips:
-    - Use the numbered column list to easily identify and select columns
-    - The text and recommendation column names in configuration are now for reference only
-    - To include them in output, select them using the column checkboxes
-    - Click on the download button that appears after processing to download the file
+    ### 📝 Example Transformations:
+
+    **Input Data with Dynamic Topic Extraction:**
+    ```
+    | Column: "Category: Service (ABC)" | ABSA: Sentiments | Categories: Issues |
+    | 1 | Service::Negative|Quality::Positive | Issues|Support |
+    ```
+
+    **System will:**
+    1. Extract "Service:" from "Category: Service (ABC)" column
+    2. Use "Service:" to identify topic columns instead of "Topic:"
+
+    **Output Binary Matrix:**
+    ```
+    | feedback_id | Verbatim_Positive | Verbatim_Neutral | Verbatim_Negative | T_Service | T_Quality | S_Service | S_Quality | C_Issues | C_Support |
+    | 1 | 1 | 0 | 1 | 1 | 1 | Negative | Positive | 1 | 1 |
+    ```
+
+    ### 💡 Dynamic Topic Prefix Logic:
+    - Searches for columns containing "Category:"
+    - Extracts text between "Category:" and "(" (e.g., "Service" from "Category: Service (ABC)")
+    - Adds ":" to create the topic prefix (e.g., "Service:")
+    - Uses this prefix to identify topic columns
+    - Falls back to manual "Topic Column Identifier" if extraction fails
+
+    ### 🔍 Key Changes in This Version:
+    - **NEW**: Automatic extraction of topic prefix from Category columns
+    - Dynamically identifies topic columns based on extracted prefix
+    - Maintains all other functionality (Verbatim columns, T_, S_, C_ logic)
+    - Provides fallback to manual topic prefix if extraction fails
     """)
 
 # Launch the app
 if __name__ == "__main__":
-    demo.launch()
+    demo.launch(share=True)