Spaces:
Sleeping
Update app.py
Browse filesimport gradio as gr
import pandas as pd
import numpy as np
import os
import traceback
from typing import Tuple, Dict, Any, Optional
import tempfile
import io
import datetime
class FeedbackTransformer:
"""
A class to transform feedback data with topic and sentiment columns
into a binary format where each topic is a separate column.
"""
def __init__(self,
topic_prefix="TOPIC_",
sentiment_prefix="SENTIMENT_",
category_prefix="Categories:",
text_column="TEXT",
recommendation_column="Q4_Weiterempfehlung"):
"""
Initialize the FeedbackTransformer with column specifications.
"""
self.topic_prefix = topic_prefix
self.sentiment_prefix = sentiment_prefix
self.category_prefix = category_prefix
self.text_column = text_column
self.recommendation_column = recommendation_column
self.data = None
self.transformed_data = None
self.topic_cols = []
self.sentiment_cols = []
self.category_cols = []
self.unique_topics = set()
self.file_name = None
self.original_filename = None
self.selected_columns = [] # Store columns selected for inclusion
def load_data(self, file_obj):
"""
Load data from the uploaded file object.
"""
if file_obj is None:
raise ValueError("No file uploaded")
# Get file extension and store original filename
file_name = file_obj if isinstance(file_obj, str) else (file_obj.name if hasattr(file_obj, 'name') else 'unknown')
self.original_filename = os.path.splitext(os.path.basename(file_name))[0]
_, file_ext = os.path.splitext(file_name)
# Read the data based on file type
try:
if file_ext.lower() in ['.xlsx', '.xls']:
self.data = pd.read_excel(file_obj)
elif file_ext.lower() == '.csv':
# Try comma delimiter first
try:
self.data = pd.read_csv(file_obj, encoding='utf-8')
except:
# If comma fails, try tab delimiter
self.data = pd.read_csv(file_obj, sep='\t', encoding='utf-8')
else:
# Default to tab-delimited
self.data = pd.read_csv(file_obj, sep='\t', encoding='utf-8')
except Exception as e:
raise ValueError(f"Error reading file: {str(e)}")
return len(self.data), len(self.data.columns)
def identify_columns(self):
"""
Identify topic, category, and sentiment columns in the data.
"""
if self.data is None:
raise ValueError("Data not loaded")
# Extract columns based on prefixes
self.topic_cols = [col for col in self.data.columns if self.topic_prefix in col]
self.sentiment_cols = [col for col in self.data.columns if self.sentiment_prefix in col]
self.category_cols = [col for col in self.data.columns if col.startswith(self.category_prefix)]
# If no columns found with specified prefixes, return all columns for manual selection
all_cols = list(self.data.columns)
return {
'topic_cols': self.topic_cols,
'sentiment_cols': self.sentiment_cols,
'category_cols': self.category_cols,
'all_columns': all_cols
}
def extract_unique_topics(self):
"""
Extract all unique topics from the topic columns.
"""
self.unique_topics = set()
# Extract from topic columns
for col in self.topic_cols:
self.unique_topics.update(self.data[col].dropna().unique())
# Also extract from category columns if they exist
for col in self.category_cols:
self.unique_topics.update(self.data[col].dropna().unique())
# Remove empty topics
self.unique_topics = {t for t in self.unique_topics if isinstance(t, str) and t.strip()}
return len(self.unique_topics)
@staticmethod
def create_column_name(topic):
"""
Create a standardized column name from a topic string.
"""
# Remove special characters and standardize
topic_clean = str(topic).strip()
# Remove brackets and special characters
topic_clean = topic_clean.replace('[', '').replace(']', '').replace('(', '').replace(')', '')
topic_clean = topic_clean.replace('**', '').replace('*', '')
topic_clean = topic_clean.replace('.', '_').replace(' ', '_').replace('&', 'and')
topic_clean = topic_clean.replace(':', '_').replace('-', '_').replace('/', '_')
# Remove multiple underscores
while '__' in topic_clean:
topic_clean = topic_clean.replace('__', '_')
return topic_clean.lower().strip('_')
def set_selected_columns(self, selected_columns):
"""
Set which original columns should be included in the output.
"""
self.selected_columns = selected_columns if selected_columns else []
def transform_data(self):
"""
Transform the data into binary topic columns with sentiment values.
"""
if not self.unique_topics:
self.extract_unique_topics()
# Create output dataframe starting with feedback_id
self.transformed_data = pd.DataFrame({'feedback_id': range(1, len(self.data) + 1)})
# Add selected original columns first (right after feedback_id)
for col in self.selected_columns:
if col in self.data.columns:
self.transformed_data[col] = self.data[col]
# Initialize all topic columns to 0
for topic in sorted(self.unique_topics):
topic_col = self.create_column_name(topic)
self.transformed_data[topic_col] = 0
self.transformed_data[f'{topic_col}_sentiment'] = None
# Fill in the data from topic columns
for idx, row in self.data.iterrows():
# Process topic columns with sentiments
for i, t_col in enumerate(self.topic_cols):
topic = row.get(t_col)
# Find corresponding sentiment column
if i < len(self.sentiment_cols):
sentiment = row.get(self.sentiment_cols[i])
else:
sentiment = None
if pd.notna(topic) and isinstance(topic, str) and topic.strip():
topic_col = self.create_column_name(topic)
if topic_col in self.transformed_data.columns:
self.transformed_data.loc[idx, topic_col] = 1
# Convert sentiment to numeric value
if pd.notna(sentiment) and isinstance(sentiment, str):
sentiment_lower = sentiment.lower()
if 'positive' in sentiment_lower:
self.transformed_data.loc[idx, f'{topic_col}_sentiment'] = 1
elif 'negative' in sentiment_lower:
self.transformed_data.loc[idx, f'{topic_col}_sentiment'] = 0
elif 'neutral' in sentiment_lower:
self.transformed_data.loc[idx, f'{topic_col}_sentiment'] = 0.5
# Process category columns (these typically don't have sentiments)
for c_col in self.category_cols:
category = row.get(c_col)
if pd.notna(category) and isinstance(category, str) and category.strip():
category_col = self.create_column_name(category)
if category_col in self.transformed_data.columns:
self.transformed_data.loc[idx, category_col] = 1
return self.transformed_data.shape
def analyze_data(self):
"""
Analyze the transformed data to provide insights.
"""
if self.transformed_data is None:
raise ValueError("No transformed data to analyze")
# Identify topic columns (exclude feedback_id, selected original columns, and sentiment columns)
excluded_cols = ['feedback_id'] + self.selected_columns
topic_cols = [col for col in self.transformed_data.columns
if col not in excluded_cols and not col.endswith('_sentiment')]
# Count occurrences of each topic
topic_counts = {}
for topic in topic_cols:
topic_counts[topic] = self.transformed_data[topic].sum()
# Sort topics by frequency
sorted_topics = sorted(topic_counts.items(), key=lambda x: x[1], reverse=True)
# Prepare analysis summary
analysis_text = f"**Analysis Results**\n\n"
analysis_text += f"Total feedbacks: {len(self.transformed_data)}\n"
analysis_text += f"Selected original columns: {len(self.selected_columns)}\n"
analysis_text += f"Unique topics: {len(topic_cols)}\n\n"
if self.selected_columns:
analysis_text += f"**Included Original Columns:** {', '.join(self.selected_columns)}\n\n"
analysis_text += "**Top 10 Most Frequent Topics:**\n"
for topic, count in sorted_topics[:10]:
analysis_text += f"- {topic}: {count} occurrences\n"
# Calculate sentiment distributions for top topics
analysis_text += "\n**Sentiment Distributions for Top 5 Topics:**\n"
for topic, _ in sorted_topics[:5]:
sentiment_col = f"{topic}_sentiment"
if sentiment_col in self.transformed_data.columns:
# Filter rows where the topic is present
topic_rows = self.transformed_data[self.transformed_data[topic] == 1]
positive = (topic_rows[sentiment_col] == 1.0).sum()
negative = (topic_rows[sentiment_col] == 0.0).sum()
neutral = (topic_rows[sentiment_col] == 0.5).sum()
total = positive + negative + neutral
if total > 0:
analysis_text += f"\n{topic} ({total} occurrences):\n"
analysis_text += f" - Positive: {pos
@@ -1,592 +0,0 @@
|
|
1 |
-
import gradio as gr
|
2 |
-
import pandas as pd
|
3 |
-
import numpy as np
|
4 |
-
import os
|
5 |
-
import traceback
|
6 |
-
from typing import Tuple, Dict, Any, Optional
|
7 |
-
import tempfile
|
8 |
-
import io
|
9 |
-
|
10 |
-
class FeedbackTransformer:
|
11 |
-
"""
|
12 |
-
A class to transform feedback data with topic and sentiment columns
|
13 |
-
into a binary format where each topic is a separate column.
|
14 |
-
"""
|
15 |
-
|
16 |
-
def __init__(self,
|
17 |
-
topic_prefix="TOPIC_",
|
18 |
-
sentiment_prefix="SENTIMENT_",
|
19 |
-
category_prefix="Categories:",
|
20 |
-
text_column="TEXT",
|
21 |
-
recommendation_column="Q4_Weiterempfehlung"):
|
22 |
-
"""
|
23 |
-
Initialize the FeedbackTransformer with column specifications.
|
24 |
-
"""
|
25 |
-
self.topic_prefix = topic_prefix
|
26 |
-
self.sentiment_prefix = sentiment_prefix
|
27 |
-
self.category_prefix = category_prefix
|
28 |
-
self.text_column = text_column
|
29 |
-
self.recommendation_column = recommendation_column
|
30 |
-
self.data = None
|
31 |
-
self.transformed_data = None
|
32 |
-
self.topic_cols = []
|
33 |
-
self.sentiment_cols = []
|
34 |
-
self.category_cols = []
|
35 |
-
self.unique_topics = set()
|
36 |
-
self.file_name = None
|
37 |
-
self.original_filename = None
|
38 |
-
self.selected_columns = [] # Store columns selected for inclusion
|
39 |
-
|
40 |
-
def load_data(self, file_obj):
|
41 |
-
"""
|
42 |
-
Load data from the uploaded file object.
|
43 |
-
"""
|
44 |
-
if file_obj is None:
|
45 |
-
raise ValueError("No file uploaded")
|
46 |
-
|
47 |
-
# Get file extension and store original filename
|
48 |
-
file_name = file_obj if isinstance(file_obj, str) else (file_obj.name if hasattr(file_obj, 'name') else 'unknown')
|
49 |
-
self.original_filename = os.path.splitext(os.path.basename(file_name))[0]
|
50 |
-
_, file_ext = os.path.splitext(file_name)
|
51 |
-
|
52 |
-
# Read the data based on file type
|
53 |
-
try:
|
54 |
-
if file_ext.lower() in ['.xlsx', '.xls']:
|
55 |
-
self.data = pd.read_excel(file_obj)
|
56 |
-
elif file_ext.lower() == '.csv':
|
57 |
-
# Try comma delimiter first
|
58 |
-
try:
|
59 |
-
self.data = pd.read_csv(file_obj, encoding='utf-8')
|
60 |
-
except:
|
61 |
-
# If comma fails, try tab delimiter
|
62 |
-
self.data = pd.read_csv(file_obj, sep='\t', encoding='utf-8')
|
63 |
-
else:
|
64 |
-
# Default to tab-delimited
|
65 |
-
self.data = pd.read_csv(file_obj, sep='\t', encoding='utf-8')
|
66 |
-
except Exception as e:
|
67 |
-
raise ValueError(f"Error reading file: {str(e)}")
|
68 |
-
|
69 |
-
return len(self.data), len(self.data.columns)
|
70 |
-
|
71 |
-
def identify_columns(self):
|
72 |
-
"""
|
73 |
-
Identify topic, category, and sentiment columns in the data.
|
74 |
-
"""
|
75 |
-
if self.data is None:
|
76 |
-
raise ValueError("Data not loaded")
|
77 |
-
|
78 |
-
# Extract columns based on prefixes
|
79 |
-
self.topic_cols = [col for col in self.data.columns if self.topic_prefix in col]
|
80 |
-
self.sentiment_cols = [col for col in self.data.columns if self.sentiment_prefix in col]
|
81 |
-
self.category_cols = [col for col in self.data.columns if col.startswith(self.category_prefix)]
|
82 |
-
|
83 |
-
# If no columns found with specified prefixes, return all columns for manual selection
|
84 |
-
all_cols = list(self.data.columns)
|
85 |
-
|
86 |
-
return {
|
87 |
-
'topic_cols': self.topic_cols,
|
88 |
-
'sentiment_cols': self.sentiment_cols,
|
89 |
-
'category_cols': self.category_cols,
|
90 |
-
'all_columns': all_cols
|
91 |
-
}
|
92 |
-
|
93 |
-
def extract_unique_topics(self):
|
94 |
-
"""
|
95 |
-
Extract all unique topics from the topic columns.
|
96 |
-
"""
|
97 |
-
self.unique_topics = set()
|
98 |
-
|
99 |
-
# Extract from topic columns
|
100 |
-
for col in self.topic_cols:
|
101 |
-
self.unique_topics.update(self.data[col].dropna().unique())
|
102 |
-
|
103 |
-
# Also extract from category columns if they exist
|
104 |
-
for col in self.category_cols:
|
105 |
-
self.unique_topics.update(self.data[col].dropna().unique())
|
106 |
-
|
107 |
-
# Remove empty topics
|
108 |
-
self.unique_topics = {t for t in self.unique_topics if isinstance(t, str) and t.strip()}
|
109 |
-
|
110 |
-
return len(self.unique_topics)
|
111 |
-
|
112 |
-
@staticmethod
|
113 |
-
def create_column_name(topic):
|
114 |
-
"""
|
115 |
-
Create a standardized column name from a topic string.
|
116 |
-
"""
|
117 |
-
# Remove special characters and standardize
|
118 |
-
topic_clean = str(topic).strip()
|
119 |
-
# Remove brackets and special characters
|
120 |
-
topic_clean = topic_clean.replace('[', '').replace(']', '').replace('(', '').replace(')', '')
|
121 |
-
topic_clean = topic_clean.replace('**', '').replace('*', '')
|
122 |
-
topic_clean = topic_clean.replace('.', '_').replace(' ', '_').replace('&', 'and')
|
123 |
-
topic_clean = topic_clean.replace(':', '_').replace('-', '_').replace('/', '_')
|
124 |
-
# Remove multiple underscores
|
125 |
-
while '__' in topic_clean:
|
126 |
-
topic_clean = topic_clean.replace('__', '_')
|
127 |
-
return topic_clean.lower().strip('_')
|
128 |
-
|
129 |
-
def set_selected_columns(self, selected_columns):
|
130 |
-
"""
|
131 |
-
Set which original columns should be included in the output.
|
132 |
-
"""
|
133 |
-
self.selected_columns = selected_columns if selected_columns else []
|
134 |
-
|
135 |
-
def transform_data(self):
|
136 |
-
"""
|
137 |
-
Transform the data into binary topic columns with sentiment values.
|
138 |
-
"""
|
139 |
-
if not self.unique_topics:
|
140 |
-
self.extract_unique_topics()
|
141 |
-
|
142 |
-
# Create output dataframe starting with feedback_id
|
143 |
-
self.transformed_data = pd.DataFrame({'feedback_id': range(1, len(self.data) + 1)})
|
144 |
-
|
145 |
-
# Add selected original columns first (right after feedback_id)
|
146 |
-
for col in self.selected_columns:
|
147 |
-
if col in self.data.columns:
|
148 |
-
self.transformed_data[col] = self.data[col]
|
149 |
-
|
150 |
-
# Initialize all topic columns to 0
|
151 |
-
for topic in sorted(self.unique_topics):
|
152 |
-
topic_col = self.create_column_name(topic)
|
153 |
-
self.transformed_data[topic_col] = 0
|
154 |
-
self.transformed_data[f'{topic_col}_sentiment'] = None
|
155 |
-
|
156 |
-
# Fill in the data from topic columns
|
157 |
-
for idx, row in self.data.iterrows():
|
158 |
-
# Process topic columns with sentiments
|
159 |
-
for i, t_col in enumerate(self.topic_cols):
|
160 |
-
topic = row.get(t_col)
|
161 |
-
|
162 |
-
# Find corresponding sentiment column
|
163 |
-
if i < len(self.sentiment_cols):
|
164 |
-
sentiment = row.get(self.sentiment_cols[i])
|
165 |
-
else:
|
166 |
-
sentiment = None
|
167 |
-
|
168 |
-
if pd.notna(topic) and isinstance(topic, str) and topic.strip():
|
169 |
-
topic_col = self.create_column_name(topic)
|
170 |
-
if topic_col in self.transformed_data.columns:
|
171 |
-
self.transformed_data.loc[idx, topic_col] = 1
|
172 |
-
|
173 |
-
# Convert sentiment to numeric value
|
174 |
-
if pd.notna(sentiment) and isinstance(sentiment, str):
|
175 |
-
sentiment_lower = sentiment.lower()
|
176 |
-
if 'positive' in sentiment_lower:
|
177 |
-
self.transformed_data.loc[idx, f'{topic_col}_sentiment'] = 1
|
178 |
-
elif 'negative' in sentiment_lower:
|
179 |
-
self.transformed_data.loc[idx, f'{topic_col}_sentiment'] = 0
|
180 |
-
elif 'neutral' in sentiment_lower:
|
181 |
-
self.transformed_data.loc[idx, f'{topic_col}_sentiment'] = 0.5
|
182 |
-
|
183 |
-
# Process category columns (these typically don't have sentiments)
|
184 |
-
for c_col in self.category_cols:
|
185 |
-
category = row.get(c_col)
|
186 |
-
if pd.notna(category) and isinstance(category, str) and category.strip():
|
187 |
-
category_col = self.create_column_name(category)
|
188 |
-
if category_col in self.transformed_data.columns:
|
189 |
-
self.transformed_data.loc[idx, category_col] = 1
|
190 |
-
|
191 |
-
# Note: We no longer automatically add text and recommendation columns here
|
192 |
-
# as they should be selected through the column selection interface
|
193 |
-
|
194 |
-
return self.transformed_data.shape
|
195 |
-
|
196 |
-
def analyze_data(self):
|
197 |
-
"""
|
198 |
-
Analyze the transformed data to provide insights.
|
199 |
-
"""
|
200 |
-
if self.transformed_data is None:
|
201 |
-
raise ValueError("No transformed data to analyze")
|
202 |
-
|
203 |
-
# Identify topic columns (exclude feedback_id, selected original columns, and sentiment columns)
|
204 |
-
excluded_cols = ['feedback_id'] + self.selected_columns
|
205 |
-
topic_cols = [col for col in self.transformed_data.columns
|
206 |
-
if col not in excluded_cols and not col.endswith('_sentiment')]
|
207 |
-
|
208 |
-
# Count occurrences of each topic
|
209 |
-
topic_counts = {}
|
210 |
-
for topic in topic_cols:
|
211 |
-
topic_counts[topic] = self.transformed_data[topic].sum()
|
212 |
-
|
213 |
-
# Sort topics by frequency
|
214 |
-
sorted_topics = sorted(topic_counts.items(), key=lambda x: x[1], reverse=True)
|
215 |
-
|
216 |
-
# Prepare analysis summary
|
217 |
-
analysis_text = f"**Analysis Results**\n\n"
|
218 |
-
analysis_text += f"Total feedbacks: {len(self.transformed_data)}\n"
|
219 |
-
analysis_text += f"Selected original columns: {len(self.selected_columns)}\n"
|
220 |
-
analysis_text += f"Unique topics: {len(topic_cols)}\n\n"
|
221 |
-
|
222 |
-
if self.selected_columns:
|
223 |
-
analysis_text += f"**Included Original Columns:** {', '.join(self.selected_columns)}\n\n"
|
224 |
-
|
225 |
-
analysis_text += "**Top 10 Most Frequent Topics:**\n"
|
226 |
-
for topic, count in sorted_topics[:10]:
|
227 |
-
analysis_text += f"- {topic}: {count} occurrences\n"
|
228 |
-
|
229 |
-
# Calculate sentiment distributions for top topics
|
230 |
-
analysis_text += "\n**Sentiment Distributions for Top 5 Topics:**\n"
|
231 |
-
for topic, _ in sorted_topics[:5]:
|
232 |
-
sentiment_col = f"{topic}_sentiment"
|
233 |
-
if sentiment_col in self.transformed_data.columns:
|
234 |
-
# Filter rows where the topic is present
|
235 |
-
topic_rows = self.transformed_data[self.transformed_data[topic] == 1]
|
236 |
-
|
237 |
-
positive = (topic_rows[sentiment_col] == 1.0).sum()
|
238 |
-
negative = (topic_rows[sentiment_col] == 0.0).sum()
|
239 |
-
neutral = (topic_rows[sentiment_col] == 0.5).sum()
|
240 |
-
|
241 |
-
total = positive + negative + neutral
|
242 |
-
|
243 |
-
if total > 0:
|
244 |
-
analysis_text += f"\n{topic} ({total} occurrences):\n"
|
245 |
-
analysis_text += f" - Positive: {positive} ({positive/total*100:.1f}%)\n"
|
246 |
-
analysis_text += f" - Negative: {negative} ({negative/total*100:.1f}%)\n"
|
247 |
-
analysis_text += f" - Neutral: {neutral} ({neutral/total*100:.1f}%)\n"
|
248 |
-
|
249 |
-
# Calculate number of topics per feedback
|
250 |
-
self.transformed_data['topic_count'] = self.transformed_data[topic_cols].sum(axis=1)
|
251 |
-
avg_topics = self.transformed_data['topic_count'].mean()
|
252 |
-
max_topics = self.transformed_data['topic_count'].max()
|
253 |
-
|
254 |
-
analysis_text += f"\n**Topics per Feedback:**\n"
|
255 |
-
analysis_text += f"- Average: {avg_topics:.2f}\n"
|
256 |
-
analysis_text += f"- Maximum: {max_topics}\n"
|
257 |
-
|
258 |
-
# Remove the temporary topic_count column
|
259 |
-
self.transformed_data.drop('topic_count', axis=1, inplace=True)
|
260 |
-
|
261 |
-
return analysis_text
|
262 |
-
|
263 |
-
def save_transformed_data(self, output_format='xlsx'):
|
264 |
-
"""
|
265 |
-
Save the transformed data and return the file path.
|
266 |
-
Modified to use original filename as prefix.
|
267 |
-
"""
|
268 |
-
if self.transformed_data is None:
|
269 |
-
raise ValueError("No transformed data to save")
|
270 |
-
|
271 |
-
# Create output directory if it doesn't exist
|
272 |
-
output_dir = "outputs"
|
273 |
-
os.makedirs(output_dir, exist_ok=True)
|
274 |
-
|
275 |
-
# Create filename with original filename prefix and timestamp to avoid conflicts
|
276 |
-
import datetime
|
277 |
-
timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
|
278 |
-
|
279 |
-
# Use original filename as prefix, or fallback to 'transformed_feedback' if not available
|
280 |
-
prefix = self.original_filename if self.original_filename else 'transformed_feedback'
|
281 |
-
|
282 |
-
if output_format == 'xlsx':
|
283 |
-
filename = f"{prefix}_transformed_{timestamp}.xlsx"
|
284 |
-
filepath = os.path.join(output_dir, filename)
|
285 |
-
self.transformed_data.to_excel(filepath, index=False)
|
286 |
-
else: # csv
|
287 |
-
filename = f"{prefix}_transformed_{timestamp}.csv"
|
288 |
-
filepath = os.path.join(output_dir, filename)
|
289 |
-
self.transformed_data.to_csv(filepath, index=False)
|
290 |
-
|
291 |
-
# Verify file was created and is readable
|
292 |
-
if not os.path.exists(filepath):
|
293 |
-
raise ValueError(f"Failed to create output file: {filepath}")
|
294 |
-
|
295 |
-
return filepath
|
296 |
-
|
297 |
-
|
298 |
-
# Gradio interface functions
|
299 |
-
def get_column_selector(file_obj):
|
300 |
-
"""
|
301 |
-
Get a combined column preview and selector interface.
|
302 |
-
"""
|
303 |
-
try:
|
304 |
-
if file_obj is None:
|
305 |
-
return gr.CheckboxGroup(
|
306 |
-
choices=[],
|
307 |
-
value=[],
|
308 |
-
label="📋 Select Columns to Include",
|
309 |
-
info="Upload a file first to see available columns"
|
310 |
-
)
|
311 |
-
|
312 |
-
# Read first few rows to get column names
|
313 |
-
file_name = file_obj if isinstance(file_obj, str) else (file_obj.name if hasattr(file_obj, 'name') else 'unknown')
|
314 |
-
_, file_ext = os.path.splitext(file_name)
|
315 |
-
|
316 |
-
if file_ext.lower() in ['.xlsx', '.xls']:
|
317 |
-
df = pd.read_excel(file_obj, nrows=5)
|
318 |
-
elif file_ext.lower() == '.csv':
|
319 |
-
try:
|
320 |
-
df = pd.read_csv(file_obj, nrows=5)
|
321 |
-
except:
|
322 |
-
df = pd.read_csv(file_obj, sep='\t', nrows=5)
|
323 |
-
else:
|
324 |
-
df = pd.read_csv(file_obj, sep='\t', nrows=5)
|
325 |
-
|
326 |
-
columns = list(df.columns)
|
327 |
-
|
328 |
-
# Create column display with indices for easier reference
|
329 |
-
column_choices = [f"{i+1:2d}. {col}" for i, col in enumerate(columns)]
|
330 |
-
|
331 |
-
# Return updated CheckboxGroup with numbered columns and individual rows
|
332 |
-
return gr.CheckboxGroup(
|
333 |
-
choices=column_choices,
|
334 |
-
value=[], # No columns selected by default
|
335 |
-
label=f"📋 Select Columns to Include ({len(columns)} available)",
|
336 |
-
info="Choose which original columns to include in the transformed file (in addition to feedback_id). Columns are numbered for easy reference.",
|
337 |
-
elem_classes=["column-selector"] # Add CSS class for styling
|
338 |
-
)
|
339 |
-
|
340 |
-
except Exception as e:
|
341 |
-
return gr.CheckboxGroup(
|
342 |
-
choices=[],
|
343 |
-
value=[],
|
344 |
-
label="📋 Select Columns to Include",
|
345 |
-
info=f"Error reading file: {str(e)}"
|
346 |
-
)
|
347 |
-
|
348 |
-
|
349 |
-
def extract_column_names(selected_display_names):
|
350 |
-
"""
|
351 |
-
Extract actual column names from the numbered display format.
|
352 |
-
"""
|
353 |
-
if not selected_display_names:
|
354 |
-
return []
|
355 |
-
|
356 |
-
actual_names = []
|
357 |
-
for display_name in selected_display_names:
|
358 |
-
# Remove the number prefix (e.g., "1. Column Name" -> "Column Name")
|
359 |
-
if '. ' in display_name:
|
360 |
-
actual_name = display_name.split('. ', 1)[1]
|
361 |
-
actual_names.append(actual_name)
|
362 |
-
else:
|
363 |
-
actual_names.append(display_name)
|
364 |
-
|
365 |
-
return actual_names
|
366 |
-
|
367 |
-
|
368 |
-
def process_file(file_obj, topic_prefix, sentiment_prefix, category_prefix,
|
369 |
-
text_column, recommendation_column, output_format, analyze_data, selected_columns):
|
370 |
-
"""
|
371 |
-
Main processing function for Gradio interface.
|
372 |
-
"""
|
373 |
-
try:
|
374 |
-
# Extract actual column names from display format
|
375 |
-
actual_column_names = extract_column_names(selected_columns)
|
376 |
-
|
377 |
-
# Initialize transformer
|
378 |
-
transformer = FeedbackTransformer(
|
379 |
-
topic_prefix=topic_prefix,
|
380 |
-
sentiment_prefix=sentiment_prefix,
|
381 |
-
category_prefix=category_prefix,
|
382 |
-
text_column=text_column,
|
383 |
-
recommendation_column=recommendation_column
|
384 |
-
)
|
385 |
-
|
386 |
-
# Load data
|
387 |
-
rows, cols = transformer.load_data(file_obj)
|
388 |
-
status_msg = f"✅ Loaded {rows} rows and {cols} columns\n"
|
389 |
-
|
390 |
-
# Set selected columns for inclusion
|
391 |
-
transformer.set_selected_columns(actual_column_names)
|
392 |
-
status_msg += f"📋 Selected {len(actual_column_names)} original columns for inclusion\n"
|
393 |
-
if actual_column_names:
|
394 |
-
status_msg += f" Selected columns: {', '.join(actual_column_names)}\n"
|
395 |
-
|
396 |
-
# Identify columns
|
397 |
-
col_info = transformer.identify_columns()
|
398 |
-
status_msg += f"\n📊 Found columns:\n"
|
399 |
-
status_msg += f"- Topic columns: {len(col_info['topic_cols'])}\n"
|
400 |
-
status_msg += f"- Sentiment columns: {len(col_info['sentiment_cols'])}\n"
|
401 |
-
status_msg += f"- Category columns: {len(col_info['category_cols'])}\n"
|
402 |
-
|
403 |
-
# Extract unique topics
|
404 |
-
num_topics = transformer.extract_unique_topics()
|
405 |
-
status_msg += f"\n🎯 Found {num_topics} unique topics\n"
|
406 |
-
|
407 |
-
# Transform data
|
408 |
-
shape = transformer.transform_data()
|
409 |
-
status_msg += f"\n✨ Transformed data shape: {shape[0]} rows × {shape[1]} columns\n"
|
410 |
-
|
411 |
-
# Analyze if requested
|
412 |
-
analysis_result = ""
|
413 |
-
if analyze_data:
|
414 |
-
analysis_result = transformer.analyze_data()
|
415 |
-
|
416 |
-
# Save transformed data
|
417 |
-
output_file = transformer.save_transformed_data(output_format)
|
418 |
-
status_msg += f"\n💾 File saved successfully: {os.path.basename(output_file)}\n"
|
419 |
-
|
420 |
-
return status_msg, analysis_result, output_file
|
421 |
-
|
422 |
-
except Exception as e:
|
423 |
-
error_msg = f"❌ Error: {str(e)}\n\n{traceback.format_exc()}"
|
424 |
-
return error_msg, "", None
|
425 |
-
|
426 |
-
|
427 |
-
# Create Gradio interface
|
428 |
-
with gr.Blocks(title="Feedback Topic & Sentiment Transformer", css="""
|
429 |
-
.column-selector .form-check {
|
430 |
-
display: block !important;
|
431 |
-
margin-bottom: 8px !important;
|
432 |
-
}
|
433 |
-
.column-selector .form-check-input {
|
434 |
-
margin-right: 8px !important;
|
435 |
-
}
|
436 |
-
""") as demo:
|
437 |
-
gr.Markdown("""
|
438 |
-
# 📊 Feedback Topic & Sentiment Transformer
|
439 |
-
|
440 |
-
Transform feedback data with topic and sentiment columns into a binary matrix format.
|
441 |
-
Each unique topic becomes a separate column with 0/1 values and associated sentiment scores.
|
442 |
-
|
443 |
-
### 📋 Instructions:
|
444 |
-
1. Upload your Excel, CSV, or tab-delimited text file
|
445 |
-
2. Select which original columns to include in the output
|
446 |
-
3. Configure column prefixes (or use defaults)
|
447 |
-
4. Click "Transform Data" to process
|
448 |
-
5. Download the transformed file
|
449 |
-
""")
|
450 |
-
|
451 |
-
with gr.Row():
|
452 |
-
with gr.Column(scale=1):
|
453 |
-
# File upload
|
454 |
-
input_file = gr.File(
|
455 |
-
label="Upload Input File",
|
456 |
-
file_types=[".xlsx", ".xls", ".csv", ".txt"],
|
457 |
-
type="filepath"
|
458 |
-
)
|
459 |
-
|
460 |
-
# Combined column selector (replaces both preview and checkboxes)
|
461 |
-
gr.Markdown("### 📋 Column Selection")
|
462 |
-
column_selector = gr.CheckboxGroup(
|
463 |
-
choices=[],
|
464 |
-
value=[],
|
465 |
-
label="Select Columns to Include",
|
466 |
-
info="Upload a file first to see available columns"
|
467 |
-
)
|
468 |
-
|
469 |
-
with gr.Column(scale=1):
|
470 |
-
# Configuration parameters
|
471 |
-
gr.Markdown("### ⚙️ Configuration")
|
472 |
-
|
473 |
-
topic_prefix = gr.Textbox(
|
474 |
-
label="Topic Column Prefix",
|
475 |
-
value="[**WORKSHOP] SwissLife Taxonomy",
|
476 |
-
info="Prefix to identify topic columns"
|
477 |
-
)
|
478 |
-
|
479 |
-
sentiment_prefix = gr.Textbox(
|
480 |
-
label="Sentiment Column Prefix",
|
481 |
-
value="ABSA:",
|
482 |
-
info="Prefix to identify sentiment columns"
|
483 |
-
)
|
484 |
-
|
485 |
-
category_prefix = gr.Textbox(
|
486 |
-
label="Category Column Prefix",
|
487 |
-
value="Categories:",
|
488 |
-
info="Prefix to identify category columns"
|
489 |
-
)
|
490 |
-
|
491 |
-
text_column = gr.Textbox(
|
492 |
-
label="Text Column Name",
|
493 |
-
value="TEXT",
|
494 |
-
info="Column containing original feedback text (for reference only)"
|
495 |
-
)
|
496 |
-
|
497 |
-
recommendation_column = gr.Textbox(
|
498 |
-
label="Recommendation Column Name",
|
499 |
-
value="Q4_Weiterempfehlung",
|
500 |
-
info="Column containing recommendation scores (for reference only)"
|
501 |
-
)
|
502 |
-
|
503 |
-
output_format = gr.Radio(
|
504 |
-
label="Output Format",
|
505 |
-
choices=["xlsx", "csv"],
|
506 |
-
value="xlsx"
|
507 |
-
)
|
508 |
-
|
509 |
-
analyze_checkbox = gr.Checkbox(
|
510 |
-
label="Analyze transformed data",
|
511 |
-
value=True
|
512 |
-
)
|
513 |
-
|
514 |
-
# Transform button
|
515 |
-
transform_btn = gr.Button("🔄 Transform Data", variant="primary", size="lg")
|
516 |
-
|
517 |
-
# Output sections
|
518 |
-
with gr.Row():
|
519 |
-
with gr.Column():
|
520 |
-
status_output = gr.Textbox(
|
521 |
-
label="Processing Status",
|
522 |
-
lines=10,
|
523 |
-
interactive=False
|
524 |
-
)
|
525 |
-
|
526 |
-
with gr.Column():
|
527 |
-
analysis_output = gr.Markdown(
|
528 |
-
label="Data Analysis"
|
529 |
-
)
|
530 |
-
|
531 |
-
# Download section with clickable filename
|
532 |
-
with gr.Row():
|
533 |
-
with gr.Column():
|
534 |
-
gr.Markdown("### 📥 Download Transformed File")
|
535 |
-
output_file = gr.File(
|
536 |
-
label="Click filename below to download",
|
537 |
-
interactive=False,
|
538 |
-
show_label=True,
|
539 |
-
file_count="single",
|
540 |
-
file_types=None
|
541 |
-
)
|
542 |
-
|
543 |
-
# Event handlers
|
544 |
-
input_file.change(
|
545 |
-
fn=get_column_selector,
|
546 |
-
inputs=[input_file],
|
547 |
-
outputs=[column_selector]
|
548 |
-
)
|
549 |
-
|
550 |
-
transform_btn.click(
|
551 |
-
fn=process_file,
|
552 |
-
inputs=[
|
553 |
-
input_file,
|
554 |
-
topic_prefix,
|
555 |
-
sentiment_prefix,
|
556 |
-
category_prefix,
|
557 |
-
text_column,
|
558 |
-
recommendation_column,
|
559 |
-
output_format,
|
560 |
-
analyze_checkbox,
|
561 |
-
column_selector
|
562 |
-
],
|
563 |
-
outputs=[status_output, analysis_output, output_file]
|
564 |
-
)
|
565 |
-
|
566 |
-
# Examples section
|
567 |
-
gr.Markdown("""
|
568 |
-
### 📝 Example Column Formats:
|
569 |
-
- **Topic columns**: `[**WORKSHOP] SwissLife Taxonomy(Kommentar) 1`, `[**WORKSHOP] SwissLife Taxonomy(Kommentar) 2`
|
570 |
-
- **Category columns**: `Categories:Topic1`, `Categories:Topic2`
|
571 |
-
- **Sentiment columns**: `ABSA:Sentiment1`, `ABSA:Sentiment2`
|
572 |
-
|
573 |
-
### 🎯 Output Format:
|
574 |
-
- **feedback_id**: Unique identifier for each row
|
575 |
-
- **Selected original columns**: Any columns you selected from the original file
|
576 |
-
- **Topic columns**: Each unique topic becomes a column with values 0 (absent) or 1 (present)
|
577 |
-
- **Sentiment columns**: Each topic has an associated `_sentiment` column with values:
|
578 |
-
- 1.0 = Positive
|
579 |
-
- 0.5 = Neutral
|
580 |
-
- 0.0 = Negative
|
581 |
-
- **Output filename**: `[original_filename]_transformed_[timestamp].[format]`
|
582 |
-
|
583 |
-
### 💡 Tips:
|
584 |
-
- Use the numbered column list to easily identify and select columns
|
585 |
-
- The text and recommendation column names in configuration are now for reference only
|
586 |
-
- To include them in output, select them using the column checkboxes
|
587 |
-
- Click on the filename in the download section to download the file
|
588 |
-
""")
|
589 |
-
|
590 |
-
# Launch the app
|
591 |
-
if __name__ == "__main__":
|
592 |
-
demo.launch()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|