Update app.py

app.py

@@ -3,8 +3,12 @@ import re
 import requests
 from transformers import pipeline, AutoTokenizer, AutoModelForSeq2SeqLM
 from youtube_transcript_api import YouTubeTranscriptApi
+from youtube_transcript_api.formatters import TextFormatter
 import torch
 import gc
+import time
+from urllib.parse import urlparse, parse_qs
+import json
 
 # Optimize for HuggingFace Spaces - Use smaller models and efficient loading
 print("🚀 Loading models for HuggingFace Spaces...")
@@ -13,72 +17,141 @@ print("🚀 Loading models for HuggingFace Spaces...")
 @torch.no_grad()
 def load_summarizer():
     model_name = "facebook/bart-large-cnn"
-    tokenizer = AutoTokenizer.from_pretrained(model_name)
-    model = AutoModelForSeq2SeqLM.from_pretrained(model_name, torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32)
-    return pipeline("summarization", model=model, tokenizer=tokenizer, device=0 if torch.cuda.is_available() else -1)
+    try:
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        model = AutoModelForSeq2SeqLM.from_pretrained(
+            model_name, 
+            torch_dtype=torch.float16 if torch.cuda.is_available() else torch.float32
+        )
+        return pipeline("summarization", model=model, tokenizer=tokenizer, 
+                       device=0 if torch.cuda.is_available() else -1)
+    except Exception as e:
+        print(f"Error loading summarizer: {e}")
+        # Fallback to a smaller model if BART fails
+        try:
+            return pipeline("summarization", model="sshleifer/distilbart-cnn-12-6", 
+                          device=0 if torch.cuda.is_available() else -1)
+        except:
+            return None
 
 # Initialize summarizer
 summarizer = load_summarizer()
 
 def extract_video_id(url):
     """Extract video ID from various YouTube URL formats"""
+    if not url:
+        return None
+    
+    # Clean the URL
+    url = url.strip()
+    
     patterns = [
         r'(?:v=|\/)([0-9A-Za-z_-]{11}).*',
         r'(?:embed\/)([0-9A-Za-z_-]{11})',
         r'(?:v\/)([0-9A-Za-z_-]{11})',
-        r'(?:youtu\.be\/)([0-9A-Za-z_-]{11})'
+        r'(?:youtu\.be\/)([0-9A-Za-z_-]{11})',
+        r'(?:watch\?v=)([0-9A-Za-z_-]{11})'
     ]
+    
     for pattern in patterns:
         match = re.search(pattern, url)
         if match:
-            return match.group(1)
+            video_id = match.group(1)
+            # Validate video ID length
+            if len(video_id) == 11:
+                return video_id
     return None
 
-def get_youtube_transcript(video_id):
-    """Get transcript using YouTube Transcript API - Most reliable for HF Spaces"""
+def get_video_info(video_id):
+    """Get basic video information"""
     try:
-        # Priority order for languages (Hindi, English variants)
-        language_codes = ['hi', 'en', 'en-IN', 'en-US', 'en-GB']
-        
-        transcript_data = None
-        used_language = None
-        
-        # Try each language
-        for lang_code in language_codes:
-            try:
-                transcript_list = YouTubeTranscriptApi.get_transcript(video_id, languages=[lang_code])
-                transcript_data = transcript_list
-                used_language = lang_code
-                break
-            except:
-                continue
-        
-        # If specific languages fail, try auto-generated
-        if not transcript_data:
-            try:
-                transcript_list = YouTubeTranscriptApi.get_transcript(video_id)
-                transcript_data = transcript_list
-                used_language = "auto-detected"
-            except Exception as e:
-                return None, f"No transcript available: {str(e)}"
-        
-        # Process transcript
-        if transcript_data:
-            transcript_text = ' '.join([item['text'].replace('\n', ' ') for item in transcript_data])
-            # Clean up common transcript artifacts
-            transcript_text = re.sub(r'\[.*?\]', '', transcript_text)  # Remove [Music], [Applause] etc
-            transcript_text = re.sub(r'\s+', ' ', transcript_text).strip()  # Clean whitespace
+        # This is a simple way to check if video exists
+        # In production, you might want to use YouTube Data API
+        return f"https://www.youtube.com/watch?v={video_id}"
+    except:
+        return None
+
+def get_youtube_transcript_with_retry(video_id, max_retries=3):
+    """Get transcript with retry mechanism and better error handling"""
+    
+    if not video_id:
+        return None, "Invalid video ID"
+    
+    # Language priority order
+    language_codes = ['hi', 'en', 'en-IN', 'en-US', 'en-GB', 'auto']
+    
+    for attempt in range(max_retries):
+        try:
+            transcript_data = None
+            used_language = None
             
-            return transcript_text, f"Transcript found in: {used_language}"
-        
-        return None, "No transcript data found"
-        
-    except Exception as e:
-        return None, f"Transcript API Error: {str(e)}"
+            # Try each language
+            for lang_code in language_codes:
+                try:
+                    if lang_code == 'auto':
+                        # Try auto-generated as last resort
+                        transcript_list = YouTubeTranscriptApi.get_transcript(video_id)
+                    else:
+                        transcript_list = YouTubeTranscriptApi.get_transcript(video_id, languages=[lang_code])
+                    
+                    transcript_data = transcript_list
+                    used_language = lang_code
+                    break
+                except Exception as lang_error:
+                    continue
+            
+            # Process transcript if found
+            if transcript_data:
+                formatter = TextFormatter()
+                transcript_text = formatter.format_transcript(transcript_data)
+                
+                # Clean up the transcript
+                transcript_text = re.sub(r'\[.*?\]', '', transcript_text)  # Remove [Music], [Applause] etc
+                transcript_text = re.sub(r'\s+', ' ', transcript_text).strip()  # Clean whitespace
+                transcript_text = re.sub(r'\.{2,}', '.', transcript_text)  # Fix multiple dots
+                
+                if len(transcript_text) < 50:
+                    return None, "Transcript too short or empty"
+                
+                return transcript_text, f"Success - Language: {used_language}"
+            
+            # If no transcript found, wait before retry
+            if attempt < max_retries - 1:
+                time.sleep(2 ** attempt)  # Exponential backoff
+                
+        except Exception as e:
+            error_msg = str(e).lower()
+            
+            # Handle specific YouTube API errors
+            if "transcript disabled" in error_msg:
+                return None, "❌ Transcripts are disabled for this video"
+            elif "not available" in error_msg:
+                return None, "❌ No transcript available for this video"
+            elif "video unavailable" in error_msg:
+                return None, "❌ Video is unavailable or private"
+            elif "quota exceeded" in error_msg:
+                return None, "❌ API quota exceeded, please try again later"
+            elif any(block_term in error_msg for block_term in ["ip", "block", "banned", "rate limit"]):
+                if attempt < max_retries - 1:
+                    time.sleep(5 * (attempt + 1))  # Longer wait for IP blocks
+                    continue
+                else:
+                    return None, "❌ IP blocked by YouTube. Try using a VPN or proxy, or try again later"
+            else:
+                print(f"Attempt {attempt + 1} failed: {e}")
+                if attempt < max_retries - 1:
+                    time.sleep(2 ** attempt)
+                    continue
+    
+    return None, f"❌ Failed to get transcript after {max_retries} attempts"
 
 def chunk_text_for_summarization(text, max_chunk_size=800):
     """Split text into chunks for summarization"""
-    sentences = text.replace('।', '.').split('.')  # Handle Hindi sentences
+    if not text:
+        return []
+    
+    # Handle different sentence endings (English and Hindi)
+    sentences = re.split(r'[.।!?]+', text)
     chunks = []
     current_chunk = ""
     
@@ -88,23 +161,26 @@ def chunk_text_for_summarization(text, max_chunk_size=800):
             continue
             
         # Check if adding this sentence would exceed limit
-        if len(current_chunk) + len(sentence) + 1 < max_chunk_size:
+        if len(current_chunk) + len(sentence) + 2 < max_chunk_size:
             current_chunk += sentence + ". "
         else:
-            if current_chunk:
+            if current_chunk.strip():
                 chunks.append(current_chunk.strip())
             current_chunk = sentence + ". "
     
     # Add the last chunk
-    if current_chunk:
+    if current_chunk.strip():
         chunks.append(current_chunk.strip())
     
-    return chunks
+    return [chunk for chunk in chunks if len(chunk.strip()) > 20]
 
 def summarize_text_optimized(text):
     """Optimized summarization for HuggingFace Spaces"""
+    if not summarizer:
+        return "❌ Summarization model not available"
+    
     if not text or len(text.strip()) < 100:
-        return "Text too short to summarize (minimum 100 characters required)"
+        return "❌ Text too short to summarize (minimum 100 characters required)"
     
     try:
         # Clean memory before processing
@@ -113,23 +189,24 @@ def summarize_text_optimized(text):
         gc.collect()
         
         # For very long texts, chunk them
-        if len(text) > 1500:
-            chunks = chunk_text_for_summarization(text, max_chunk_size=900)
+        if len(text) > 1200:
+            chunks = chunk_text_for_summarization(text, max_chunk_size=800)
             summaries = []
             
-            # Process first 3 chunks to avoid timeout
-            for i, chunk in enumerate(chunks[:3]):
+            # Process chunks (limit to first 4 to avoid timeout)
+            for i, chunk in enumerate(chunks[:4]):
                 if len(chunk.strip()) < 50:
                     continue
                     
                 try:
                     summary = summarizer(
                         chunk,
-                        max_length=120,
-                        min_length=30,
+                        max_length=min(120, len(chunk.split()) // 3 + 20),
+                        min_length=20,
                         do_sample=False,
-                        num_beams=2,  # Reduced for speed
-                        length_penalty=1.0
+                        num_beams=2,
+                        length_penalty=1.0,
+                        early_stopping=True
                     )[0]["summary_text"]
                     summaries.append(summary)
                 except Exception as chunk_error:
@@ -138,86 +215,108 @@ def summarize_text_optimized(text):
             
             if summaries:
                 combined_summary = " ".join(summaries)
+                
                 # If combined summary is still too long, summarize it again
-                if len(combined_summary) > 600:
+                if len(combined_summary) > 500:
                     try:
                         final_summary = summarizer(
                             combined_summary,
                             max_length=200,
-                            min_length=80,
+                            min_length=60,
                             do_sample=False,
-                            num_beams=2
+                            num_beams=2,
+                            early_stopping=True
                         )[0]["summary_text"]
                         return final_summary
                     except:
-                        return combined_summary
+                        return combined_summary[:500] + "..."
                 return combined_summary
             else:
-                return "Could not generate summary from chunks"
+                return "❌ Could not generate summary from the provided text"
         else:
             # For shorter texts, direct summarization
+            word_count = len(text.split())
+            max_length = min(150, word_count // 2 + 30)
+            min_length = min(30, word_count // 4)
+            
             summary = summarizer(
                 text,
-                max_length=150,
-                min_length=50,
+                max_length=max_length,
+                min_length=min_length,
                 do_sample=False,
                 num_beams=2,
-                length_penalty=1.0
+                length_penalty=1.0,
+                early_stopping=True
             )[0]["summary_text"]
             return summary
             
     except Exception as e:
-        return f"Summarization error: {str(e)}"
+        return f"❌ Summarization error: {str(e)}"
 
-def process_youtube_video(url):
+def process_youtube_video(url, progress=gr.Progress()):
     """Main processing function optimized for HuggingFace Spaces"""
     
     # Input validation
     if not url or not url.strip():
-        return "❌ Please enter a YouTube URL", "", "No summary available"
+        return "❌ Please enter a YouTube URL", "", "❌ No summary available - URL required"
+    
+    progress(0.1, desc="Validating URL...")
     
     # Extract video ID
     video_id = extract_video_id(url.strip())
     if not video_id:
-        return "❌ Invalid YouTube URL format", "Please check the URL format", "No summary available"
+        return ("❌ Invalid YouTube URL format", 
+                "Please use a valid YouTube URL like:\n- https://www.youtube.com/watch?v=VIDEO_ID\n- https://youtu.be/VIDEO_ID", 
+                "❌ Cannot generate summary without valid URL")
     
-    # Update progress
-    progress_msg = "🔍 Extracting video transcript..."
+    progress(0.2, desc="Extracting video transcript...")
     
     # Get transcript
-    transcript, status = get_youtube_transcript(video_id)
+    transcript, status = get_youtube_transcript_with_retry(video_id)
     
     if not transcript:
         return (
             "❌ Could not extract transcript", 
-            f"Status: {status}\n\nThis video might not have captions/subtitles available.",
-            "Cannot generate summary without transcript"
+            f"Status: {status}\n\n💡 Troubleshooting tips:\n"
+            "• Check if the video has captions/subtitles enabled\n"
+            "• Try a different video\n"
+            "• If using HuggingFace Spaces, try again later due to IP restrictions\n"
+            "• Consider using a VPN if the issue persists",
+            "❌ Cannot generate summary without transcript"
         )
     
+    progress(0.7, desc="Generating AI summary...")
+    
     # Generate summary
-    progress_msg = "🤖 Generating AI summary..."
     summary = summarize_text_optimized(transcript)
     
+    progress(1.0, desc="Complete!")
+    
     # Create video embed
     embed_html = f'''
-    <div style="text-align: center;">
-        <iframe width="560" height="315" 
+    <div style="text-align: center; margin: 10px 0;">
+        <iframe width="100%" height="315" 
                 src="https://www.youtube.com/embed/{video_id}" 
                 frameborder="0" 
                 allowfullscreen
-                style="max-width: 100%; border-radius: 10px;">
+                style="max-width: 560px; border-radius: 10px; box-shadow: 0 4px 8px rgba(0,0,0,0.1);">
         </iframe>
     </div>
     '''
     
     # Format transcript info
-    transcript_info = f"""📊 Processing Status: ✅ Success
-🎯 Method: YouTube Transcript API
-🌐 Language: {status}
-📝 Transcript Length: {len(transcript)} characters
-📄 Word Count: ~{len(transcript.split())} words
+    word_count = len(transcript.split())
+    char_count = len(transcript)
+    
+    transcript_info = f"""✅ **Processing Status**: Success
+🎯 **Method**: YouTube Transcript API  
+🌐 **Language**: {status}
+📊 **Statistics**:
+   • Characters: {char_count:,}
+   • Words: ~{word_count:,}
+   • Estimated reading time: ~{word_count//200 + 1} minutes
 
-📋 Full Transcript:
+📋 **Full Transcript**:
 {transcript}"""
     
     return embed_html, transcript_info, summary
@@ -225,106 +324,134 @@ def process_youtube_video(url):
 # Custom CSS for better UI
 custom_css = """
 #component-0 {
-    max-width: 900px;
+    max-width: 1000px;
     margin: auto;
 }
 .gradio-container {
     font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
 }
+.progress-bar {
+    background: linear-gradient(90deg, #667eea 0%, #764ba2 100%);
+}
 """
 
 # Create Gradio Interface optimized for HuggingFace Spaces
-with gr.Blocks(css=custom_css, title="YouTube Video Summarizer", theme=gr.themes.Soft()) as demo:
+with gr.Blocks(css=custom_css, title="YouTube Video Summarizer AI", theme=gr.themes.Soft()) as demo:
     gr.HTML("""
-    <div style="text-align: center; padding: 20px;">
-        <h1>🎓 YouTube Video Summarizer</h1>
-        <p style="font-size: 18px; color: #666;">
+    <div style="text-align: center; padding: 20px; background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); border-radius: 15px; margin-bottom: 20px; color: white;">
+        <h1 style="margin: 0; font-size: 2.5em;">🎓 YouTube Video Summarizer AI</h1>
+        <p style="font-size: 18px; margin: 10px 0; opacity: 0.9;">
             AI-powered summarization for Hindi, Hinglish & English videos
         </p>
-        <p style="color: #888;">
-            Optimized for HuggingFace Spaces • Uses YouTube Transcript API
+        <p style="opacity: 0.8; margin: 0;">
+            ⚡ Fast • 🎯 Accurate • 🌐 Multi-language Support
         </p>
     </div>
     """)
     
     with gr.Row():
-        with gr.Column(scale=2):
+        with gr.Column(scale=3):
             url_input = gr.Textbox(
                 label="📺 YouTube URL",
-                placeholder="https://www.youtube.com/watch?v=...",
+                placeholder="https://www.youtube.com/watch?v=dQw4w9WgXcQ",
                 lines=1,
-                info="Paste any YouTube video URL here"
+                info="Paste any YouTube video URL here (must have captions/subtitles)"
             )
             
         with gr.Column(scale=1):
             submit_btn = gr.Button(
-                "🚀 Summarize Video", 
+                "🚀 Analyze Video", 
                 variant="primary",
                 size="lg"
             )
     
+    # Status indicator
+    status_text = gr.HTML("")
+    
     # Results section
     with gr.Row():
-        with gr.Column():
+        with gr.Column(scale=1):
             video_embed = gr.HTML(label="📺 Video Player")
             
-        with gr.Column():
+        with gr.Column(scale=1):
             summary_output = gr.Textbox(
                 label="📋 AI Summary",
-                lines=8,
-                max_lines=12,
-                info="AI-generated summary of the video content"
+                lines=12,
+                max_lines=15,
+                info="AI-generated summary of the video content",
+                show_copy_button=True
             )
     
     # Expandable transcript section
-    with gr.Accordion("📝 Full Transcript & Details", open=False):
+    with gr.Accordion("📝 Full Transcript & Processing Details", open=False):
         transcript_output = gr.Textbox(
-            label="Complete Transcript",
-            lines=15,
-            max_lines=25,
-            info="Full video transcript with processing details"
+            label="Complete Transcript with Metadata",
+            lines=20,
+            max_lines=30,
+            info="Full video transcript with processing details",
+            show_copy_button=True
         )
     
     # Examples section
-    gr.HTML("<h3 style='margin-top: 30px;'>🎯 Try these examples:</h3>")
+    gr.HTML("<h3 style='margin-top: 30px; text-align: center;'>🎯 Try these examples:</h3>")
     
+    # Note: Using placeholder examples - replace with actual working video IDs
     gr.Examples(
         examples=[
-            ["https://www.youtube.com/watch?v=dQw4w9WgXcQ"],
-            ["https://youtu.be/dQw4w9WgXcQ"],
+            ["https://www.youtube.com/watch?v=kJQP7kiw5Fk"],  # TED Talk example
+            ["https://youtu.be/9bZkp7q19f0"],  # Educational content
+            ["https://www.youtube.com/watch?v=aircAruvnKk"],  # Popular educational channel
         ],
         inputs=url_input,
-        label="Sample URLs"
+        label="Sample URLs (Educational Content)"
     )
     
     # Info section
-    with gr.Accordion("ℹ️ How it works", open=False):
+    with gr.Accordion("ℹ️ How it works & Troubleshooting", open=False):
         gr.Markdown("""
         ### 🔧 How this tool works:
         
-        1. **Extract Video ID**: Parses the YouTube URL to get the video identifier
-        2. **Fetch Transcript**: Uses YouTube Transcript API to get captions/subtitles
-        3. **AI Summarization**: Processes text through BART model for intelligent summarization
-        4. **Multi-language Support**: Handles Hindi, Hinglish, and English content
+        1. **🎯 URL Parsing**: Extracts video ID from various YouTube URL formats
+        2. **📝 Transcript Extraction**: Uses YouTube Transcript API with retry logic
+        3. **🤖 AI Summarization**: Processes text through BART/DistilBART models
+        4. **🌐 Multi-language Support**: Handles Hindi, Hinglish, and English content
+        5. **⚡ Smart Processing**: Chunks long videos and optimizes for performance
         
         ### 📋 Supported Languages:
         - 🇮🇳 **Hindi**: Full support for Hindi captions
-        - 🌐 **Hinglish**: Mixed Hindi-English content
+        - 🌐 **Hinglish**: Mixed Hindi-English content  
         - 🇺🇸 **English**: All English variants
+        - 🔄 **Auto-generated**: Automatic language detection
+        
+        ### ⚠️ Known Limitations & Solutions:
+        
+        **IP Blocking Issues:**
+        - YouTube blocks many cloud provider IPs (HuggingFace Spaces, AWS, etc.)
+        - **Solution**: Try again later, use VPN, or run locally
+        
+        **Video Requirements:**
+        - Video must have captions/subtitles (auto-generated or manual)
+        - Video must be public (not private or unlisted)
+        
+        **Performance Optimizations:**
+        - Long videos are automatically chunked to prevent timeouts
+        - Memory management for stable processing
+        - Fallback to smaller models if needed
         
-        ### ⚡ Optimizations for HuggingFace Spaces:
-        - Efficient model loading with memory management
-        - Chunked processing for long videos
-        - GPU acceleration when available
-        - Automatic text cleanup and formatting
+        ### 🛠️ Troubleshooting:
+        - **"No transcript available"**: Video lacks captions - try another video
+        - **"IP blocked"**: Common on cloud platforms - try VPN or local setup
+        - **"Video unavailable"**: Check if video is public and exists
+        - **Slow processing**: Normal for long videos - please wait
         
-        ### ⚠️ Limitations:
-        - Requires videos to have captions/subtitles
-        - Processing time depends on transcript length
-        - Very long videos are chunked to prevent timeouts
+        ### 💡 Tips for Best Results:
+        - Use videos with clear speech and good audio quality
+        - Educational/tutorial videos often have better transcripts
+        - Shorter videos (< 20 minutes) process faster
+        - Popular channels often have better auto-generated captions
         """)
     
-    # Event handlers
+    # Event handlers with progress tracking
     submit_btn.click(
         fn=process_youtube_video,
         inputs=[url_input],
@@ -339,11 +466,12 @@ with gr.Blocks(css=custom_css, title="YouTube Video Summarizer", theme=gr.themes
 
 # Launch configuration for HuggingFace Spaces
 if __name__ == "__main__":
-    demo.queue(max_size=10)  # Limit queue size for stability
+    demo.queue(max_size=5, default_concurrency_limit=2)  # Limit for stability
     demo.launch(
         server_name="0.0.0.0",
         server_port=7860,
-        share=False,  # Don't need share link in HF Spaces
-        debug=False,  # Disable debug in production
-        show_error=True
+        share=False,
+        debug=False,
+        show_error=True,
+        max_threads=2  # Limit threads for better memory management
     )