Spaces:
Paused
Paused
Commit
·
19b4b73
1
Parent(s):
b5223b4
moved to stremlit
Browse files- .claude/settings.local.json +2 -1
- CLAUDE.md +26 -16
- CLAUDE_SVELTE_FRONTEND_GUIDE.md +0 -186
- DEPLOYMENT.md +213 -0
- DIGIPAL_V2_GUIDE.md +0 -355
- Dockerfile +4 -4
- QUICK_START.md +0 -169
- QUICK_UI_TEST.md +63 -0
- README.md +84 -123
- app.py +26 -54
- requirements.txt +6 -5
- run_digipal.py +80 -0
- src/ai/speech_engine.py +186 -91
- src/core/monster_3d_hunyuan_integration.py +0 -326
- src/pipelines/hunyuan3d_pipeline.py +0 -958
- src/pipelines/opensource_3d_pipeline_v2.py +351 -308
- src/ui/gradio_interface.py +0 -1263
- src/ui/streamlit_interface.py +565 -0
- streamlit_app.py +55 -0
- test_ui.py +50 -0
.claude/settings.local.json
CHANGED
@@ -22,7 +22,8 @@
|
|
22 |
"Bash(git reset:*)",
|
23 |
"WebFetch(domain:github.com)",
|
24 |
"Bash(timeout:*)",
|
25 |
-
"Bash(git rm:*)"
|
|
|
26 |
],
|
27 |
"deny": []
|
28 |
}
|
|
|
22 |
"Bash(git reset:*)",
|
23 |
"WebFetch(domain:github.com)",
|
24 |
"Bash(timeout:*)",
|
25 |
+
"Bash(git rm:*)",
|
26 |
+
"Bash(chmod:*)"
|
27 |
],
|
28 |
"deny": []
|
29 |
}
|
CLAUDE.md
CHANGED
@@ -4,34 +4,33 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
|
|
4 |
|
5 |
## Project Overview
|
6 |
|
7 |
-
DigiPal is an advanced AI-powered virtual monster companion application built with
|
8 |
|
9 |
## Architecture
|
10 |
|
11 |
### Core Technologies
|
12 |
-
- **Frontend**:
|
13 |
-
- **
|
14 |
-
- **
|
|
|
15 |
- **Framework**: Python 3.11+ with asyncio for concurrent operations
|
16 |
- **Database**: SQLite for monster persistence with async operations
|
17 |
-
- **Deployment**:
|
18 |
|
19 |
### Component Structure
|
20 |
```
|
21 |
src/
|
22 |
├── ai/ # AI processing components
|
23 |
│ ├── qwen_processor.py # Qwen 2.5 conversation engine
|
24 |
-
│ └── speech_engine.py #
|
25 |
├── core/ # Core game logic
|
26 |
│ ├── monster_engine.py # Monster stats, evolution, persistence
|
27 |
│ ├── monster_engine_dw1.py # DW1-aligned monster mechanics (reference)
|
28 |
-
│
|
29 |
-
│ └── monster_3d_hunyuan_integration.py # Hunyuan3D specific integration
|
30 |
├── pipelines/ # 3D generation pipelines
|
31 |
-
│
|
32 |
-
│ └── opensource_3d_pipeline_v2.py # Enhanced 3D pipeline with MCP
|
33 |
├── ui/ # User interface
|
34 |
-
│ ├──
|
35 |
│ └── state_manager.py # Browser state management
|
36 |
├── deployment/ # Deployment optimization
|
37 |
│ └── zero_gpu_optimizer.py # Zero GPU resource management
|
@@ -43,14 +42,22 @@ src/
|
|
43 |
|
44 |
### Running the Application
|
45 |
```bash
|
46 |
-
# Run
|
|
|
|
|
|
|
|
|
|
|
47 |
python app.py
|
48 |
|
|
|
|
|
|
|
49 |
# Run with debug logging
|
50 |
LOG_LEVEL=DEBUG python app.py
|
51 |
|
52 |
# Run with specific configuration
|
53 |
-
|
54 |
|
55 |
# Run with MCP enabled
|
56 |
MCP_ENDPOINT=https://your-mcp-server MCP_API_KEY=your-key python app.py
|
@@ -110,17 +117,20 @@ pytest
|
|
110 |
|
111 |
### AI Conversation System
|
112 |
- **Qwen 2.5 integration** with quantization support (8-bit) for GPU efficiency
|
|
|
113 |
- **Context-aware conversations** with personality-based system prompts
|
114 |
- **Mood-responsive dialogue** based on current monster stats
|
115 |
- **Conversation history management** with automatic truncation
|
116 |
- **Flash Attention 2** optimization when available
|
117 |
|
118 |
### 3D Generation Pipeline
|
119 |
-
- **
|
120 |
-
- **
|
|
|
|
|
|
|
121 |
- **Model caching**: Efficient reuse of generated 3D assets
|
122 |
- **Async generation**: Non-blocking 3D model creation
|
123 |
-
- **MCP integration**: Access to external model services via Model Context Protocol
|
124 |
|
125 |
### State Management
|
126 |
- **Async SQLite operations** for monster persistence
|
|
|
4 |
|
5 |
## Project Overview
|
6 |
|
7 |
+
DigiPal is an advanced AI-powered virtual monster companion application built with Streamlit, featuring deep AI conversations using Qwen 2.5 models, Kyutai STT speech recognition, comprehensive monster care systems, sophisticated evolution mechanics, and cutting-edge 3D model generation via OmniGen2 → Hunyuan3D-2.1 → UniRig pipeline. This is a streamlined multi-component system designed for modern deployment with HuggingFace integration.
|
8 |
|
9 |
## Architecture
|
10 |
|
11 |
### Core Technologies
|
12 |
+
- **Frontend**: Streamlit with modern cyberpunk UI design
|
13 |
+
- **Backend**: FastAPI with WebSocket support for real-time updates
|
14 |
+
- **AI Models**: Qwen 2.5-1.5B-Instruct for conversations, Kyutai STT-2.6b-en for speech
|
15 |
+
- **3D Pipeline**: OmniGen2 → Hunyuan3D-2.1 → UniRig (text-to-image-to-3D-to-rigged)
|
16 |
- **Framework**: Python 3.11+ with asyncio for concurrent operations
|
17 |
- **Database**: SQLite for monster persistence with async operations
|
18 |
+
- **Deployment**: Modern architecture with HuggingFace integration, Docker support
|
19 |
|
20 |
### Component Structure
|
21 |
```
|
22 |
src/
|
23 |
├── ai/ # AI processing components
|
24 |
│ ├── qwen_processor.py # Qwen 2.5 conversation engine
|
25 |
+
│ └── speech_engine.py # Kyutai STT speech recognition
|
26 |
├── core/ # Core game logic
|
27 |
│ ├── monster_engine.py # Monster stats, evolution, persistence
|
28 |
│ ├── monster_engine_dw1.py # DW1-aligned monster mechanics (reference)
|
29 |
+
│ └── evolution_system.py # Evolution mechanics
|
|
|
30 |
├── pipelines/ # 3D generation pipelines
|
31 |
+
│ └── opensource_3d_pipeline_v2.py # Production 3D pipeline: OmniGen2→Hunyuan3D→UniRig
|
|
|
32 |
├── ui/ # User interface
|
33 |
+
│ ├── streamlit_interface.py # Modern Streamlit interface
|
34 |
│ └── state_manager.py # Browser state management
|
35 |
├── deployment/ # Deployment optimization
|
36 |
│ └── zero_gpu_optimizer.py # Zero GPU resource management
|
|
|
42 |
|
43 |
### Running the Application
|
44 |
```bash
|
45 |
+
# Run complete application (FastAPI + Streamlit)
|
46 |
+
python run_digipal.py
|
47 |
+
|
48 |
+
# Or run components separately:
|
49 |
+
|
50 |
+
# Run FastAPI backend server
|
51 |
python app.py
|
52 |
|
53 |
+
# Run Streamlit frontend (in another terminal)
|
54 |
+
streamlit run src/ui/streamlit_interface.py
|
55 |
+
|
56 |
# Run with debug logging
|
57 |
LOG_LEVEL=DEBUG python app.py
|
58 |
|
59 |
# Run with specific configuration
|
60 |
+
API_PORT=8081 python app.py
|
61 |
|
62 |
# Run with MCP enabled
|
63 |
MCP_ENDPOINT=https://your-mcp-server MCP_API_KEY=your-key python app.py
|
|
|
117 |
|
118 |
### AI Conversation System
|
119 |
- **Qwen 2.5 integration** with quantization support (8-bit) for GPU efficiency
|
120 |
+
- **Kyutai STT-2.6b-en** for high-quality speech-to-text conversion
|
121 |
- **Context-aware conversations** with personality-based system prompts
|
122 |
- **Mood-responsive dialogue** based on current monster stats
|
123 |
- **Conversation history management** with automatic truncation
|
124 |
- **Flash Attention 2** optimization when available
|
125 |
|
126 |
### 3D Generation Pipeline
|
127 |
+
- **OmniGen2**: Advanced text-to-image generation with multi-view consistency
|
128 |
+
- **Hunyuan3D-2.1**: State-of-the-art image-to-3D conversion via official HuggingFace Space API
|
129 |
+
- **UniRig**: Automatic 3D model rigging via HuggingFace integration
|
130 |
+
- **Complete Pipeline**: text → multi-view images → 3D mesh → rigged model
|
131 |
+
- **Fallback Systems**: Graceful degradation when APIs are unavailable
|
132 |
- **Model caching**: Efficient reuse of generated 3D assets
|
133 |
- **Async generation**: Non-blocking 3D model creation
|
|
|
134 |
|
135 |
### State Management
|
136 |
- **Async SQLite operations** for monster persistence
|
CLAUDE_SVELTE_FRONTEND_GUIDE.md
DELETED
@@ -1,186 +0,0 @@
|
|
1 |
-
# **Claude Development Guide: DigiPal Svelte Frontend**
|
2 |
-
|
3 |
-
This document outlines the plan for the complete UI overhaul of DigiPal, moving from Gradio to a custom SvelteKit application with voice-first interaction.
|
4 |
-
|
5 |
-
## **Status: Implementation Complete ✅**
|
6 |
-
|
7 |
-
All major components have been implemented:
|
8 |
-
- ✅ Unified backend with FastAPI + WebSocket support
|
9 |
-
- ✅ SvelteKit frontend structure
|
10 |
-
- ✅ Voice-first interaction system
|
11 |
-
- ✅ DigiVice-style UI components
|
12 |
-
- ✅ 3D rendering with Threlte
|
13 |
-
- ✅ MCP integration
|
14 |
-
- ✅ Real-time WebSocket updates
|
15 |
-
- ✅ Cyberpunk-retro styling
|
16 |
-
|
17 |
-
## **1. Project Architecture**
|
18 |
-
|
19 |
-
### **Backend (Python)**
|
20 |
-
- **Main Application**: `app.py` - Unified application with all features enabled
|
21 |
-
- **API Server**: FastAPI on port 7861
|
22 |
-
- **Gradio Admin**: Running on port 7860 as fallback/admin interface
|
23 |
-
- **WebSocket**: Real-time stat updates and model changes
|
24 |
-
- **MCP Support**: Configurable via environment variables
|
25 |
-
|
26 |
-
### **Frontend (SvelteKit)**
|
27 |
-
Located in `/frontend` directory:
|
28 |
-
- **Framework**: SvelteKit with TypeScript
|
29 |
-
- **3D Rendering**: Threlte (Three.js wrapper for Svelte)
|
30 |
-
- **Styling**: Tailwind CSS with custom cyberpunk-retro theme
|
31 |
-
- **Voice**: Web Speech API with intent parsing
|
32 |
-
- **State**: Svelte stores for reactive state management
|
33 |
-
|
34 |
-
## **2. Running the Application**
|
35 |
-
|
36 |
-
### **Backend Setup**
|
37 |
-
```bash
|
38 |
-
# Install Python dependencies
|
39 |
-
pip install -r requirements.txt
|
40 |
-
|
41 |
-
# Run the unified backend
|
42 |
-
python app.py
|
43 |
-
|
44 |
-
# Or with MCP enabled
|
45 |
-
MCP_ENDPOINT=https://your-mcp-endpoint MCP_API_KEY=your-key python app.py
|
46 |
-
```
|
47 |
-
|
48 |
-
### **Frontend Setup**
|
49 |
-
```bash
|
50 |
-
# Navigate to frontend directory
|
51 |
-
cd frontend
|
52 |
-
|
53 |
-
# Install dependencies
|
54 |
-
npm install
|
55 |
-
|
56 |
-
# Run development server
|
57 |
-
npm run dev
|
58 |
-
|
59 |
-
# Build for production
|
60 |
-
npm run build
|
61 |
-
```
|
62 |
-
|
63 |
-
## **3. API Endpoints**
|
64 |
-
|
65 |
-
The backend exposes these REST endpoints on port 7861:
|
66 |
-
|
67 |
-
- `GET /api/monsters` - List all monsters
|
68 |
-
- `POST /api/monsters` - Create new monster
|
69 |
-
- `GET /api/monsters/{id}` - Get monster details
|
70 |
-
- `POST /api/monsters/{id}/action` - Perform care action
|
71 |
-
- `POST /api/monsters/{id}/talk` - Send message
|
72 |
-
- `POST /api/monsters/{id}/generate-3d` - Generate 3D model
|
73 |
-
- `WS /api/monsters/{id}/ws` - WebSocket for real-time updates
|
74 |
-
|
75 |
-
## **4. Voice Commands**
|
76 |
-
|
77 |
-
The system recognizes these voice intents:
|
78 |
-
|
79 |
-
### **Care Actions**
|
80 |
-
- "Feed [food type]" → `feed` action
|
81 |
-
- "Train [skill]" → `train` action
|
82 |
-
- "Play with monster" → `play` action
|
83 |
-
- "Clean monster" → `clean` action
|
84 |
-
- "Heal monster" → `heal` action
|
85 |
-
- "Let monster rest" → `rest` action
|
86 |
-
- "Discipline monster" → `discipline` action
|
87 |
-
|
88 |
-
### **3D Generation**
|
89 |
-
- "Generate 3D model" → triggers 3D generation
|
90 |
-
- "Create a [description]" → generates with description
|
91 |
-
|
92 |
-
### **Conversation**
|
93 |
-
- Any other speech → sent as conversation to monster
|
94 |
-
|
95 |
-
## **5. Component Structure**
|
96 |
-
|
97 |
-
### **Core Components**
|
98 |
-
- `Device.svelte` - Main device container
|
99 |
-
- `Screen.svelte` - 3D display with CRT effect
|
100 |
-
- `MonsterScene.svelte` - Three.js scene for monster
|
101 |
-
- `Dpad.svelte` - Directional pad control
|
102 |
-
- `ActionButton.svelte` - A/B buttons
|
103 |
-
- `VoiceButton.svelte` - Voice activation
|
104 |
-
- `HolographicStats.svelte` - Stats overlay
|
105 |
-
|
106 |
-
### **Services**
|
107 |
-
- `api.ts` - Backend communication
|
108 |
-
- `voice.ts` - Speech recognition & intent parsing
|
109 |
-
- `mcp.ts` - Model Context Protocol integration
|
110 |
-
|
111 |
-
### **Stores**
|
112 |
-
- `monsterStore.ts` - Monster state management
|
113 |
-
- `voiceStore.ts` - Voice input state
|
114 |
-
|
115 |
-
## **6. Styling Guide**
|
116 |
-
|
117 |
-
The UI uses a cyberpunk-retro aesthetic:
|
118 |
-
|
119 |
-
### **Color Palette**
|
120 |
-
- DigiPal Orange: `#FF6B00`
|
121 |
-
- DigiPal Teal: `#00CED1`
|
122 |
-
- DigiPal Gray: `#2D2D2D`
|
123 |
-
- Neon Magenta: `#FF00FF`
|
124 |
-
- Neon Cyan: `#00FFFF`
|
125 |
-
|
126 |
-
### **Effects**
|
127 |
-
- CRT scanlines animation
|
128 |
-
- Holographic glitch effect
|
129 |
-
- Neon glow shadows
|
130 |
-
- Pixel fonts for UI text
|
131 |
-
|
132 |
-
## **7. MCP Integration**
|
133 |
-
|
134 |
-
When MCP is configured, the system will:
|
135 |
-
1. Use MCP for AI conversations instead of local Qwen
|
136 |
-
2. Use MCP for 3D generation instead of local pipeline
|
137 |
-
3. Optionally use MCP for speech-to-text
|
138 |
-
|
139 |
-
Configure via environment variables:
|
140 |
-
```bash
|
141 |
-
MCP_ENDPOINT=https://your-mcp-server.com
|
142 |
-
MCP_API_KEY=your-api-key
|
143 |
-
```
|
144 |
-
|
145 |
-
## **8. Development Tips**
|
146 |
-
|
147 |
-
### **Adding New Voice Commands**
|
148 |
-
1. Update intent parsing in `voice.ts`
|
149 |
-
2. Add handler in `voiceStore.ts`
|
150 |
-
3. Implement action in `monsterStore.ts`
|
151 |
-
|
152 |
-
### **Adding New UI Components**
|
153 |
-
1. Create component in `src/lib/components/`
|
154 |
-
2. Apply cyberpunk-retro styling
|
155 |
-
3. Connect to appropriate stores
|
156 |
-
|
157 |
-
### **Extending 3D Features**
|
158 |
-
1. Update `MonsterScene.svelte` for new animations
|
159 |
-
2. Add model loading logic
|
160 |
-
3. Implement interactive features
|
161 |
-
|
162 |
-
## **9. Production Deployment**
|
163 |
-
|
164 |
-
### **Build Process**
|
165 |
-
```bash
|
166 |
-
# Backend
|
167 |
-
docker build -t digipal .
|
168 |
-
|
169 |
-
# Frontend
|
170 |
-
cd frontend && npm run build
|
171 |
-
```
|
172 |
-
|
173 |
-
### **Environment Variables**
|
174 |
-
- `API_PORT`: Backend API port (default: 7861)
|
175 |
-
- `SERVER_PORT`: Gradio port (default: 7860)
|
176 |
-
- `MCP_ENDPOINT`: MCP server URL
|
177 |
-
- `MCP_API_KEY`: MCP authentication
|
178 |
-
|
179 |
-
## **10. Future Enhancements**
|
180 |
-
|
181 |
-
- [ ] PWA support for mobile devices
|
182 |
-
- [ ] Offline voice recognition
|
183 |
-
- [ ] Multiplayer monster interactions
|
184 |
-
- [ ] AR mode using device camera
|
185 |
-
- [ ] Custom shader effects for monsters
|
186 |
-
- [ ] Voice synthesis for monster responses
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
DEPLOYMENT.md
ADDED
@@ -0,0 +1,213 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# DigiPal Deployment Guide
|
2 |
+
|
3 |
+
## Quick Start
|
4 |
+
|
5 |
+
### Prerequisites
|
6 |
+
- Python 3.11+
|
7 |
+
- Node.js 18+ (for Svelte frontend, if using)
|
8 |
+
- Git
|
9 |
+
|
10 |
+
### Installation
|
11 |
+
|
12 |
+
1. **Clone the repository:**
|
13 |
+
```bash
|
14 |
+
git clone <repository-url>
|
15 |
+
cd digiPal
|
16 |
+
```
|
17 |
+
|
18 |
+
2. **Install Python dependencies:**
|
19 |
+
```bash
|
20 |
+
pip install -r requirements.txt
|
21 |
+
```
|
22 |
+
|
23 |
+
3. **Set up environment variables (optional):**
|
24 |
+
```bash
|
25 |
+
export HF_TOKEN="your_huggingface_token" # For private models/spaces
|
26 |
+
export MCP_ENDPOINT="your_mcp_endpoint" # For MCP integration
|
27 |
+
export MCP_API_KEY="your_mcp_key"
|
28 |
+
```
|
29 |
+
|
30 |
+
### Running DigiPal
|
31 |
+
|
32 |
+
#### Option 1: Complete Application (Recommended)
|
33 |
+
```bash
|
34 |
+
python run_digipal.py
|
35 |
+
```
|
36 |
+
This starts both the FastAPI backend and Streamlit frontend.
|
37 |
+
|
38 |
+
**Access:**
|
39 |
+
- **Streamlit UI**: http://localhost:8501
|
40 |
+
- **API Backend**: http://localhost:7861
|
41 |
+
|
42 |
+
#### Option 2: Manual Startup
|
43 |
+
Terminal 1 (Backend):
|
44 |
+
```bash
|
45 |
+
python app.py
|
46 |
+
```
|
47 |
+
|
48 |
+
Terminal 2 (Frontend):
|
49 |
+
```bash
|
50 |
+
streamlit run src/ui/streamlit_interface.py
|
51 |
+
```
|
52 |
+
|
53 |
+
#### Option 3: Svelte Frontend (Advanced)
|
54 |
+
```bash
|
55 |
+
# Terminal 1: Start backend
|
56 |
+
python app.py
|
57 |
+
|
58 |
+
# Terminal 2: Start Svelte frontend
|
59 |
+
cd frontend
|
60 |
+
npm install
|
61 |
+
npm run dev
|
62 |
+
```
|
63 |
+
|
64 |
+
## Architecture Overview
|
65 |
+
|
66 |
+
### Technology Stack
|
67 |
+
- **Frontend**: Streamlit (modern cyberpunk UI)
|
68 |
+
- **Backend**: FastAPI with WebSocket support
|
69 |
+
- **AI Models**:
|
70 |
+
- Qwen 2.5-1.5B-Instruct (conversations)
|
71 |
+
- Kyutai STT-2.6b-en (speech recognition)
|
72 |
+
- **3D Pipeline**: OmniGen2 → Hunyuan3D-2.1 → UniRig
|
73 |
+
- **Database**: SQLite with async operations
|
74 |
+
|
75 |
+
### API Endpoints
|
76 |
+
|
77 |
+
**Monster Management:**
|
78 |
+
- `GET /api/monsters` - List all monsters
|
79 |
+
- `POST /api/monsters` - Create new monster
|
80 |
+
- `GET /api/monsters/{id}` - Get monster details
|
81 |
+
- `POST /api/monsters/{id}/action` - Perform care action
|
82 |
+
- `POST /api/monsters/{id}/talk` - Send message to monster
|
83 |
+
- `POST /api/monsters/{id}/generate-3d` - Generate 3D model
|
84 |
+
|
85 |
+
**WebSocket:**
|
86 |
+
- `WS /api/monsters/{id}/ws` - Real-time updates
|
87 |
+
|
88 |
+
## Configuration
|
89 |
+
|
90 |
+
### Environment Variables
|
91 |
+
|
92 |
+
| Variable | Description | Default |
|
93 |
+
|----------|-------------|---------|
|
94 |
+
| `LOG_LEVEL` | Logging level | `INFO` |
|
95 |
+
| `API_PORT` | FastAPI backend port | `7861` |
|
96 |
+
| `HF_TOKEN` | HuggingFace API token | None |
|
97 |
+
| `MCP_ENDPOINT` | MCP service endpoint | None |
|
98 |
+
| `MCP_API_KEY` | MCP API key | None |
|
99 |
+
|
100 |
+
### Hardware Requirements
|
101 |
+
|
102 |
+
**Minimum:**
|
103 |
+
- 8GB RAM
|
104 |
+
- 4GB free disk space
|
105 |
+
- Internet connection (for HuggingFace APIs)
|
106 |
+
|
107 |
+
**Recommended:**
|
108 |
+
- 16GB RAM
|
109 |
+
- NVIDIA GPU with 8GB+ VRAM
|
110 |
+
- SSD storage
|
111 |
+
- High-speed internet
|
112 |
+
|
113 |
+
## 3D Generation Pipeline
|
114 |
+
|
115 |
+
The application uses a modern 3D generation pipeline:
|
116 |
+
|
117 |
+
1. **Text Input** → User describes their monster
|
118 |
+
2. **OmniGen2** → Generates multi-view images
|
119 |
+
3. **Hunyuan3D-2.1** → Converts images to 3D mesh
|
120 |
+
4. **UniRig** → Automatically rigs the 3D model
|
121 |
+
5. **Output** → Fully rigged 3D model ready for animation
|
122 |
+
|
123 |
+
### API Integration
|
124 |
+
- **OmniGen2**: Via transformers/diffusers pipeline
|
125 |
+
- **Hunyuan3D-2.1**: Via official HuggingFace Space API
|
126 |
+
- **UniRig**: Via HuggingFace model repository
|
127 |
+
|
128 |
+
## Deployment Options
|
129 |
+
|
130 |
+
### Local Development
|
131 |
+
Use the quick start guide above.
|
132 |
+
|
133 |
+
### Docker (Future)
|
134 |
+
```bash
|
135 |
+
docker build -t digipal .
|
136 |
+
docker run -p 7861:7861 -p 8501:8501 digipal
|
137 |
+
```
|
138 |
+
|
139 |
+
### HuggingFace Spaces
|
140 |
+
1. Fork/upload repository to HuggingFace Spaces
|
141 |
+
2. Set Space type to "Streamlit"
|
142 |
+
3. Configure secrets for HF_TOKEN if needed
|
143 |
+
4. Space will auto-deploy
|
144 |
+
|
145 |
+
## Troubleshooting
|
146 |
+
|
147 |
+
### Common Issues
|
148 |
+
|
149 |
+
**Port Already in Use:**
|
150 |
+
```bash
|
151 |
+
# Change ports
|
152 |
+
API_PORT=8081 python app.py
|
153 |
+
streamlit run src/ui/streamlit_interface.py --server.port 8502
|
154 |
+
```
|
155 |
+
|
156 |
+
**Missing Dependencies:**
|
157 |
+
```bash
|
158 |
+
pip install -r requirements.txt --upgrade
|
159 |
+
```
|
160 |
+
|
161 |
+
**3D Generation Fails:**
|
162 |
+
- Check internet connection
|
163 |
+
- Verify HF_TOKEN if using private models
|
164 |
+
- Pipeline includes fallback mechanisms
|
165 |
+
|
166 |
+
**Streamlit Not Starting:**
|
167 |
+
```bash
|
168 |
+
pip install streamlit --upgrade
|
169 |
+
streamlit --version
|
170 |
+
```
|
171 |
+
|
172 |
+
### Performance Optimization
|
173 |
+
|
174 |
+
**For GPU Systems:**
|
175 |
+
- Ensure CUDA is properly installed
|
176 |
+
- Models will automatically use GPU when available
|
177 |
+
|
178 |
+
**For CPU-Only Systems:**
|
179 |
+
- Increase timeout values for 3D generation
|
180 |
+
- Consider using smaller model variants
|
181 |
+
|
182 |
+
## Monitoring
|
183 |
+
|
184 |
+
### Logs
|
185 |
+
- Application logs: `logs/digipal.log`
|
186 |
+
- Streamlit logs: Console output
|
187 |
+
- FastAPI logs: Console output with timestamps
|
188 |
+
|
189 |
+
### Health Check
|
190 |
+
```bash
|
191 |
+
curl http://localhost:7861/health
|
192 |
+
```
|
193 |
+
|
194 |
+
## Support
|
195 |
+
|
196 |
+
For issues and questions:
|
197 |
+
1. Check this deployment guide
|
198 |
+
2. Review `CLAUDE.md` for development details
|
199 |
+
3. Check console logs for error messages
|
200 |
+
|
201 |
+
## New Tech Stack Summary
|
202 |
+
|
203 |
+
**Replaced:**
|
204 |
+
- Gradio → Streamlit (modern UI)
|
205 |
+
- Faster Whisper → Kyutai STT-2.6b-en (better accuracy)
|
206 |
+
- Complex 3D pipeline → Streamlined OmniGen2→Hunyuan3D→UniRig
|
207 |
+
|
208 |
+
**Benefits:**
|
209 |
+
- Modern, responsive UI with cyberpunk theme
|
210 |
+
- Better speech recognition quality
|
211 |
+
- State-of-the-art 3D generation pipeline
|
212 |
+
- Simplified deployment and maintenance
|
213 |
+
- Better separation of frontend/backend
|
DIGIPAL_V2_GUIDE.md
DELETED
@@ -1,355 +0,0 @@
|
|
1 |
-
# DigiPal V2 - Complete Development Guide
|
2 |
-
|
3 |
-
## Overview
|
4 |
-
|
5 |
-
DigiPal V2 is a revolutionary digital monster companion application that combines:
|
6 |
-
- **Authentic Digimon World 1 mechanics** based on comprehensive reverse engineering
|
7 |
-
- **Modern AI conversations** using Qwen 2.5 models
|
8 |
-
- **Optional 3D generation** using cutting-edge open-source pipeline (Flux → Sparc3D → UniRig)
|
9 |
-
- **Real-time care simulation** with mortality and complex evolution
|
10 |
-
|
11 |
-
## Architecture
|
12 |
-
|
13 |
-
### Core Components
|
14 |
-
|
15 |
-
```
|
16 |
-
digiPal/
|
17 |
-
├── src/
|
18 |
-
│ ├── core/
|
19 |
-
│ │ ├── monster_engine_dw1.py # DW1-accurate monster mechanics
|
20 |
-
│ │ └── monster_3d_integration.py # 3D model management
|
21 |
-
│ ├── pipelines/
|
22 |
-
│ │ ├── text_to_3d_pipeline.py # Commercial 3D pipeline (Meshy AI)
|
23 |
-
│ │ └── opensource_3d_pipeline_v2.py # Open-source pipeline (Flux/Sparc3D/UniRig)
|
24 |
-
│ ├── ui/
|
25 |
-
│ │ └── gradio_interface_v2.py # Enhanced Gradio interface
|
26 |
-
│ └── ai/
|
27 |
-
│ └── qwen_processor.py # AI conversation system
|
28 |
-
├── app_v2.py # Main application entry
|
29 |
-
└── README.md # HuggingFace Spaces config
|
30 |
-
```
|
31 |
-
|
32 |
-
## Key Features Implementation
|
33 |
-
|
34 |
-
### 1. DW1-Accurate Monster System
|
35 |
-
|
36 |
-
The `DW1Monster` class implements authentic mechanics:
|
37 |
-
|
38 |
-
```python
|
39 |
-
# Lifespan system with mortality
|
40 |
-
LIFESPAN_DAYS = {
|
41 |
-
"BABY": 1, "CHILD": 3, "ADULT": 5,
|
42 |
-
"PERFECT": 6, "ULTIMATE": 7, "MEGA": 8
|
43 |
-
}
|
44 |
-
|
45 |
-
# Complex stat system
|
46 |
-
- HP/MP (max 9999)
|
47 |
-
- Offense/Defense/Speed/Brains (max 999)
|
48 |
-
- Weight (5-99, affects evolution)
|
49 |
-
- Nature (0-100, affects techniques)
|
50 |
-
|
51 |
-
# Care mechanics
|
52 |
-
- Hunger (depletes faster during activity)
|
53 |
-
- Energy (restored by sleep)
|
54 |
-
- Toilet needs (every 3 hours)
|
55 |
-
- Happiness (affects training)
|
56 |
-
- Discipline (affects obedience)
|
57 |
-
- Sickness system (Cold, Injury, Fatigue, Stomach)
|
58 |
-
|
59 |
-
# Evolution system
|
60 |
-
- Time-based progression
|
61 |
-
- Stat requirements
|
62 |
-
- Care quality thresholds
|
63 |
-
- Weight requirements
|
64 |
-
- Special conditions (perfect week, tournament wins)
|
65 |
-
```
|
66 |
-
|
67 |
-
### 2. Training System
|
68 |
-
|
69 |
-
Based on DW1's gym mechanics:
|
70 |
-
```python
|
71 |
-
# Training types match DW1 locations
|
72 |
-
- HP Training (Green Gym)
|
73 |
-
- MP Training (Beetle Land)
|
74 |
-
- Offense Training (Dojo)
|
75 |
-
- Defense Training (Ice Gym)
|
76 |
-
- Speed Training (Speed Gym)
|
77 |
-
- Brains Training (Library)
|
78 |
-
|
79 |
-
# Performance-based gains
|
80 |
-
- PERFECT: 150% gain
|
81 |
-
- GREAT: 120% gain
|
82 |
-
- GOOD: 100% gain
|
83 |
-
- MISS: 50% gain + care mistake
|
84 |
-
```
|
85 |
-
|
86 |
-
### 3. Evolution Mechanics
|
87 |
-
|
88 |
-
Implements DW1's complex branching:
|
89 |
-
```python
|
90 |
-
# Baby → Child (6 hours)
|
91 |
-
- Personality-based (Brave→Agumon, Calm→Gabumon)
|
92 |
-
- Care quality affects outcome
|
93 |
-
|
94 |
-
# Child → Adult (24 hours)
|
95 |
-
- Stat requirements (total > 200)
|
96 |
-
- Weight requirements (15-35)
|
97 |
-
- Care mistakes < 5
|
98 |
-
- Highest stat determines form
|
99 |
-
|
100 |
-
# Adult → Perfect (72 hours)
|
101 |
-
- High stats (total > 500)
|
102 |
-
- Excellent care (mistakes < 3)
|
103 |
-
- Battle experience (15+ wins)
|
104 |
-
|
105 |
-
# Perfect → Ultimate (96 hours)
|
106 |
-
- Perfect care week required
|
107 |
-
- Very high stats (total > 800)
|
108 |
-
- Extensive battles (30+ wins)
|
109 |
-
```
|
110 |
-
|
111 |
-
### 4. 3D Generation Pipeline
|
112 |
-
|
113 |
-
Two implementation options:
|
114 |
-
|
115 |
-
#### Commercial Pipeline (Meshy AI)
|
116 |
-
```python
|
117 |
-
# Fast, high-quality, API-based
|
118 |
-
- Text → Meshy AI → 3D Model
|
119 |
-
- 2-12 credits per generation
|
120 |
-
- Built-in rigging support
|
121 |
-
- PBR textures included
|
122 |
-
```
|
123 |
-
|
124 |
-
#### Open-Source Pipeline (Production Ready)
|
125 |
-
```python
|
126 |
-
# Free, cutting-edge, locally runnable
|
127 |
-
1. Text → Flux (HF Spaces API)
|
128 |
-
- Multi-view generation (6 views)
|
129 |
-
- Consistent character design
|
130 |
-
|
131 |
-
2. Images → Sparc3D (1024³ resolution)
|
132 |
-
- Ultra-high quality mesh
|
133 |
-
- Texture projection
|
134 |
-
|
135 |
-
3. Mesh → UniRig (Auto-rigging)
|
136 |
-
- Procedural skeleton generation
|
137 |
-
- Weight painting
|
138 |
-
- Animation-ready output
|
139 |
-
```
|
140 |
-
|
141 |
-
## Setup Instructions
|
142 |
-
|
143 |
-
### Basic Setup (No 3D)
|
144 |
-
|
145 |
-
```bash
|
146 |
-
# Clone repository
|
147 |
-
git clone https://github.com/yourusername/digiPal
|
148 |
-
cd digiPal
|
149 |
-
|
150 |
-
# Install dependencies
|
151 |
-
pip install -r requirements.txt
|
152 |
-
|
153 |
-
# Run application
|
154 |
-
python app_v2.py
|
155 |
-
```
|
156 |
-
|
157 |
-
### Full Setup (With 3D)
|
158 |
-
|
159 |
-
```bash
|
160 |
-
# Install additional dependencies
|
161 |
-
pip install gradio_client trimesh pillow
|
162 |
-
|
163 |
-
# Clone required repositories
|
164 |
-
git clone https://github.com/lizhihao6/Sparc3D
|
165 |
-
git clone https://github.com/VAST-AI-Research/UniRig
|
166 |
-
|
167 |
-
# Configure API keys (optional)
|
168 |
-
export HF_TOKEN="your_huggingface_token"
|
169 |
-
|
170 |
-
# Run with 3D enabled
|
171 |
-
ENABLE_3D=true python app_v2.py
|
172 |
-
```
|
173 |
-
|
174 |
-
### HuggingFace Spaces Deployment
|
175 |
-
|
176 |
-
The application is optimized for Spaces:
|
177 |
-
|
178 |
-
1. **YAML Configuration** (in README.md):
|
179 |
-
```yaml
|
180 |
-
suggested_hardware: zero-a10g
|
181 |
-
suggested_storage: medium
|
182 |
-
```
|
183 |
-
|
184 |
-
2. **Environment Variables**:
|
185 |
-
```
|
186 |
-
ENABLE_3D=true
|
187 |
-
ENABLE_AI=true
|
188 |
-
LOG_LEVEL=INFO
|
189 |
-
```
|
190 |
-
|
191 |
-
## Usage Guide
|
192 |
-
|
193 |
-
### Creating Your First Monster
|
194 |
-
|
195 |
-
1. **Choose Species Type**: Affects evolution paths
|
196 |
-
- Data: Mechanical/Digital evolutions
|
197 |
-
- Vaccine: Holy/Light evolutions
|
198 |
-
- Virus: Dark/Chaos evolutions
|
199 |
-
- Free: Balanced evolutions
|
200 |
-
|
201 |
-
2. **Select Personality**: Affects behavior and growth
|
202 |
-
- Brave: +Offense, -Defense
|
203 |
-
- Calm: +Defense, -Speed
|
204 |
-
- Energetic: +Speed, -MP
|
205 |
-
- Clever: +Brains/MP, -HP
|
206 |
-
- Friendly: Balanced, easier care
|
207 |
-
|
208 |
-
### Care Guidelines
|
209 |
-
|
210 |
-
1. **Feeding**:
|
211 |
-
- Feed when hunger > 50
|
212 |
-
- Don't overfeed (causes care mistakes)
|
213 |
-
- Different foods have different effects
|
214 |
-
|
215 |
-
2. **Training**:
|
216 |
-
- Train when energy > 50
|
217 |
-
- Match training to desired evolution
|
218 |
-
- Perfect performance gives bonus gains
|
219 |
-
|
220 |
-
3. **Health**:
|
221 |
-
- Check toilet needs every 3 hours
|
222 |
-
- Heal sickness immediately
|
223 |
-
- Let monster sleep at night (8PM-8AM)
|
224 |
-
|
225 |
-
### Evolution Tips
|
226 |
-
|
227 |
-
1. **For Strong Evolutions**:
|
228 |
-
- Minimize care mistakes (< 3)
|
229 |
-
- Balanced stat growth
|
230 |
-
- Regular training
|
231 |
-
- Win battles
|
232 |
-
|
233 |
-
2. **Special Evolutions**:
|
234 |
-
- Perfect care for 7 days
|
235 |
-
- Win tournaments
|
236 |
-
- Specific stat distributions
|
237 |
-
- Weight requirements
|
238 |
-
|
239 |
-
## Technical Details
|
240 |
-
|
241 |
-
### Performance Optimization
|
242 |
-
|
243 |
-
1. **ZeroGPU Support**:
|
244 |
-
```python
|
245 |
-
@spaces.GPU(duration=60)
|
246 |
-
def generate_3d_model():
|
247 |
-
# GPU-intensive operations
|
248 |
-
```
|
249 |
-
|
250 |
-
2. **Async Operations**:
|
251 |
-
- Database operations
|
252 |
-
- 3D generation pipeline
|
253 |
-
- Multi-view image generation
|
254 |
-
|
255 |
-
3. **Memory Management**:
|
256 |
-
- Model quantization (8-bit)
|
257 |
-
- Texture compression
|
258 |
-
- Mesh decimation
|
259 |
-
|
260 |
-
### Customization Options
|
261 |
-
|
262 |
-
1. **Monster Definitions**:
|
263 |
-
Edit `monster_engine_dw1.py` to add:
|
264 |
-
- New evolution paths
|
265 |
-
- Custom techniques
|
266 |
-
- Special forms
|
267 |
-
|
268 |
-
2. **3D Styles**:
|
269 |
-
Modify prompts in `_generate_3d_prompt()`:
|
270 |
-
- Art style preferences
|
271 |
-
- Detail levels
|
272 |
-
- Color schemes
|
273 |
-
|
274 |
-
3. **UI Themes**:
|
275 |
-
Edit `CUSTOM_CSS` in interface:
|
276 |
-
- Color schemes
|
277 |
-
- Fonts
|
278 |
-
- Animations
|
279 |
-
|
280 |
-
## Troubleshooting
|
281 |
-
|
282 |
-
### Common Issues
|
283 |
-
|
284 |
-
1. **"No GPU detected"**
|
285 |
-
- Application works on CPU but slower
|
286 |
-
- 3D generation may timeout
|
287 |
-
|
288 |
-
2. **"3D pipeline not available"**
|
289 |
-
- Check Sparc3D/UniRig installation
|
290 |
-
- Verify CUDA installation
|
291 |
-
- Try commercial pipeline instead
|
292 |
-
|
293 |
-
3. **"Monster died unexpectedly"**
|
294 |
-
- Check care history
|
295 |
-
- Monitor lifespan remaining
|
296 |
-
- Reduce care mistakes
|
297 |
-
|
298 |
-
### Performance Tips
|
299 |
-
|
300 |
-
1. **For Faster 3D Generation**:
|
301 |
-
- Use "draft" quality
|
302 |
-
- Reduce texture resolution
|
303 |
-
- Enable model caching
|
304 |
-
|
305 |
-
2. **For Better Evolution Rates**:
|
306 |
-
- Train regularly but don't overtrain
|
307 |
-
- Keep happiness > 70
|
308 |
-
- Minimize toilet accidents
|
309 |
-
|
310 |
-
## Future Enhancements
|
311 |
-
|
312 |
-
### Planned Features
|
313 |
-
|
314 |
-
1. **Battle System**:
|
315 |
-
- Turn-based combat
|
316 |
-
- Technique learning
|
317 |
-
- PvP battles
|
318 |
-
|
319 |
-
2. **World Exploration**:
|
320 |
-
- File City building
|
321 |
-
- NPC recruitment
|
322 |
-
- Item collection
|
323 |
-
|
324 |
-
3. **Breeding System**:
|
325 |
-
- Genetic inheritance
|
326 |
-
- Hybrid forms
|
327 |
-
- Egg management
|
328 |
-
|
329 |
-
4. **Mini-Games**:
|
330 |
-
- Training challenges
|
331 |
-
- Fishing
|
332 |
-
- Treasure hunting
|
333 |
-
|
334 |
-
## Credits
|
335 |
-
|
336 |
-
- **DW1 Reverse Engineering**: SydMontague, Vicen04, Romsstar
|
337 |
-
- **3D Pipeline**: Flux (Black Forest Labs), Sparc3D (lizhihao6), UniRig (VAST-AI)
|
338 |
-
- **AI Models**: Qwen 2.5 (Alibaba)
|
339 |
-
- **Framework**: Gradio (HuggingFace)
|
340 |
-
|
341 |
-
## Philosophy
|
342 |
-
|
343 |
-
Built following Rick Rubin's creative philosophy:
|
344 |
-
- **Strip to essentials**: Focus on core monster-raising mechanics
|
345 |
-
- **Amplify emotion**: Every interaction should create connection
|
346 |
-
- **Respect the source**: Authentic to Digimon World 1's spirit
|
347 |
-
- **Push boundaries**: Modern tech serving classic gameplay
|
348 |
-
|
349 |
-
## License
|
350 |
-
|
351 |
-
MIT License - See LICENSE file for details
|
352 |
-
|
353 |
-
---
|
354 |
-
|
355 |
-
*"The goal is not to recreate the past, but to capture what made it special and express it through modern possibilities."*
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
Dockerfile
CHANGED
@@ -27,12 +27,12 @@ COPY . .
|
|
27 |
# Create necessary directories
|
28 |
RUN mkdir -p data/saves data/models data/cache logs config
|
29 |
|
30 |
-
# Expose ports for
|
31 |
-
EXPOSE
|
32 |
|
33 |
# Health check - check API server on port 7861
|
34 |
HEALTHCHECK --interval=30s --timeout=30s --start-period=60s --retries=3 \
|
35 |
CMD curl -f http://localhost:7861/health || exit 1
|
36 |
|
37 |
-
# Run the application
|
38 |
-
CMD ["python", "
|
|
|
27 |
# Create necessary directories
|
28 |
RUN mkdir -p data/saves data/models data/cache logs config
|
29 |
|
30 |
+
# Expose ports for FastAPI backend and Streamlit frontend
|
31 |
+
EXPOSE 7861 8501
|
32 |
|
33 |
# Health check - check API server on port 7861
|
34 |
HEALTHCHECK --interval=30s --timeout=30s --start-period=60s --retries=3 \
|
35 |
CMD curl -f http://localhost:7861/health || exit 1
|
36 |
|
37 |
+
# Run the complete application (FastAPI + Streamlit)
|
38 |
+
CMD ["python", "run_digipal.py"]
|
QUICK_START.md
DELETED
@@ -1,169 +0,0 @@
|
|
1 |
-
# DigiPal Quick Start Guide
|
2 |
-
|
3 |
-
## 🚀 Current Status: FULLY FUNCTIONAL
|
4 |
-
|
5 |
-
**✅ All major issues resolved!** The application is now ready for deployment.
|
6 |
-
|
7 |
-
## 📋 What's Working Right Now
|
8 |
-
|
9 |
-
### **✅ Core Systems**
|
10 |
-
- Monster creation and management
|
11 |
-
- Evolution system with requirements
|
12 |
-
- State persistence (SQLite database)
|
13 |
-
- Performance tracking
|
14 |
-
- Zero GPU optimization
|
15 |
-
|
16 |
-
### **✅ AI Systems**
|
17 |
-
- Qwen 2.5 text generation (with fallbacks)
|
18 |
-
- Speech processing (voice-to-text)
|
19 |
-
- Multiple 3D generation pipelines
|
20 |
-
- Emotional impact calculation
|
21 |
-
|
22 |
-
### **✅ Deployment Ready**
|
23 |
-
- Hugging Face Spaces compatible
|
24 |
-
- Zero GPU support
|
25 |
-
- Automatic resource detection
|
26 |
-
- Graceful fallbacks
|
27 |
-
|
28 |
-
### **✅ Interfaces**
|
29 |
-
- FastAPI backend (Port 7861)
|
30 |
-
- Gradio admin panel (Port 7860)
|
31 |
-
- WebSocket real-time updates
|
32 |
-
- RESTful API endpoints
|
33 |
-
|
34 |
-
## 🎯 How to Use Right Now
|
35 |
-
|
36 |
-
### **1. Start the Application**
|
37 |
-
```bash
|
38 |
-
python app.py
|
39 |
-
```
|
40 |
-
|
41 |
-
### **2. Access the Interfaces**
|
42 |
-
- **Gradio Admin**: http://localhost:7860
|
43 |
-
- **API Documentation**: http://localhost:7861/docs
|
44 |
-
- **Health Check**: http://localhost:7861/health
|
45 |
-
|
46 |
-
### **3. Create Your First Monster**
|
47 |
-
1. Go to http://localhost:7860
|
48 |
-
2. Click "Create New Monster"
|
49 |
-
3. Enter name and personality
|
50 |
-
4. Start interacting!
|
51 |
-
|
52 |
-
## 🔧 What Each Port Does
|
53 |
-
|
54 |
-
| Port | Service | Purpose |
|
55 |
-
|------|---------|---------|
|
56 |
-
| 7860 | Gradio | Admin panel, monster creation, debugging |
|
57 |
-
| 7861 | FastAPI | REST API, WebSocket, 3D generation |
|
58 |
-
| 5173 | Svelte | Modern web UI (when implemented) |
|
59 |
-
|
60 |
-
## 🎮 Current Features
|
61 |
-
|
62 |
-
### **Monster Management**
|
63 |
-
- ✅ Create monsters with personalities
|
64 |
-
- ✅ Feed, train, and care for monsters
|
65 |
-
- ✅ Real-time stat updates
|
66 |
-
- ✅ Evolution tracking
|
67 |
-
- ✅ Conversation history
|
68 |
-
|
69 |
-
### **AI Interactions**
|
70 |
-
- ✅ Text conversations with AI responses
|
71 |
-
- ✅ Personality-aware responses
|
72 |
-
- ✅ Emotional impact on monster stats
|
73 |
-
- ✅ Fallback responses when AI unavailable
|
74 |
-
|
75 |
-
### **3D Generation**
|
76 |
-
- ✅ Multiple pipeline support
|
77 |
-
- ✅ Hugging Face Spaces integration
|
78 |
-
- ✅ Automatic model optimization
|
79 |
-
- ✅ Texture generation
|
80 |
-
|
81 |
-
### **System Features**
|
82 |
-
- ✅ Automatic saving
|
83 |
-
- ✅ Performance monitoring
|
84 |
-
- ✅ Error handling
|
85 |
-
- ✅ Resource optimization
|
86 |
-
|
87 |
-
## 🚀 Next Steps
|
88 |
-
|
89 |
-
### **Immediate (Ready Now)**
|
90 |
-
1. **Deploy to Hugging Face Spaces**
|
91 |
-
```bash
|
92 |
-
git add .
|
93 |
-
git commit -m "Ready for deployment"
|
94 |
-
git push origin main
|
95 |
-
```
|
96 |
-
|
97 |
-
2. **Test on Spaces**
|
98 |
-
- Monitor logs for any issues
|
99 |
-
- Verify GPU allocation
|
100 |
-
- Test monster creation
|
101 |
-
|
102 |
-
### **Future Enhancements**
|
103 |
-
1. **Svelte Frontend** (Port 5173)
|
104 |
-
- Modern web interface
|
105 |
-
- 3D model viewer
|
106 |
-
- Voice chat interface
|
107 |
-
|
108 |
-
2. **Advanced Features**
|
109 |
-
- Mini-games implementation
|
110 |
-
- Breeding system
|
111 |
-
- Advanced evolution paths
|
112 |
-
|
113 |
-
## 🔍 Troubleshooting
|
114 |
-
|
115 |
-
### **Common Issues**
|
116 |
-
1. **Import Errors**: All dependencies installed ✅
|
117 |
-
2. **Dataclass Errors**: Fixed with `field(default_factory=...)` ✅
|
118 |
-
3. **GPU Issues**: Zero GPU optimization implemented ✅
|
119 |
-
4. **Port Conflicts**: All ports properly configured ✅
|
120 |
-
|
121 |
-
### **If Something Goes Wrong**
|
122 |
-
1. Check logs in terminal output
|
123 |
-
2. Verify all dependencies: `pip install -r requirements.txt`
|
124 |
-
3. Clear cache: `rm -rf data/cache/*`
|
125 |
-
4. Restart: `python app.py`
|
126 |
-
|
127 |
-
## 📊 Performance Notes
|
128 |
-
|
129 |
-
### **Local Development**
|
130 |
-
- Fast startup (< 30 seconds)
|
131 |
-
- Low memory usage
|
132 |
-
- CPU-optimized AI models
|
133 |
-
- Real-time responses
|
134 |
-
|
135 |
-
### **Hugging Face Spaces**
|
136 |
-
- Automatic GPU detection
|
137 |
-
- Memory optimization
|
138 |
-
- Graceful CPU fallback
|
139 |
-
- Spaces.GPU decorators applied
|
140 |
-
|
141 |
-
## 🎯 Key Achievements
|
142 |
-
|
143 |
-
### **✅ Unified Architecture**
|
144 |
-
- Single entry point (`app.py`)
|
145 |
-
- Shared state across components
|
146 |
-
- Consistent data flow
|
147 |
-
- Modular design
|
148 |
-
|
149 |
-
### **✅ Production Ready**
|
150 |
-
- Error handling
|
151 |
-
- Logging
|
152 |
-
- Performance monitoring
|
153 |
-
- Graceful fallbacks
|
154 |
-
|
155 |
-
### **✅ Zero GPU Compatible**
|
156 |
-
- Dynamic resource detection
|
157 |
-
- CPU optimization
|
158 |
-
- Memory management
|
159 |
-
- Spaces integration
|
160 |
-
|
161 |
-
## 🚀 Ready for Deployment!
|
162 |
-
|
163 |
-
The application is now **fully functional** and ready for:
|
164 |
-
- ✅ Local development
|
165 |
-
- ✅ Hugging Face Spaces deployment
|
166 |
-
- ✅ Production use
|
167 |
-
- ✅ Further development
|
168 |
-
|
169 |
-
**All the effort has matured the codebase into a robust, scalable, and maintainable system!** 🎉
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
QUICK_UI_TEST.md
ADDED
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
# 🎨 Quick UI Test Guide
|
2 |
+
|
3 |
+
## See the New DigiPal UI Now!
|
4 |
+
|
5 |
+
### Option 1: UI Only Preview (Fastest)
|
6 |
+
```bash
|
7 |
+
python test_ui.py
|
8 |
+
```
|
9 |
+
This shows you the new cyberpunk Streamlit interface without needing the backend.
|
10 |
+
|
11 |
+
### Option 2: Full Application
|
12 |
+
```bash
|
13 |
+
python run_digipal.py
|
14 |
+
```
|
15 |
+
This runs both backend and frontend for full functionality.
|
16 |
+
|
17 |
+
## What You'll See
|
18 |
+
|
19 |
+
### 🎨 **Modern Cyberpunk Theme:**
|
20 |
+
- Dark gradient backgrounds with neon accents
|
21 |
+
- Glowing cyan and magenta color scheme
|
22 |
+
- Orbitron and Rajdhani fonts for sci-fi feel
|
23 |
+
- Animated neon effects on titles and buttons
|
24 |
+
|
25 |
+
### 🖥️ **Interface Features:**
|
26 |
+
- **Welcome Screen**: Feature overview with holographic styling
|
27 |
+
- **Sidebar**: Monster management with neon buttons
|
28 |
+
- **Monster Stats**: Holographic containers with progress bars
|
29 |
+
- **Chat Interface**: Cyberpunk-styled conversation area
|
30 |
+
- **3D Generation**: Modern controls for model creation
|
31 |
+
|
32 |
+
### 🚀 **Interactive Elements:**
|
33 |
+
- Hover effects on buttons with glow animations
|
34 |
+
- Gradient backgrounds that shift and pulse
|
35 |
+
- Neon text effects with shadows
|
36 |
+
- Holographic containers with backdrop blur
|
37 |
+
|
38 |
+
## Access URLs
|
39 |
+
|
40 |
+
After starting:
|
41 |
+
- **Streamlit UI**: http://localhost:8501
|
42 |
+
- **API Backend**: http://localhost:7861 (if running full app)
|
43 |
+
|
44 |
+
## Notes
|
45 |
+
|
46 |
+
- The UI test mode shows the interface but backend features won't work
|
47 |
+
- Create a monster in the sidebar to see the full interface
|
48 |
+
- All the cyberpunk styling and animations will be visible
|
49 |
+
- The design is optimized for both desktop and tablet viewing
|
50 |
+
|
51 |
+
## Troubleshooting
|
52 |
+
|
53 |
+
**If Streamlit won't start:**
|
54 |
+
```bash
|
55 |
+
pip install streamlit --upgrade
|
56 |
+
```
|
57 |
+
|
58 |
+
**If you see port conflicts:**
|
59 |
+
```bash
|
60 |
+
STREAMLIT_PORT=8502 python test_ui.py
|
61 |
+
```
|
62 |
+
|
63 |
+
Enjoy the new futuristic DigiPal experience! 🐉✨
|
README.md
CHANGED
@@ -1,16 +1,19 @@
|
|
1 |
---
|
2 |
title: DigiPal Advanced Monster Companion
|
3 |
-
emoji:
|
4 |
colorFrom: purple
|
5 |
colorTo: blue
|
6 |
-
sdk:
|
7 |
-
sdk_version:
|
8 |
-
app_file:
|
9 |
pinned: false
|
10 |
license: mit
|
11 |
models:
|
12 |
- Qwen/Qwen2.5-1.5B-Instruct
|
13 |
-
-
|
|
|
|
|
|
|
14 |
datasets: []
|
15 |
tags:
|
16 |
- gaming
|
@@ -20,147 +23,105 @@ tags:
|
|
20 |
- speech-recognition
|
21 |
- 3d-generation
|
22 |
- text-to-3d
|
|
|
|
|
23 |
suggested_hardware: zero-a10g
|
24 |
suggested_storage: medium
|
25 |
---
|
26 |
|
27 |
-
#
|
28 |
-
|
29 |
-
The next generation of virtual monster companions powered by **Qwen 2.5**, **Whisper**, advanced AI technologies, and **3D model generation**. Experience deep emotional connections with your digital pet through natural conversation, comprehensive care systems, sophisticated evolution mechanics, and bring your monsters to life in 3D!
|
30 |
-
|
31 |
-
## ✨ Features
|
32 |
-
|
33 |
-
### 🧠 Advanced AI Personality System
|
34 |
-
- **Qwen 2.5-powered conversations** with contextual memory
|
35 |
-
- **Dynamic personality traits** that evolve with care
|
36 |
-
- **Emotional state recognition** and appropriate responses
|
37 |
-
- **Voice chat support** with Whisper speech recognition
|
38 |
-
|
39 |
-
### 🎮 Comprehensive Monster Care
|
40 |
-
- **Six-dimensional care system** (health, happiness, hunger, energy, discipline, cleanliness)
|
41 |
-
- **Real-time stat degradation** that continues even when offline
|
42 |
-
- **Complex evolution requirements** inspired by classic monster-raising games
|
43 |
-
- **Training mini-games** that affect monster development
|
44 |
-
- **DW1-aligned mechanics** option for authentic Digimon World 1 experience
|
45 |
-
|
46 |
-
### 🎨 3D Model Generation (V2)
|
47 |
-
- **Text-to-3D conversion** using Hunyuan3D and open-source models
|
48 |
-
- **Multiple model providers** including HuggingFace, local models, and MCP protocol
|
49 |
-
- **Real-time 3D visualization** of your monster companions
|
50 |
-
- **Async generation pipeline** for smooth user experience
|
51 |
-
- **Model caching** for efficient reuse of generated assets
|
52 |
-
|
53 |
-
### 🌟 Next-Generation Features
|
54 |
-
- **Cross-session persistence** with browser state management
|
55 |
-
- **Real-time streaming updates** using Gradio 5.34.2
|
56 |
-
- **Zero GPU optimization** for efficient resource usage
|
57 |
-
- **Advanced breeding system** with genetic inheritance
|
58 |
-
- **MCP (Model Context Protocol)** support for flexible model deployment
|
59 |
-
|
60 |
-
## 🚀 Technology Stack
|
61 |
-
|
62 |
-
- **HuggingFace Transformers v4.52.4** with Flash Attention 2
|
63 |
-
- **Gradio 5.34.2** with modern state management
|
64 |
-
- **Qwen 2.5 models** optimized for conversation
|
65 |
-
- **Faster Whisper** for efficient speech processing
|
66 |
-
- **Hunyuan3D** for high-quality 3D model generation
|
67 |
-
- **Zero GPU deployment** for scalable AI inference
|
68 |
-
- **Docker support** for containerized deployment
|
69 |
-
|
70 |
-
## 🎯 Getting Started
|
71 |
-
|
72 |
-
### Option 1: Basic Version (V1)
|
73 |
-
1. **Create Your Monster**: Choose a name and personality type
|
74 |
-
2. **Start Caring**: Feed, train, and interact with your companion
|
75 |
-
3. **Build Relationships**: Use voice or text chat to bond
|
76 |
-
4. **Watch Evolution**: Meet requirements to unlock new forms
|
77 |
-
|
78 |
-
### Option 2: Enhanced Version with 3D (V2)
|
79 |
-
1. **All V1 features** plus:
|
80 |
-
2. **Generate 3D Models**: Create visual representations of your monsters
|
81 |
-
3. **Customize Appearance**: Use text descriptions to shape your monster's look
|
82 |
-
4. **View in 3D**: Interact with generated 3D models in real-time
|
83 |
-
|
84 |
-
## 🛠️ Running Locally
|
85 |
-
|
86 |
-
### Requirements
|
87 |
-
- Python 3.11+
|
88 |
-
- CUDA-capable GPU (recommended) or CPU
|
89 |
-
- 8GB+ RAM (16GB+ recommended for 3D features)
|
90 |
-
|
91 |
-
### Installation
|
92 |
-
```bash
|
93 |
-
# Clone the repository
|
94 |
-
git clone https://github.com/yourusername/digipal.git
|
95 |
-
cd digipal
|
96 |
|
97 |
-
|
98 |
-
pip install -r requirements.txt
|
99 |
|
100 |
-
|
101 |
-
python app.py
|
102 |
|
103 |
-
|
104 |
-
|
105 |
-
|
|
|
|
|
|
|
|
|
106 |
|
107 |
-
|
108 |
-
```bash
|
109 |
-
# Build the Docker image
|
110 |
-
docker build -t digipal .
|
111 |
|
112 |
-
|
113 |
-
|
114 |
-
|
|
|
|
|
|
|
115 |
|
116 |
-
###
|
117 |
-
|
118 |
-
|
119 |
-
|
|
|
120 |
|
121 |
-
|
122 |
-
ENABLE_3D=false python app_v2.py
|
123 |
|
124 |
-
|
125 |
-
SERVER_PORT=8080 SHARE=true python app.py
|
126 |
-
```
|
127 |
|
128 |
-
|
|
|
|
|
|
|
|
|
129 |
|
130 |
-
|
131 |
-
- **Balanced care** prevents evolution mistakes
|
132 |
-
- **Voice chat** creates deeper emotional connections
|
133 |
-
- **Training variety** unlocks special evolution paths
|
134 |
-
- **3D generation** works best with detailed descriptions
|
135 |
-
- **Save frequently** to preserve your monster's progress
|
136 |
|
137 |
-
|
|
|
|
|
|
|
|
|
138 |
|
139 |
-
|
140 |
-
DigiPal supports Model Context Protocol for flexible AI model deployment:
|
141 |
-
- Configure external model services via `MCP_ENDPOINT` and `MCP_API_KEY`
|
142 |
-
- Access various AI models through a standardized protocol
|
143 |
-
- Enable MCP server mode for integration with other tools
|
144 |
|
145 |
-
|
146 |
-
- **
|
147 |
-
- **
|
148 |
-
- **
|
|
|
149 |
|
150 |
-
##
|
151 |
|
152 |
-
-
|
153 |
-
-
|
154 |
-
-
|
|
|
|
|
155 |
|
156 |
-
##
|
157 |
|
158 |
-
|
|
|
|
|
|
|
|
|
159 |
|
160 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
161 |
|
162 |
-
This project
|
163 |
|
164 |
---
|
165 |
|
166 |
-
*Experience the future of AI
|
|
|
1 |
---
|
2 |
title: DigiPal Advanced Monster Companion
|
3 |
+
emoji: 🐉
|
4 |
colorFrom: purple
|
5 |
colorTo: blue
|
6 |
+
sdk: streamlit
|
7 |
+
sdk_version: 1.25.0
|
8 |
+
app_file: streamlit_app.py
|
9 |
pinned: false
|
10 |
license: mit
|
11 |
models:
|
12 |
- Qwen/Qwen2.5-1.5B-Instruct
|
13 |
+
- kyutai/stt-2.6b-en
|
14 |
+
- shitao/OmniGen-v1
|
15 |
+
- tencent/Hunyuan3D-2.1
|
16 |
+
- VAST-AI/UniRig
|
17 |
datasets: []
|
18 |
tags:
|
19 |
- gaming
|
|
|
23 |
- speech-recognition
|
24 |
- 3d-generation
|
25 |
- text-to-3d
|
26 |
+
- cyberpunk
|
27 |
+
- streamlit
|
28 |
suggested_hardware: zero-a10g
|
29 |
suggested_storage: medium
|
30 |
---
|
31 |
|
32 |
+
# 🐉 DigiPal - Advanced AI Monster Companion
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
33 |
|
34 |
+
**The most advanced AI-powered virtual monster companion with cutting-edge 3D generation!**
|
|
|
35 |
|
36 |
+
## 🚀 Revolutionary Features
|
|
|
37 |
|
38 |
+
- 🤖 **Advanced AI Conversations** with Qwen 2.5-1.5B-Instruct
|
39 |
+
- 🎤 **High-Quality Speech Recognition** with Kyutai STT-2.6b-en
|
40 |
+
- 🎨 **State-of-the-Art 3D Generation** via OmniGen2 → Hunyuan3D-2.1 → UniRig
|
41 |
+
- 📊 **Complex Care System** inspired by Digimon World mechanics
|
42 |
+
- 🧬 **Dynamic Evolution** based on care quality and interaction
|
43 |
+
- 💬 **Personality-Driven Responses** with emotional intelligence
|
44 |
+
- 🎮 **Cyberpunk UI** with neon effects and holographic styling
|
45 |
|
46 |
+
## 🛠️ Technology Stack
|
|
|
|
|
|
|
47 |
|
48 |
+
### AI Models
|
49 |
+
- **Conversations**: Qwen 2.5-1.5B-Instruct (quantized for efficiency)
|
50 |
+
- **Speech-to-Text**: Kyutai STT-2.6b-en (latest multilingual model)
|
51 |
+
- **Text-to-Image**: OmniGen2 (multi-view generation)
|
52 |
+
- **Image-to-3D**: Hunyuan3D-2.1 (official Tencent model)
|
53 |
+
- **3D Rigging**: UniRig (automatic model rigging)
|
54 |
|
55 |
+
### Architecture
|
56 |
+
- **Frontend**: Streamlit with cyberpunk theme
|
57 |
+
- **Backend**: Integrated FastAPI services
|
58 |
+
- **Database**: SQLite with async operations
|
59 |
+
- **3D Pipeline**: Complete text → image → 3D → rigged workflow
|
60 |
|
61 |
+
## 🎯 3D Generation Pipeline
|
|
|
62 |
|
63 |
+
The crown jewel of DigiPal is its revolutionary 3D generation system:
|
|
|
|
|
64 |
|
65 |
+
1. **Text Description** → User describes their monster
|
66 |
+
2. **OmniGen2** → Generates consistent multi-view images
|
67 |
+
3. **Hunyuan3D-2.1** → Converts images to high-quality 3D mesh
|
68 |
+
4. **UniRig** → Automatically rigs the model for animation
|
69 |
+
5. **Result** → Fully rigged 3D model ready for games/animation
|
70 |
|
71 |
+
## 🎮 How to Use
|
|
|
|
|
|
|
|
|
|
|
72 |
|
73 |
+
1. **Create Your Monster**: Choose name and personality type
|
74 |
+
2. **Care & Interact**: Feed, train, play, and talk with your companion
|
75 |
+
3. **Watch Evolution**: Your monster grows based on care quality
|
76 |
+
4. **Generate 3D Model**: Create a unique 3D representation
|
77 |
+
5. **Download & Use**: Get your rigged model for other applications
|
78 |
|
79 |
+
## 🎨 Monster Care System
|
|
|
|
|
|
|
|
|
80 |
|
81 |
+
- **Six Core Stats**: Health, Happiness, Hunger, Energy, Discipline, Cleanliness
|
82 |
+
- **Real-Time Degradation**: Stats change even when you're away
|
83 |
+
- **Evolution Stages**: Egg → Baby → Child → Adult → Champion → Ultimate
|
84 |
+
- **Personality Types**: Friendly, Energetic, Calm, Curious, Brave
|
85 |
+
- **Complex Requirements**: Age, level, care quality all matter
|
86 |
|
87 |
+
## 💫 Technical Highlights
|
88 |
|
89 |
+
- **Zero GPU Optimization**: Efficient model loading and inference
|
90 |
+
- **Graceful Fallbacks**: Pipeline continues even if some APIs fail
|
91 |
+
- **Real-Time Updates**: WebSocket integration for live stat changes
|
92 |
+
- **Model Caching**: Intelligent reuse of generated assets
|
93 |
+
- **Cross-Platform**: Works on desktop, tablet, and mobile
|
94 |
|
95 |
+
## 🔧 Development
|
96 |
|
97 |
+
### Local Setup
|
98 |
+
```bash
|
99 |
+
git clone <repository>
|
100 |
+
cd digiPal
|
101 |
+
pip install -r requirements.txt
|
102 |
|
103 |
+
# Run complete application
|
104 |
+
python run_digipal.py
|
105 |
+
|
106 |
+
# Or run Streamlit only
|
107 |
+
streamlit run streamlit_app.py
|
108 |
+
```
|
109 |
+
|
110 |
+
### Environment Variables
|
111 |
+
```bash
|
112 |
+
HF_TOKEN=your_token # For private models
|
113 |
+
MCP_ENDPOINT=your_endpoint # For MCP integration
|
114 |
+
LOG_LEVEL=INFO # Logging level
|
115 |
+
```
|
116 |
+
|
117 |
+
## 📝 License
|
118 |
+
|
119 |
+
MIT License - Feel free to use, modify, and distribute!
|
120 |
+
|
121 |
+
## 🤝 Contributing
|
122 |
|
123 |
+
Contributions welcome! This project pushes the boundaries of AI companions and 3D generation.
|
124 |
|
125 |
---
|
126 |
|
127 |
+
*Experience the future of AI companions with DigiPal! 🐉✨*
|
app.py
CHANGED
@@ -16,9 +16,7 @@ from fastapi import FastAPI, WebSocket, WebSocketDisconnect, HTTPException
|
|
16 |
from fastapi.middleware.cors import CORSMiddleware
|
17 |
from fastapi.responses import JSONResponse
|
18 |
from pydantic import BaseModel
|
19 |
-
import gradio as gr
|
20 |
import torch
|
21 |
-
import spaces
|
22 |
from contextlib import asynccontextmanager
|
23 |
|
24 |
# Add src to path
|
@@ -35,7 +33,7 @@ logger = logging.getLogger(__name__)
|
|
35 |
ENV_CONFIG = {
|
36 |
"LOG_LEVEL": os.getenv("LOG_LEVEL", "INFO"),
|
37 |
"SERVER_NAME": os.getenv("SERVER_NAME", "0.0.0.0"),
|
38 |
-
"
|
39 |
"API_PORT": int(os.getenv("API_PORT", "7861")),
|
40 |
"SHARE": os.getenv("SHARE", "false").lower() == "true",
|
41 |
"DEBUG": os.getenv("DEBUG", "false").lower() == "true",
|
@@ -71,14 +69,13 @@ try:
|
|
71 |
from src.ai.speech_engine import AdvancedSpeechEngine as SpeechEngine, SpeechConfig
|
72 |
from src.ui.state_manager import AdvancedStateManager as StateManager
|
73 |
from src.deployment.zero_gpu_optimizer import get_optimal_device
|
74 |
-
from src.pipelines.hunyuan3d_pipeline import Hunyuan3DClient
|
75 |
from src.pipelines.opensource_3d_pipeline_v2 import (
|
76 |
ProductionPipeline,
|
77 |
ProductionConfig
|
78 |
)
|
79 |
|
80 |
-
# UI imports
|
81 |
-
from src.ui.
|
82 |
except ImportError as e:
|
83 |
logger.error(f"Failed to import required modules: {e}")
|
84 |
sys.exit(1)
|
@@ -127,13 +124,14 @@ class AppState:
|
|
127 |
|
128 |
self.qwen_processor = QwenProcessor(qwen_config)
|
129 |
|
130 |
-
# Create speech engine config
|
131 |
speech_config = SpeechConfig(
|
132 |
-
|
133 |
device="auto", # Auto-detect device
|
134 |
-
|
135 |
use_vad=True,
|
136 |
-
vad_aggressiveness=2
|
|
|
137 |
)
|
138 |
|
139 |
self.speech_engine = SpeechEngine(speech_config)
|
@@ -432,11 +430,8 @@ async def websocket_endpoint(websocket: WebSocket, monster_id: str):
|
|
432 |
except WebSocketDisconnect:
|
433 |
manager.disconnect(monster_id)
|
434 |
|
435 |
-
#
|
436 |
-
|
437 |
-
"""Create Gradio interface as admin panel"""
|
438 |
-
interface = create_interface()
|
439 |
-
return interface
|
440 |
|
441 |
# Main entry point
|
442 |
if __name__ == "__main__":
|
@@ -451,46 +446,23 @@ if __name__ == "__main__":
|
|
451 |
logger.info("DigiPal - Advanced AI Monster Companion")
|
452 |
logger.info("=" * 60)
|
453 |
logger.info(f"Environment: {'HuggingFace Spaces' if IS_SPACES else 'Local'}")
|
454 |
-
logger.info(f"
|
455 |
-
logger.info(f"
|
456 |
logger.info(f"MCP Enabled: {bool(ENV_CONFIG['MCP_ENDPOINT'])}")
|
457 |
logger.info("=" * 60)
|
458 |
|
459 |
-
#
|
460 |
-
|
461 |
-
|
462 |
-
|
463 |
-
|
464 |
-
|
465 |
-
|
466 |
-
|
467 |
-
|
468 |
-
|
469 |
-
|
470 |
-
|
471 |
-
"share": ENV_CONFIG["SHARE"],
|
472 |
-
"max_threads": ENV_CONFIG["MAX_THREADS"],
|
473 |
-
"show_error": True,
|
474 |
-
"prevent_thread_lock": True # Important for running alongside FastAPI
|
475 |
-
}
|
476 |
-
)
|
477 |
-
gradio_thread.daemon = True
|
478 |
-
gradio_thread.start()
|
479 |
-
|
480 |
-
# Give Gradio a moment to start
|
481 |
-
await asyncio.sleep(2)
|
482 |
-
|
483 |
-
# Start FastAPI server
|
484 |
-
config = uvicorn.Config(
|
485 |
-
app,
|
486 |
-
host=ENV_CONFIG["SERVER_NAME"],
|
487 |
-
port=ENV_CONFIG["API_PORT"],
|
488 |
-
log_level=ENV_CONFIG["LOG_LEVEL"].lower()
|
489 |
-
)
|
490 |
-
server = uvicorn.Server(config)
|
491 |
-
|
492 |
-
# Run FastAPI server (this will block)
|
493 |
-
await server.serve()
|
494 |
|
495 |
-
# Run
|
496 |
-
asyncio.run(
|
|
|
16 |
from fastapi.middleware.cors import CORSMiddleware
|
17 |
from fastapi.responses import JSONResponse
|
18 |
from pydantic import BaseModel
|
|
|
19 |
import torch
|
|
|
20 |
from contextlib import asynccontextmanager
|
21 |
|
22 |
# Add src to path
|
|
|
33 |
ENV_CONFIG = {
|
34 |
"LOG_LEVEL": os.getenv("LOG_LEVEL", "INFO"),
|
35 |
"SERVER_NAME": os.getenv("SERVER_NAME", "0.0.0.0"),
|
36 |
+
"STREAMLIT_PORT": int(os.getenv("STREAMLIT_PORT", "8501")),
|
37 |
"API_PORT": int(os.getenv("API_PORT", "7861")),
|
38 |
"SHARE": os.getenv("SHARE", "false").lower() == "true",
|
39 |
"DEBUG": os.getenv("DEBUG", "false").lower() == "true",
|
|
|
69 |
from src.ai.speech_engine import AdvancedSpeechEngine as SpeechEngine, SpeechConfig
|
70 |
from src.ui.state_manager import AdvancedStateManager as StateManager
|
71 |
from src.deployment.zero_gpu_optimizer import get_optimal_device
|
|
|
72 |
from src.pipelines.opensource_3d_pipeline_v2 import (
|
73 |
ProductionPipeline,
|
74 |
ProductionConfig
|
75 |
)
|
76 |
|
77 |
+
# UI imports - now using Streamlit (separate process)
|
78 |
+
# from src.ui.streamlit_interface import main as streamlit_main
|
79 |
except ImportError as e:
|
80 |
logger.error(f"Failed to import required modules: {e}")
|
81 |
sys.exit(1)
|
|
|
124 |
|
125 |
self.qwen_processor = QwenProcessor(qwen_config)
|
126 |
|
127 |
+
# Create speech engine config for Kyutai STT
|
128 |
speech_config = SpeechConfig(
|
129 |
+
model_name="kyutai/stt-2.6b-en", # Kyutai STT model
|
130 |
device="auto", # Auto-detect device
|
131 |
+
torch_dtype="float32", # Use float32 for better compatibility
|
132 |
use_vad=True,
|
133 |
+
vad_aggressiveness=2,
|
134 |
+
use_pipeline=True # Use pipeline for easier integration
|
135 |
)
|
136 |
|
137 |
self.speech_engine = SpeechEngine(speech_config)
|
|
|
430 |
except WebSocketDisconnect:
|
431 |
manager.disconnect(monster_id)
|
432 |
|
433 |
+
# Streamlit interface runs separately
|
434 |
+
# Use: streamlit run src/ui/streamlit_interface.py
|
|
|
|
|
|
|
435 |
|
436 |
# Main entry point
|
437 |
if __name__ == "__main__":
|
|
|
446 |
logger.info("DigiPal - Advanced AI Monster Companion")
|
447 |
logger.info("=" * 60)
|
448 |
logger.info(f"Environment: {'HuggingFace Spaces' if IS_SPACES else 'Local'}")
|
449 |
+
logger.info(f"FastAPI Backend Port: {ENV_CONFIG['API_PORT']}")
|
450 |
+
logger.info(f"Streamlit UI: Run separately on port {ENV_CONFIG['STREAMLIT_PORT']}")
|
451 |
logger.info(f"MCP Enabled: {bool(ENV_CONFIG['MCP_ENDPOINT'])}")
|
452 |
logger.info("=" * 60)
|
453 |
|
454 |
+
# Start FastAPI server only
|
455 |
+
# Streamlit interface runs separately via: streamlit run src/ui/streamlit_interface.py
|
456 |
+
logger.info("Starting FastAPI backend server...")
|
457 |
+
logger.info(f"Streamlit UI: Run 'streamlit run src/ui/streamlit_interface.py' in another terminal")
|
458 |
+
|
459 |
+
config = uvicorn.Config(
|
460 |
+
app,
|
461 |
+
host=ENV_CONFIG["SERVER_NAME"],
|
462 |
+
port=ENV_CONFIG["API_PORT"],
|
463 |
+
log_level=ENV_CONFIG["LOG_LEVEL"].lower()
|
464 |
+
)
|
465 |
+
server = uvicorn.Server(config)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
466 |
|
467 |
+
# Run FastAPI server
|
468 |
+
asyncio.run(server.serve())
|
requirements.txt
CHANGED
@@ -2,7 +2,8 @@
|
|
2 |
transformers>=4.52.4 # Latest stable, supports Qwen 2.5
|
3 |
torch>=2.2.0 # PyTorch 2.0+ for torch.compile
|
4 |
torchaudio>=2.2.0
|
5 |
-
|
|
|
6 |
|
7 |
# Qwen 2.5 Optimization Stack
|
8 |
# auto-gptq>=0.7.1 # Removed - not needed, using BitsAndBytesConfig instead
|
@@ -11,23 +12,23 @@ accelerate>=0.26.1
|
|
11 |
bitsandbytes>=0.42.0
|
12 |
# FlashAttention2 will be installed at runtime if GPU is available
|
13 |
|
14 |
-
# Enhanced Audio Processing
|
15 |
-
faster-whisper>=1.0.0
|
16 |
-
librosa>=0.10.1
|
17 |
soundfile>=0.12.1
|
18 |
webrtcvad>=2.0.10
|
|
|
19 |
|
20 |
# Production Backend
|
21 |
fastapi>=0.108.0
|
22 |
uvicorn[standard]>=0.25.0
|
23 |
pydantic>=2.5.0
|
24 |
websockets>=12.0
|
|
|
25 |
|
26 |
# Advanced State Management
|
27 |
apscheduler>=3.10.4
|
28 |
aiosqlite>=0.19.0
|
29 |
|
30 |
-
# Zero GPU Optimization
|
31 |
spaces>=0.28.0
|
32 |
|
33 |
# 3D Generation Pipeline Dependencies
|
|
|
2 |
transformers>=4.52.4 # Latest stable, supports Qwen 2.5
|
3 |
torch>=2.2.0 # PyTorch 2.0+ for torch.compile
|
4 |
torchaudio>=2.2.0
|
5 |
+
diffusers>=0.30.0 # For OmniGen and other diffusion models
|
6 |
+
# gradio>=5.34.2 # Replaced with Streamlit
|
7 |
|
8 |
# Qwen 2.5 Optimization Stack
|
9 |
# auto-gptq>=0.7.1 # Removed - not needed, using BitsAndBytesConfig instead
|
|
|
12 |
bitsandbytes>=0.42.0
|
13 |
# FlashAttention2 will be installed at runtime if GPU is available
|
14 |
|
15 |
+
# Enhanced Audio Processing - Kyutai STT
|
|
|
|
|
16 |
soundfile>=0.12.1
|
17 |
webrtcvad>=2.0.10
|
18 |
+
# Note: transformers and torch/torchaudio above provide Kyutai STT support
|
19 |
|
20 |
# Production Backend
|
21 |
fastapi>=0.108.0
|
22 |
uvicorn[standard]>=0.25.0
|
23 |
pydantic>=2.5.0
|
24 |
websockets>=12.0
|
25 |
+
streamlit>=1.28.0 # Modern UI framework replacing Gradio
|
26 |
|
27 |
# Advanced State Management
|
28 |
apscheduler>=3.10.4
|
29 |
aiosqlite>=0.19.0
|
30 |
|
31 |
+
# Zero GPU Optimization (kept for speech engine compatibility)
|
32 |
spaces>=0.28.0
|
33 |
|
34 |
# 3D Generation Pipeline Dependencies
|
run_digipal.py
ADDED
@@ -0,0 +1,80 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/usr/bin/env python3
|
2 |
+
"""
|
3 |
+
DigiPal Launcher Script
|
4 |
+
Starts both FastAPI backend and Streamlit frontend
|
5 |
+
"""
|
6 |
+
|
7 |
+
import subprocess
|
8 |
+
import time
|
9 |
+
import sys
|
10 |
+
import os
|
11 |
+
import threading
|
12 |
+
import logging
|
13 |
+
|
14 |
+
# Configure logging
|
15 |
+
logging.basicConfig(
|
16 |
+
level=logging.INFO,
|
17 |
+
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
18 |
+
)
|
19 |
+
logger = logging.getLogger(__name__)
|
20 |
+
|
21 |
+
def start_fastapi():
|
22 |
+
"""Start FastAPI backend server"""
|
23 |
+
logger.info("Starting FastAPI backend server...")
|
24 |
+
try:
|
25 |
+
subprocess.run([sys.executable, "app.py"], check=True)
|
26 |
+
except subprocess.CalledProcessError as e:
|
27 |
+
logger.error(f"FastAPI server failed: {e}")
|
28 |
+
except KeyboardInterrupt:
|
29 |
+
logger.info("FastAPI server stopped")
|
30 |
+
|
31 |
+
def start_streamlit():
|
32 |
+
"""Start Streamlit frontend"""
|
33 |
+
logger.info("Starting Streamlit frontend...")
|
34 |
+
try:
|
35 |
+
port = os.getenv("STREAMLIT_PORT", "8501")
|
36 |
+
subprocess.run([
|
37 |
+
sys.executable, "-m", "streamlit", "run",
|
38 |
+
"src/ui/streamlit_interface.py",
|
39 |
+
"--server.port", port,
|
40 |
+
"--server.address", "0.0.0.0"
|
41 |
+
], check=True)
|
42 |
+
except subprocess.CalledProcessError as e:
|
43 |
+
logger.error(f"Streamlit frontend failed: {e}")
|
44 |
+
except KeyboardInterrupt:
|
45 |
+
logger.info("Streamlit frontend stopped")
|
46 |
+
|
47 |
+
def main():
|
48 |
+
"""Main launcher function"""
|
49 |
+
logger.info("🐉 DigiPal - Advanced AI Monster Companion")
|
50 |
+
logger.info("=" * 60)
|
51 |
+
logger.info("Starting both FastAPI backend and Streamlit frontend...")
|
52 |
+
api_port = os.getenv("API_PORT", "7861")
|
53 |
+
streamlit_port = os.getenv("STREAMLIT_PORT", "8501")
|
54 |
+
logger.info(f"FastAPI Backend: http://localhost:{api_port}")
|
55 |
+
logger.info(f"Streamlit Frontend: http://localhost:{streamlit_port}")
|
56 |
+
logger.info("=" * 60)
|
57 |
+
|
58 |
+
# Create necessary directories
|
59 |
+
os.makedirs("data/saves", exist_ok=True)
|
60 |
+
os.makedirs("data/models", exist_ok=True)
|
61 |
+
os.makedirs("data/cache", exist_ok=True)
|
62 |
+
os.makedirs("logs", exist_ok=True)
|
63 |
+
|
64 |
+
try:
|
65 |
+
# Start FastAPI in a separate thread
|
66 |
+
fastapi_thread = threading.Thread(target=start_fastapi, daemon=True)
|
67 |
+
fastapi_thread.start()
|
68 |
+
|
69 |
+
# Give FastAPI time to start
|
70 |
+
time.sleep(3)
|
71 |
+
|
72 |
+
# Start Streamlit (this will block)
|
73 |
+
start_streamlit()
|
74 |
+
|
75 |
+
except KeyboardInterrupt:
|
76 |
+
logger.info("Shutting down DigiPal...")
|
77 |
+
sys.exit(0)
|
78 |
+
|
79 |
+
if __name__ == "__main__":
|
80 |
+
main()
|
src/ai/speech_engine.py
CHANGED
@@ -1,7 +1,8 @@
|
|
1 |
import asyncio
|
2 |
import numpy as np
|
3 |
-
from faster_whisper import WhisperModel
|
4 |
import torch
|
|
|
|
|
5 |
import webrtcvad
|
6 |
import logging
|
7 |
from typing import Dict, List, Optional, Tuple, Any
|
@@ -13,29 +14,32 @@ import spaces
|
|
13 |
|
14 |
@dataclass
|
15 |
class SpeechConfig:
|
16 |
-
|
17 |
device: str = "auto"
|
18 |
-
|
19 |
use_vad: bool = True
|
20 |
vad_aggressiveness: int = 2 # 0-3, higher = more aggressive
|
21 |
chunk_duration_ms: int = 30 # VAD chunk size
|
22 |
sample_rate: int = 16000
|
|
|
23 |
|
24 |
class AdvancedSpeechEngine:
|
25 |
def __init__(self, config: SpeechConfig):
|
26 |
self.config = config
|
27 |
self.logger = logging.getLogger(__name__)
|
28 |
|
29 |
-
#
|
30 |
-
self.
|
31 |
-
"
|
32 |
-
"
|
33 |
-
"
|
34 |
-
"
|
35 |
-
"
|
36 |
}
|
37 |
|
38 |
-
self.
|
|
|
|
|
39 |
self.vad_model = None
|
40 |
|
41 |
# Performance tracking
|
@@ -47,11 +51,11 @@ class AdvancedSpeechEngine:
|
|
47 |
self.is_processing = False
|
48 |
|
49 |
async def initialize(self):
|
50 |
-
"""Initialize the speech recognition system"""
|
51 |
try:
|
52 |
# Enhanced device detection for local vs Spaces environments
|
53 |
device = self.config.device
|
54 |
-
|
55 |
|
56 |
if device == "auto":
|
57 |
# For Zero GPU environments, try GPU first, fallback to CPU
|
@@ -65,48 +69,93 @@ class AdvancedSpeechEngine:
|
|
65 |
except Exception as cuda_error:
|
66 |
# CUDA not properly accessible, use CPU
|
67 |
device = "cpu"
|
68 |
-
if
|
69 |
-
|
70 |
-
self.logger.info(f"CUDA not accessible ({cuda_error}), using CPU with
|
71 |
else:
|
72 |
device = "cpu"
|
73 |
-
if
|
74 |
-
|
75 |
-
self.logger.info("CUDA not available, using CPU with
|
76 |
|
77 |
-
# Adjust
|
78 |
-
if device == "cpu" and
|
79 |
-
|
80 |
-
self.logger.info("CPU detected, switching from float16 to
|
81 |
|
82 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
83 |
try:
|
84 |
-
self.
|
85 |
-
|
86 |
-
|
87 |
-
|
88 |
-
|
89 |
-
|
90 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
91 |
except Exception as model_error:
|
92 |
# Final fallback to CPU with basic settings
|
93 |
self.logger.warning(f"Failed to load on {device}, falling back to CPU: {model_error}")
|
94 |
-
|
95 |
-
|
96 |
-
|
97 |
-
|
98 |
-
|
99 |
-
|
100 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
101 |
|
102 |
# Initialize VAD if enabled
|
103 |
if self.config.use_vad:
|
104 |
self.vad_model = webrtcvad.Vad(self.config.vad_aggressiveness)
|
105 |
|
106 |
-
self.logger.info(f"
|
107 |
|
108 |
except Exception as e:
|
109 |
-
self.logger.error(f"Failed to initialize speech engine: {e}")
|
110 |
raise
|
111 |
|
112 |
async def process_audio_stream(self, audio_data: np.ndarray) -> Dict[str, Any]:
|
@@ -135,35 +184,55 @@ class AdvancedSpeechEngine:
|
|
135 |
"has_speech": False
|
136 |
}
|
137 |
|
138 |
-
# Transcribe with
|
139 |
-
|
140 |
-
|
141 |
-
|
142 |
-
|
143 |
-
|
144 |
-
|
145 |
-
|
146 |
-
|
147 |
-
|
148 |
-
|
149 |
-
|
150 |
-
|
151 |
-
|
152 |
-
|
153 |
-
|
154 |
-
|
155 |
-
for segment in segments:
|
156 |
-
transcription += segment.text + " "
|
157 |
-
avg_confidence += segment.avg_logprob
|
158 |
-
segment_count += 1
|
159 |
-
|
160 |
-
transcription = transcription.strip()
|
161 |
-
|
162 |
-
if segment_count > 0:
|
163 |
-
avg_confidence = avg_confidence / segment_count
|
164 |
-
confidence = self._logprob_to_confidence(avg_confidence)
|
165 |
else:
|
166 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
167 |
|
168 |
processing_time = time.time() - start_time
|
169 |
self.transcription_times.append(processing_time)
|
@@ -178,8 +247,9 @@ class AdvancedSpeechEngine:
|
|
178 |
"processing_time": processing_time,
|
179 |
"has_speech": True,
|
180 |
"speech_analysis": speech_analysis,
|
181 |
-
"detected_language":
|
182 |
-
"language_probability":
|
|
|
183 |
}
|
184 |
|
185 |
except Exception as e:
|
@@ -292,18 +362,31 @@ class AdvancedSpeechEngine:
|
|
292 |
}
|
293 |
|
294 |
async def batch_transcribe(self, audio_files: List[str]) -> List[Dict[str, Any]]:
|
295 |
-
"""Batch transcribe multiple audio files"""
|
296 |
results = []
|
297 |
|
298 |
for audio_file in audio_files:
|
299 |
try:
|
300 |
-
# Load audio file
|
301 |
-
|
302 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
303 |
|
304 |
# Process
|
305 |
result = await self.process_audio_stream(audio_data)
|
306 |
result["file_path"] = audio_file
|
|
|
307 |
|
308 |
results.append(result)
|
309 |
|
@@ -334,42 +417,54 @@ class AdvancedSpeechEngine:
|
|
334 |
}
|
335 |
|
336 |
def optimize_for_hardware(self, available_vram_gb: float) -> SpeechConfig:
|
337 |
-
"""Optimize
|
338 |
-
|
|
|
339 |
return SpeechConfig(
|
340 |
-
|
341 |
device="cuda",
|
342 |
-
|
343 |
-
use_vad=True
|
|
|
344 |
)
|
345 |
elif available_vram_gb >= 6:
|
346 |
return SpeechConfig(
|
347 |
-
|
348 |
device="cuda",
|
349 |
-
|
350 |
-
use_vad=True
|
|
|
351 |
)
|
352 |
-
elif available_vram_gb >=
|
353 |
return SpeechConfig(
|
354 |
-
|
355 |
device="cuda",
|
356 |
-
|
357 |
-
use_vad=True
|
|
|
358 |
)
|
359 |
else:
|
360 |
return SpeechConfig(
|
361 |
-
|
362 |
device="cpu",
|
363 |
-
|
364 |
-
use_vad=True
|
|
|
365 |
)
|
366 |
|
367 |
# Apply GPU decorator to methods after class definition for ZeroGPU compatibility
|
368 |
try:
|
369 |
import os
|
370 |
if os.getenv("SPACE_ID") is not None:
|
371 |
-
# We're in Spaces environment, apply GPU decorator
|
372 |
-
AdvancedSpeechEngine.process_audio_stream = spaces.GPU(
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
373 |
except (ImportError, NotImplementedError, AttributeError) as e:
|
374 |
# GPU decorator not available or failed, continue without it
|
375 |
pass
|
|
|
1 |
import asyncio
|
2 |
import numpy as np
|
|
|
3 |
import torch
|
4 |
+
import torchaudio
|
5 |
+
from transformers import pipeline, AutoModelForSpeechSeq2Seq, AutoProcessor
|
6 |
import webrtcvad
|
7 |
import logging
|
8 |
from typing import Dict, List, Optional, Tuple, Any
|
|
|
14 |
|
15 |
@dataclass
|
16 |
class SpeechConfig:
|
17 |
+
model_name: str = "kyutai/stt-2.6b-en" # Kyutai STT model
|
18 |
device: str = "auto"
|
19 |
+
torch_dtype: str = "float16"
|
20 |
use_vad: bool = True
|
21 |
vad_aggressiveness: int = 2 # 0-3, higher = more aggressive
|
22 |
chunk_duration_ms: int = 30 # VAD chunk size
|
23 |
sample_rate: int = 16000
|
24 |
+
use_pipeline: bool = True # Use transformers pipeline for easier integration
|
25 |
|
26 |
class AdvancedSpeechEngine:
|
27 |
def __init__(self, config: SpeechConfig):
|
28 |
self.config = config
|
29 |
self.logger = logging.getLogger(__name__)
|
30 |
|
31 |
+
# Kyutai STT model configurations
|
32 |
+
self.model_info = {
|
33 |
+
"name": "Kyutai STT-2.6B-EN",
|
34 |
+
"description": "Multilingual speech-to-text model optimized for English",
|
35 |
+
"memory_gb": 6, # Approximate memory requirement for 2.6B model
|
36 |
+
"speed": "fast",
|
37 |
+
"accuracy": "high"
|
38 |
}
|
39 |
|
40 |
+
self.speech_pipeline = None
|
41 |
+
self.model = None
|
42 |
+
self.processor = None
|
43 |
self.vad_model = None
|
44 |
|
45 |
# Performance tracking
|
|
|
51 |
self.is_processing = False
|
52 |
|
53 |
async def initialize(self):
|
54 |
+
"""Initialize the Kyutai STT speech recognition system"""
|
55 |
try:
|
56 |
# Enhanced device detection for local vs Spaces environments
|
57 |
device = self.config.device
|
58 |
+
torch_dtype = self.config.torch_dtype
|
59 |
|
60 |
if device == "auto":
|
61 |
# For Zero GPU environments, try GPU first, fallback to CPU
|
|
|
69 |
except Exception as cuda_error:
|
70 |
# CUDA not properly accessible, use CPU
|
71 |
device = "cpu"
|
72 |
+
if torch_dtype == "float16":
|
73 |
+
torch_dtype = "float32"
|
74 |
+
self.logger.info(f"CUDA not accessible ({cuda_error}), using CPU with float32")
|
75 |
else:
|
76 |
device = "cpu"
|
77 |
+
if torch_dtype == "float16":
|
78 |
+
torch_dtype = "float32"
|
79 |
+
self.logger.info("CUDA not available, using CPU with float32")
|
80 |
|
81 |
+
# Adjust torch_dtype for CPU
|
82 |
+
if device == "cpu" and torch_dtype == "float16":
|
83 |
+
torch_dtype = "float32" # Use float32 for CPU instead of float16
|
84 |
+
self.logger.info("CPU detected, switching from float16 to float32 dtype")
|
85 |
|
86 |
+
# Convert string dtype to torch dtype
|
87 |
+
dtype_map = {
|
88 |
+
"float16": torch.float16,
|
89 |
+
"float32": torch.float32,
|
90 |
+
"bfloat16": torch.bfloat16
|
91 |
+
}
|
92 |
+
torch_dtype_obj = dtype_map.get(torch_dtype, torch.float32)
|
93 |
+
|
94 |
+
# Initialize Kyutai STT with proper error handling
|
95 |
try:
|
96 |
+
if self.config.use_pipeline:
|
97 |
+
# Use transformers pipeline for easier integration
|
98 |
+
self.speech_pipeline = pipeline(
|
99 |
+
"automatic-speech-recognition",
|
100 |
+
model=self.config.model_name,
|
101 |
+
torch_dtype=torch_dtype_obj,
|
102 |
+
device=device,
|
103 |
+
cache_dir="data/models/"
|
104 |
+
)
|
105 |
+
self.logger.info(f"Kyutai STT pipeline loaded on {device} with {torch_dtype}")
|
106 |
+
else:
|
107 |
+
# Load model and processor separately for more control
|
108 |
+
self.model = AutoModelForSpeechSeq2Seq.from_pretrained(
|
109 |
+
self.config.model_name,
|
110 |
+
torch_dtype=torch_dtype_obj,
|
111 |
+
device_map="auto" if device == "cuda" else None,
|
112 |
+
cache_dir="data/models/"
|
113 |
+
)
|
114 |
+
self.processor = AutoProcessor.from_pretrained(
|
115 |
+
self.config.model_name,
|
116 |
+
cache_dir="data/models/"
|
117 |
+
)
|
118 |
+
|
119 |
+
if device == "cuda" and not hasattr(self.model, 'device_map'):
|
120 |
+
self.model = self.model.to(device)
|
121 |
+
|
122 |
+
self.logger.info(f"Kyutai STT model and processor loaded on {device} with {torch_dtype}")
|
123 |
+
|
124 |
except Exception as model_error:
|
125 |
# Final fallback to CPU with basic settings
|
126 |
self.logger.warning(f"Failed to load on {device}, falling back to CPU: {model_error}")
|
127 |
+
|
128 |
+
if self.config.use_pipeline:
|
129 |
+
self.speech_pipeline = pipeline(
|
130 |
+
"automatic-speech-recognition",
|
131 |
+
model=self.config.model_name,
|
132 |
+
torch_dtype=torch.float32,
|
133 |
+
device="cpu",
|
134 |
+
cache_dir="data/models/"
|
135 |
+
)
|
136 |
+
else:
|
137 |
+
self.model = AutoModelForSpeechSeq2Seq.from_pretrained(
|
138 |
+
self.config.model_name,
|
139 |
+
torch_dtype=torch.float32,
|
140 |
+
device_map=None,
|
141 |
+
cache_dir="data/models/"
|
142 |
+
)
|
143 |
+
self.processor = AutoProcessor.from_pretrained(
|
144 |
+
self.config.model_name,
|
145 |
+
cache_dir="data/models/"
|
146 |
+
)
|
147 |
+
self.model = self.model.to("cpu")
|
148 |
+
|
149 |
+
self.logger.info("Kyutai STT model loaded on CPU (fallback)")
|
150 |
|
151 |
# Initialize VAD if enabled
|
152 |
if self.config.use_vad:
|
153 |
self.vad_model = webrtcvad.Vad(self.config.vad_aggressiveness)
|
154 |
|
155 |
+
self.logger.info(f"Kyutai STT speech engine initialized: {self.config.model_name} on {device}")
|
156 |
|
157 |
except Exception as e:
|
158 |
+
self.logger.error(f"Failed to initialize Kyutai STT speech engine: {e}")
|
159 |
raise
|
160 |
|
161 |
async def process_audio_stream(self, audio_data: np.ndarray) -> Dict[str, Any]:
|
|
|
184 |
"has_speech": False
|
185 |
}
|
186 |
|
187 |
+
# Transcribe with Kyutai STT
|
188 |
+
if self.config.use_pipeline and self.speech_pipeline:
|
189 |
+
# Use pipeline for simpler transcription
|
190 |
+
result = self.speech_pipeline(
|
191 |
+
audio_data,
|
192 |
+
generate_kwargs={
|
193 |
+
"language": "en",
|
194 |
+
"task": "transcribe",
|
195 |
+
"max_new_tokens": 256
|
196 |
+
}
|
197 |
+
)
|
198 |
+
|
199 |
+
transcription = result["text"].strip()
|
200 |
+
# Pipeline doesn't provide confidence scores directly
|
201 |
+
confidence = 0.8 # Default confidence for pipeline
|
202 |
+
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
203 |
else:
|
204 |
+
# Use model and processor for more control
|
205 |
+
# Prepare inputs
|
206 |
+
inputs = self.processor(
|
207 |
+
audio_data,
|
208 |
+
sampling_rate=self.config.sample_rate,
|
209 |
+
return_tensors="pt"
|
210 |
+
)
|
211 |
+
|
212 |
+
# Move inputs to device
|
213 |
+
device = next(self.model.parameters()).device
|
214 |
+
inputs = {k: v.to(device) for k, v in inputs.items()}
|
215 |
+
|
216 |
+
# Generate transcription
|
217 |
+
with torch.no_grad():
|
218 |
+
generated_tokens = self.model.generate(
|
219 |
+
**inputs,
|
220 |
+
language="en",
|
221 |
+
task="transcribe",
|
222 |
+
max_new_tokens=256,
|
223 |
+
num_beams=1, # Faster inference
|
224 |
+
do_sample=False,
|
225 |
+
temperature=1.0
|
226 |
+
)
|
227 |
+
|
228 |
+
# Decode transcription
|
229 |
+
transcription = self.processor.batch_decode(
|
230 |
+
generated_tokens,
|
231 |
+
skip_special_tokens=True
|
232 |
+
)[0].strip()
|
233 |
+
|
234 |
+
# Calculate confidence (simplified)
|
235 |
+
confidence = 0.8 # Default confidence
|
236 |
|
237 |
processing_time = time.time() - start_time
|
238 |
self.transcription_times.append(processing_time)
|
|
|
247 |
"processing_time": processing_time,
|
248 |
"has_speech": True,
|
249 |
"speech_analysis": speech_analysis,
|
250 |
+
"detected_language": "en", # Kyutai model is optimized for English
|
251 |
+
"language_probability": 1.0,
|
252 |
+
"model": "kyutai-stt-2.6b-en"
|
253 |
}
|
254 |
|
255 |
except Exception as e:
|
|
|
362 |
}
|
363 |
|
364 |
async def batch_transcribe(self, audio_files: List[str]) -> List[Dict[str, Any]]:
|
365 |
+
"""Batch transcribe multiple audio files using Kyutai STT"""
|
366 |
results = []
|
367 |
|
368 |
for audio_file in audio_files:
|
369 |
try:
|
370 |
+
# Load audio file - use torchaudio for better PyTorch integration
|
371 |
+
audio_data, sample_rate = torchaudio.load(audio_file)
|
372 |
+
|
373 |
+
# Convert to numpy and ensure mono
|
374 |
+
audio_data = audio_data.numpy()
|
375 |
+
if len(audio_data.shape) > 1:
|
376 |
+
audio_data = audio_data.mean(axis=0) # Convert to mono
|
377 |
+
|
378 |
+
# Resample if necessary
|
379 |
+
if sample_rate != self.config.sample_rate:
|
380 |
+
# Use torchaudio for resampling
|
381 |
+
audio_tensor = torch.from_numpy(audio_data).unsqueeze(0)
|
382 |
+
resampler = torchaudio.transforms.Resample(sample_rate, self.config.sample_rate)
|
383 |
+
audio_tensor = resampler(audio_tensor)
|
384 |
+
audio_data = audio_tensor.squeeze(0).numpy()
|
385 |
|
386 |
# Process
|
387 |
result = await self.process_audio_stream(audio_data)
|
388 |
result["file_path"] = audio_file
|
389 |
+
result["original_sample_rate"] = sample_rate
|
390 |
|
391 |
results.append(result)
|
392 |
|
|
|
417 |
}
|
418 |
|
419 |
def optimize_for_hardware(self, available_vram_gb: float) -> SpeechConfig:
|
420 |
+
"""Optimize Kyutai STT config based on available hardware"""
|
421 |
+
# Kyutai STT-2.6B requires about 6GB VRAM for optimal performance
|
422 |
+
if available_vram_gb >= 8:
|
423 |
return SpeechConfig(
|
424 |
+
model_name="kyutai/stt-2.6b-en",
|
425 |
device="cuda",
|
426 |
+
torch_dtype="float16",
|
427 |
+
use_vad=True,
|
428 |
+
use_pipeline=True
|
429 |
)
|
430 |
elif available_vram_gb >= 6:
|
431 |
return SpeechConfig(
|
432 |
+
model_name="kyutai/stt-2.6b-en",
|
433 |
device="cuda",
|
434 |
+
torch_dtype="float32",
|
435 |
+
use_vad=True,
|
436 |
+
use_pipeline=True
|
437 |
)
|
438 |
+
elif available_vram_gb >= 4:
|
439 |
return SpeechConfig(
|
440 |
+
model_name="kyutai/stt-2.6b-en",
|
441 |
device="cuda",
|
442 |
+
torch_dtype="float32",
|
443 |
+
use_vad=True,
|
444 |
+
use_pipeline=False # More memory efficient without pipeline
|
445 |
)
|
446 |
else:
|
447 |
return SpeechConfig(
|
448 |
+
model_name="kyutai/stt-2.6b-en",
|
449 |
device="cpu",
|
450 |
+
torch_dtype="float32",
|
451 |
+
use_vad=True,
|
452 |
+
use_pipeline=True
|
453 |
)
|
454 |
|
455 |
# Apply GPU decorator to methods after class definition for ZeroGPU compatibility
|
456 |
try:
|
457 |
import os
|
458 |
if os.getenv("SPACE_ID") is not None:
|
459 |
+
# We're in Spaces environment, apply GPU decorator for Kyutai STT
|
460 |
+
AdvancedSpeechEngine.process_audio_stream = spaces.GPU(
|
461 |
+
AdvancedSpeechEngine.process_audio_stream,
|
462 |
+
duration=120 # Kyutai STT may take longer than Whisper
|
463 |
+
)
|
464 |
+
AdvancedSpeechEngine.batch_transcribe = spaces.GPU(
|
465 |
+
AdvancedSpeechEngine.batch_transcribe,
|
466 |
+
duration=300 # Batch processing may take longer
|
467 |
+
)
|
468 |
except (ImportError, NotImplementedError, AttributeError) as e:
|
469 |
# GPU decorator not available or failed, continue without it
|
470 |
pass
|
src/core/monster_3d_hunyuan_integration.py
DELETED
@@ -1,326 +0,0 @@
|
|
1 |
-
"""
|
2 |
-
Enhanced Monster 3D Integration using Hunyuan3D Pipeline
|
3 |
-
Ensures consistent visual style across all evolution stages
|
4 |
-
"""
|
5 |
-
|
6 |
-
import asyncio
|
7 |
-
import json
|
8 |
-
import logging
|
9 |
-
from pathlib import Path
|
10 |
-
from typing import Dict, List, Optional, Any
|
11 |
-
from dataclasses import dataclass
|
12 |
-
|
13 |
-
from ..pipelines.hunyuan3d_pipeline import (
|
14 |
-
DigiPalHunyuan3DIntegration,
|
15 |
-
Hunyuan3DConfig,
|
16 |
-
GenerationMode
|
17 |
-
)
|
18 |
-
|
19 |
-
logger = logging.getLogger(__name__)
|
20 |
-
|
21 |
-
@dataclass
|
22 |
-
class Monster3DProfile:
|
23 |
-
"""Visual profile for consistent 3D generation"""
|
24 |
-
|
25 |
-
# Core design elements that remain consistent
|
26 |
-
base_design: str # e.g., "dragon-like", "wolf-like", "humanoid"
|
27 |
-
primary_features: List[str] # e.g., ["wings", "horn", "tail"]
|
28 |
-
color_scheme: str # e.g., "blue crystalline", "dark purple", "golden"
|
29 |
-
texture_style: str # e.g., "metallic", "organic", "ethereal"
|
30 |
-
|
31 |
-
# Elements that change with evolution
|
32 |
-
size_modifier: Dict[str, str] # stage -> size description
|
33 |
-
detail_level: Dict[str, str] # stage -> detail description
|
34 |
-
aura_intensity: Dict[str, str] # stage -> aura/effects
|
35 |
-
|
36 |
-
class ConsistentMonster3DManager:
|
37 |
-
"""Manages 3D generation with consistent style across evolutions"""
|
38 |
-
|
39 |
-
def __init__(self, config: Optional[Hunyuan3DConfig] = None):
|
40 |
-
self.integration = DigiPalHunyuan3DIntegration(config)
|
41 |
-
self.profiles = {} # Cache monster visual profiles
|
42 |
-
self.style_guide = self._load_style_guide()
|
43 |
-
|
44 |
-
def _load_style_guide(self) -> Dict[str, Any]:
|
45 |
-
"""Load visual style guidelines"""
|
46 |
-
|
47 |
-
return {
|
48 |
-
"render_style": "AAA 3D render, well lit, white void background",
|
49 |
-
"model_requirements": "T-pose, game character model, consistent design language",
|
50 |
-
"quality_specs": "professional 3D game asset, pbr materials, clean topology",
|
51 |
-
|
52 |
-
# Species-specific base designs
|
53 |
-
"species_designs": {
|
54 |
-
"data": {
|
55 |
-
"base": "biomechanical creature",
|
56 |
-
"features": ["crystalline segments", "circuit patterns", "data streams"],
|
57 |
-
"materials": "holographic metal with glass accents"
|
58 |
-
},
|
59 |
-
"vaccine": {
|
60 |
-
"base": "angelic creature",
|
61 |
-
"features": ["feathered wings", "armor plating", "holy symbols"],
|
62 |
-
"materials": "white metal with gold trim"
|
63 |
-
},
|
64 |
-
"virus": {
|
65 |
-
"base": "corrupted beast",
|
66 |
-
"features": ["shadow tendrils", "void armor", "energy cores"],
|
67 |
-
"materials": "dark metal with purple energy"
|
68 |
-
},
|
69 |
-
"free": {
|
70 |
-
"base": "elemental creature",
|
71 |
-
"features": ["natural armor", "elemental crystals", "organic patterns"],
|
72 |
-
"materials": "stone and wood with living elements"
|
73 |
-
}
|
74 |
-
},
|
75 |
-
|
76 |
-
# Personality influences on pose/expression
|
77 |
-
"personality_modifiers": {
|
78 |
-
"brave": "heroic stance, fierce expression, forward-leaning pose",
|
79 |
-
"calm": "balanced stance, serene expression, centered pose",
|
80 |
-
"energetic": "dynamic stance, excited expression, action-ready pose",
|
81 |
-
"clever": "tactical stance, focused expression, observant pose",
|
82 |
-
"friendly": "open stance, warm expression, welcoming pose"
|
83 |
-
}
|
84 |
-
}
|
85 |
-
|
86 |
-
async def create_monster_profile(self, monster: Any) -> Monster3DProfile:
|
87 |
-
"""Create consistent visual profile for a monster"""
|
88 |
-
|
89 |
-
# Determine base design from species and initial traits
|
90 |
-
species_design = self.style_guide["species_designs"].get(
|
91 |
-
monster.species_type.value,
|
92 |
-
self.style_guide["species_designs"]["data"]
|
93 |
-
)
|
94 |
-
|
95 |
-
# Create base design combining species and personality
|
96 |
-
if monster.personality.value == "brave":
|
97 |
-
base = f"heroic {species_design['base']}"
|
98 |
-
elif monster.personality.value == "calm":
|
99 |
-
base = f"wise {species_design['base']}"
|
100 |
-
elif monster.personality.value == "energetic":
|
101 |
-
base = f"agile {species_design['base']}"
|
102 |
-
elif monster.personality.value == "clever":
|
103 |
-
base = f"cunning {species_design['base']}"
|
104 |
-
else:
|
105 |
-
base = f"gentle {species_design['base']}"
|
106 |
-
|
107 |
-
# Define consistent features based on stats
|
108 |
-
features = species_design["features"].copy()
|
109 |
-
if monster.battle_stats.offense > 70:
|
110 |
-
features.append("prominent claws or weapons")
|
111 |
-
if monster.battle_stats.defense > 70:
|
112 |
-
features.append("reinforced armor sections")
|
113 |
-
if monster.battle_stats.speed > 70:
|
114 |
-
features.append("aerodynamic body shape")
|
115 |
-
|
116 |
-
# Create profile with evolution variations
|
117 |
-
profile = Monster3DProfile(
|
118 |
-
base_design=base,
|
119 |
-
primary_features=features,
|
120 |
-
color_scheme=self._determine_color_scheme(monster),
|
121 |
-
texture_style=species_design["materials"],
|
122 |
-
|
123 |
-
# Size progression - maintains adult-like proportions
|
124 |
-
size_modifier={
|
125 |
-
"egg": "dormant embryonic",
|
126 |
-
"baby": "compact juvenile",
|
127 |
-
"child": "developing adolescent",
|
128 |
-
"adult": "fully grown",
|
129 |
-
"perfect": "enhanced mature",
|
130 |
-
"ultimate": "transcendent apex",
|
131 |
-
"mega": "legendary titan"
|
132 |
-
},
|
133 |
-
|
134 |
-
# Detail progression
|
135 |
-
detail_level={
|
136 |
-
"egg": "simplified form within crystalline shell",
|
137 |
-
"baby": "basic features emerging",
|
138 |
-
"child": "defined features developing",
|
139 |
-
"adult": "fully detailed form",
|
140 |
-
"perfect": "intricate details and enhancements",
|
141 |
-
"ultimate": "complex ornate details",
|
142 |
-
"mega": "impossibly detailed divine form"
|
143 |
-
},
|
144 |
-
|
145 |
-
# Aura/effects progression
|
146 |
-
aura_intensity={
|
147 |
-
"egg": "faint inner glow",
|
148 |
-
"baby": "soft ambient glow",
|
149 |
-
"child": "visible energy aura",
|
150 |
-
"adult": "strong power aura",
|
151 |
-
"perfect": "intense energy field",
|
152 |
-
"ultimate": "overwhelming power presence",
|
153 |
-
"mega": "reality-warping energy storm"
|
154 |
-
}
|
155 |
-
)
|
156 |
-
|
157 |
-
# Cache the profile
|
158 |
-
self.profiles[monster.id] = profile
|
159 |
-
return profile
|
160 |
-
|
161 |
-
def _determine_color_scheme(self, monster: Any) -> str:
|
162 |
-
"""Determine consistent color scheme based on monster attributes"""
|
163 |
-
|
164 |
-
# Base colors by species
|
165 |
-
species_colors = {
|
166 |
-
"data": ["blue", "cyan", "white"],
|
167 |
-
"vaccine": ["gold", "white", "silver"],
|
168 |
-
"virus": ["purple", "black", "red"],
|
169 |
-
"free": ["green", "brown", "orange"]
|
170 |
-
}
|
171 |
-
|
172 |
-
# Get base colors
|
173 |
-
colors = species_colors.get(monster.species_type.value, ["gray"])
|
174 |
-
|
175 |
-
# Modify based on personality
|
176 |
-
if monster.personality.value == "brave":
|
177 |
-
colors.append("crimson accents")
|
178 |
-
elif monster.personality.value == "calm":
|
179 |
-
colors.append("pearl highlights")
|
180 |
-
elif monster.personality.value == "energetic":
|
181 |
-
colors.append("electric highlights")
|
182 |
-
|
183 |
-
return f"{colors[0]} and {colors[1]} with {colors[2]}"
|
184 |
-
|
185 |
-
async def generate_monster_model(self,
|
186 |
-
monster: Any,
|
187 |
-
stage: Optional[str] = None,
|
188 |
-
mode: Optional[GenerationMode] = None) -> Dict[str, Any]:
|
189 |
-
"""Generate 3D model with consistent style"""
|
190 |
-
|
191 |
-
# Get or create visual profile
|
192 |
-
if monster.id not in self.profiles:
|
193 |
-
profile = await self.create_monster_profile(monster)
|
194 |
-
else:
|
195 |
-
profile = self.profiles[monster.id]
|
196 |
-
|
197 |
-
# Use current stage if not specified
|
198 |
-
if stage is None:
|
199 |
-
stage = monster.evolution.current_stage.value
|
200 |
-
|
201 |
-
# Build consistent prompt
|
202 |
-
prompt = self._build_consistent_prompt(monster, profile, stage)
|
203 |
-
|
204 |
-
# Create monster data with enhanced prompt
|
205 |
-
monster_data = {
|
206 |
-
"name": monster.name,
|
207 |
-
"stage": stage,
|
208 |
-
"personality": monster.personality.value,
|
209 |
-
"species_type": monster.species_type.value,
|
210 |
-
"stats": {
|
211 |
-
"offense": monster.battle_stats.offense,
|
212 |
-
"defense": monster.battle_stats.defense,
|
213 |
-
"speed": monster.battle_stats.speed,
|
214 |
-
"brains": monster.battle_stats.brains
|
215 |
-
},
|
216 |
-
"_custom_prompt": prompt # Override default prompt generation
|
217 |
-
}
|
218 |
-
|
219 |
-
# Generate model
|
220 |
-
result = await self.integration.pipeline.generate_monster_for_digipal(
|
221 |
-
monster_data, mode
|
222 |
-
)
|
223 |
-
|
224 |
-
return result
|
225 |
-
|
226 |
-
def _build_consistent_prompt(self,
|
227 |
-
monster: Any,
|
228 |
-
profile: Monster3DProfile,
|
229 |
-
stage: str) -> str:
|
230 |
-
"""Build prompt ensuring visual consistency"""
|
231 |
-
|
232 |
-
# Get stage-specific modifiers
|
233 |
-
size = profile.size_modifier.get(stage, "medium sized")
|
234 |
-
detail = profile.detail_level.get(stage, "detailed")
|
235 |
-
aura = profile.aura_intensity.get(stage, "glowing")
|
236 |
-
|
237 |
-
# Get personality modifier
|
238 |
-
personality_mod = self.style_guide["personality_modifiers"].get(
|
239 |
-
monster.personality.value,
|
240 |
-
"neutral stance"
|
241 |
-
)
|
242 |
-
|
243 |
-
# Build the prompt
|
244 |
-
prompt = (
|
245 |
-
f"{self.style_guide['render_style']}, "
|
246 |
-
f"{size} {profile.base_design}, "
|
247 |
-
f"{', '.join(profile.primary_features)}, "
|
248 |
-
f"{profile.color_scheme} coloration, "
|
249 |
-
f"{profile.texture_style}, "
|
250 |
-
f"{detail}, "
|
251 |
-
f"{aura}, "
|
252 |
-
f"{personality_mod}, "
|
253 |
-
f"{self.style_guide['model_requirements']}, "
|
254 |
-
f"{self.style_guide['quality_specs']}"
|
255 |
-
)
|
256 |
-
|
257 |
-
# Add stage-specific quality hints
|
258 |
-
if stage in ["ultimate", "mega", "perfect"]:
|
259 |
-
prompt += ", hero character quality, highly detailed, epic presence"
|
260 |
-
elif stage in ["baby", "egg"]:
|
261 |
-
prompt += ", cute proportions, simplified details, approachable design"
|
262 |
-
|
263 |
-
return prompt
|
264 |
-
|
265 |
-
async def generate_evolution_sequence(self,
|
266 |
-
monster: Any,
|
267 |
-
stages: Optional[List[str]] = None) -> List[Dict[str, Any]]:
|
268 |
-
"""Generate complete evolution sequence with consistent design"""
|
269 |
-
|
270 |
-
# Default to all possible stages
|
271 |
-
if stages is None:
|
272 |
-
stages = ["baby", "child", "adult", "perfect", "ultimate"]
|
273 |
-
|
274 |
-
# Ensure we have a profile
|
275 |
-
if monster.id not in self.profiles:
|
276 |
-
await self.create_monster_profile(monster)
|
277 |
-
|
278 |
-
# Generate all stages
|
279 |
-
results = []
|
280 |
-
for stage in stages:
|
281 |
-
# Use faster generation for early stages
|
282 |
-
mode = GenerationMode.TURBO if stage in ["baby", "child"] else GenerationMode.FAST
|
283 |
-
|
284 |
-
result = await self.generate_monster_model(monster, stage, mode)
|
285 |
-
results.append(result)
|
286 |
-
|
287 |
-
# Add small delay between generations to avoid rate limits
|
288 |
-
await asyncio.sleep(1)
|
289 |
-
|
290 |
-
return results
|
291 |
-
|
292 |
-
async def generate_showcase_views(self,
|
293 |
-
monster: Any,
|
294 |
-
include_action_poses: bool = True) -> Dict[str, Any]:
|
295 |
-
"""Generate multiple views/poses for showcase"""
|
296 |
-
|
297 |
-
base_result = await self.generate_monster_model(monster)
|
298 |
-
|
299 |
-
if not base_result["success"] or not include_action_poses:
|
300 |
-
return {"base": base_result}
|
301 |
-
|
302 |
-
# Generate additional views with modified prompts
|
303 |
-
views = {"base": base_result}
|
304 |
-
|
305 |
-
# Action pose
|
306 |
-
action_prompt = base_result.get("prompt", "").replace("T-pose", "dynamic action pose")
|
307 |
-
# Battle pose
|
308 |
-
battle_prompt = base_result.get("prompt", "").replace("T-pose", "battle-ready stance")
|
309 |
-
|
310 |
-
# TODO: Implement multi-pose generation when API supports it
|
311 |
-
|
312 |
-
return views
|
313 |
-
|
314 |
-
# Convenience function for integration
|
315 |
-
async def setup_hunyuan3d_for_monster(monster: Any) -> ConsistentMonster3DManager:
|
316 |
-
"""Setup Hunyuan3D manager for a specific monster"""
|
317 |
-
|
318 |
-
config = Hunyuan3DConfig(
|
319 |
-
output_dir=Path(f"./3d_models/{monster.name}"),
|
320 |
-
cache_dir=Path("./model_cache")
|
321 |
-
)
|
322 |
-
|
323 |
-
manager = ConsistentMonster3DManager(config)
|
324 |
-
await manager.create_monster_profile(monster)
|
325 |
-
|
326 |
-
return manager
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/pipelines/hunyuan3d_pipeline.py
DELETED
@@ -1,958 +0,0 @@
|
|
1 |
-
"""
|
2 |
-
Hunyuan3D-2.1 Integration Pipeline for DigiPal
|
3 |
-
Implements Tencent's state-of-the-art 3D generation model
|
4 |
-
Built with Rick Rubin philosophy: Focus on the core, eliminate friction
|
5 |
-
"""
|
6 |
-
|
7 |
-
import asyncio
|
8 |
-
import base64
|
9 |
-
import json
|
10 |
-
import logging
|
11 |
-
import os
|
12 |
-
import time
|
13 |
-
from dataclasses import dataclass, field
|
14 |
-
from enum import Enum
|
15 |
-
from pathlib import Path
|
16 |
-
from typing import Dict, List, Optional, Tuple, Any, Union
|
17 |
-
import aiohttp
|
18 |
-
import numpy as np
|
19 |
-
from PIL import Image
|
20 |
-
import torch
|
21 |
-
import trimesh
|
22 |
-
from gradio_client import Client, handle_file
|
23 |
-
import tempfile
|
24 |
-
import shutil
|
25 |
-
|
26 |
-
# Configure logging
|
27 |
-
logging.basicConfig(level=logging.INFO)
|
28 |
-
logger = logging.getLogger(__name__)
|
29 |
-
|
30 |
-
class GenerationMode(Enum):
|
31 |
-
"""Hunyuan3D generation speed/quality modes"""
|
32 |
-
TURBO = "Turbo (Fast)" # Fastest, lower quality
|
33 |
-
FAST = "Fast" # Balanced speed/quality
|
34 |
-
STANDARD = "Standard" # Best quality, slower
|
35 |
-
|
36 |
-
class TextureMethod(Enum):
|
37 |
-
"""Texture generation methods"""
|
38 |
-
RGB = "RGB" # Standard color texture
|
39 |
-
PBR = "PBR" # Physically-based rendering (metallic, roughness, normal)
|
40 |
-
|
41 |
-
class ExportFormat(Enum):
|
42 |
-
"""3D model export formats"""
|
43 |
-
GLB = "glb" # GLTF binary format (recommended)
|
44 |
-
OBJ = "obj" # Wavefront OBJ format
|
45 |
-
PLY = "ply" # Polygon File Format
|
46 |
-
STL = "stl" # Stereolithography format
|
47 |
-
|
48 |
-
@dataclass
|
49 |
-
class Hunyuan3DConfig:
|
50 |
-
"""Configuration for Hunyuan3D pipeline"""
|
51 |
-
# API Configuration
|
52 |
-
space_id: str = "Tencent/Hunyuan3D-2" # Official Hugging Face Space
|
53 |
-
use_auth_token: Optional[str] = None # HF auth token if needed
|
54 |
-
max_retries: int = 3
|
55 |
-
timeout: int = 600 # 10 minutes max per generation
|
56 |
-
|
57 |
-
# Generation Settings
|
58 |
-
default_mode: GenerationMode = GenerationMode.FAST
|
59 |
-
texture_method: TextureMethod = TextureMethod.RGB
|
60 |
-
export_format: ExportFormat = ExportFormat.GLB
|
61 |
-
|
62 |
-
# Quality Settings
|
63 |
-
shape_resolution: int = 256 # Internal shape generation resolution
|
64 |
-
texture_resolution: int = 1024 # Output texture resolution
|
65 |
-
remove_bg: bool = True # Remove background from input images
|
66 |
-
foreground_ratio: float = 0.85 # Object size in frame
|
67 |
-
|
68 |
-
# Optimization Settings
|
69 |
-
enable_optimization: bool = True
|
70 |
-
target_polycount: int = 30000 # Higher quality than other pipelines
|
71 |
-
simplify_ratio: float = 0.5 # Mesh simplification ratio
|
72 |
-
|
73 |
-
# File Management
|
74 |
-
output_dir: Path = Path("./hunyuan3d_output")
|
75 |
-
cache_dir: Path = Path("./hunyuan3d_cache")
|
76 |
-
temp_dir: Path = Path("./hunyuan3d_temp")
|
77 |
-
keep_intermediates: bool = False # Keep intermediate files for debugging
|
78 |
-
|
79 |
-
# Multi-view Settings
|
80 |
-
enable_multi_view: bool = True
|
81 |
-
views: List[str] = field(default_factory=lambda: ["front", "back", "left", "right"])
|
82 |
-
|
83 |
-
class Hunyuan3DClient:
|
84 |
-
"""Client for interacting with Hunyuan3D Space API"""
|
85 |
-
|
86 |
-
def __init__(self, config: Hunyuan3DConfig):
|
87 |
-
self.config = config
|
88 |
-
self.client = None
|
89 |
-
self._initialize_client()
|
90 |
-
|
91 |
-
def _initialize_client(self):
|
92 |
-
"""Initialize Gradio client for Hunyuan3D Space"""
|
93 |
-
try:
|
94 |
-
logger.info(f"Connecting to Hunyuan3D Space: {self.config.space_id}")
|
95 |
-
self.client = Client(
|
96 |
-
self.config.space_id,
|
97 |
-
hf_token=self.config.use_auth_token
|
98 |
-
)
|
99 |
-
logger.info("Successfully connected to Hunyuan3D Space")
|
100 |
-
except Exception as e:
|
101 |
-
logger.error(f"Failed to connect to Hunyuan3D Space: {e}")
|
102 |
-
raise
|
103 |
-
|
104 |
-
async def generate_from_image(self,
|
105 |
-
image_path: Union[str, Path],
|
106 |
-
mode: GenerationMode,
|
107 |
-
remove_bg: bool = True,
|
108 |
-
foreground_ratio: float = 0.85) -> Dict[str, Any]:
|
109 |
-
"""Generate 3D model from single image"""
|
110 |
-
|
111 |
-
logger.info(f"Generating 3D model from image: {image_path}")
|
112 |
-
|
113 |
-
try:
|
114 |
-
# Prepare parameters
|
115 |
-
params = {
|
116 |
-
"image": handle_file(str(image_path)),
|
117 |
-
"mode": mode.value,
|
118 |
-
"remove_bg": remove_bg,
|
119 |
-
"foreground_ratio": foreground_ratio,
|
120 |
-
"texture_method": self.config.texture_method.value
|
121 |
-
}
|
122 |
-
|
123 |
-
# Stage 1: Shape generation
|
124 |
-
logger.info("Stage 1: Generating 3D shape...")
|
125 |
-
shape_result = await self._run_generation(
|
126 |
-
self.client.predict,
|
127 |
-
"/generate_shape",
|
128 |
-
**params
|
129 |
-
)
|
130 |
-
|
131 |
-
# Stage 2: Texture generation
|
132 |
-
logger.info("Stage 2: Generating textures...")
|
133 |
-
texture_result = await self._run_generation(
|
134 |
-
self.client.predict,
|
135 |
-
"/generate_texture",
|
136 |
-
shape_data=shape_result,
|
137 |
-
**params
|
138 |
-
)
|
139 |
-
|
140 |
-
# Process results
|
141 |
-
return self._process_results(shape_result, texture_result)
|
142 |
-
|
143 |
-
except Exception as e:
|
144 |
-
logger.error(f"Generation failed: {e}")
|
145 |
-
raise
|
146 |
-
|
147 |
-
async def generate_from_multi_view(self,
|
148 |
-
image_paths: Dict[str, Union[str, Path]],
|
149 |
-
mode: GenerationMode) -> Dict[str, Any]:
|
150 |
-
"""Generate 3D model from multiple view images"""
|
151 |
-
|
152 |
-
logger.info(f"Generating 3D model from {len(image_paths)} views")
|
153 |
-
|
154 |
-
try:
|
155 |
-
# Validate views
|
156 |
-
required_views = ["front", "back", "left", "right"]
|
157 |
-
for view in required_views:
|
158 |
-
if view not in image_paths:
|
159 |
-
raise ValueError(f"Missing required view: {view}")
|
160 |
-
|
161 |
-
# Prepare multi-view parameters
|
162 |
-
params = {
|
163 |
-
"front_image": handle_file(str(image_paths["front"])),
|
164 |
-
"back_image": handle_file(str(image_paths["back"])),
|
165 |
-
"left_image": handle_file(str(image_paths["left"])),
|
166 |
-
"right_image": handle_file(str(image_paths["right"])),
|
167 |
-
"mode": mode.value,
|
168 |
-
"texture_method": self.config.texture_method.value,
|
169 |
-
"use_multi_view": True
|
170 |
-
}
|
171 |
-
|
172 |
-
# Multi-view generation
|
173 |
-
logger.info("Generating from multi-view inputs...")
|
174 |
-
result = await self._run_generation(
|
175 |
-
self.client.predict,
|
176 |
-
"/generate_multi_view",
|
177 |
-
**params
|
178 |
-
)
|
179 |
-
|
180 |
-
return self._process_multi_view_results(result)
|
181 |
-
|
182 |
-
except Exception as e:
|
183 |
-
logger.error(f"Multi-view generation failed: {e}")
|
184 |
-
raise
|
185 |
-
|
186 |
-
async def _run_generation(self, func, endpoint: str, **kwargs) -> Any:
|
187 |
-
"""Run generation with retries and error handling"""
|
188 |
-
|
189 |
-
for attempt in range(self.config.max_retries):
|
190 |
-
try:
|
191 |
-
# Run prediction
|
192 |
-
result = await asyncio.get_event_loop().run_in_executor(
|
193 |
-
None,
|
194 |
-
lambda: func(endpoint, **kwargs)
|
195 |
-
)
|
196 |
-
return result
|
197 |
-
|
198 |
-
except Exception as e:
|
199 |
-
if attempt < self.config.max_retries - 1:
|
200 |
-
wait_time = 2 ** attempt # Exponential backoff
|
201 |
-
logger.warning(f"Attempt {attempt + 1} failed, retrying in {wait_time}s...")
|
202 |
-
await asyncio.sleep(wait_time)
|
203 |
-
else:
|
204 |
-
raise e
|
205 |
-
|
206 |
-
def _process_results(self, shape_result: Any, texture_result: Any) -> Dict[str, Any]:
|
207 |
-
"""Process generation results"""
|
208 |
-
|
209 |
-
# Extract file paths from results
|
210 |
-
model_files = {}
|
211 |
-
|
212 |
-
# Handle different result formats
|
213 |
-
if isinstance(texture_result, tuple):
|
214 |
-
# Multiple outputs (model, textures, etc.)
|
215 |
-
model_files["model"] = texture_result[0]
|
216 |
-
if len(texture_result) > 1:
|
217 |
-
model_files["texture"] = texture_result[1]
|
218 |
-
if len(texture_result) > 2:
|
219 |
-
model_files["preview"] = texture_result[2]
|
220 |
-
else:
|
221 |
-
model_files["model"] = texture_result
|
222 |
-
|
223 |
-
# Extract metadata
|
224 |
-
metadata = {
|
225 |
-
"generation_mode": self.config.default_mode.value,
|
226 |
-
"texture_method": self.config.texture_method.value,
|
227 |
-
"timestamp": time.time()
|
228 |
-
}
|
229 |
-
|
230 |
-
return {
|
231 |
-
"success": True,
|
232 |
-
"files": model_files,
|
233 |
-
"metadata": metadata
|
234 |
-
}
|
235 |
-
|
236 |
-
def _process_multi_view_results(self, result: Any) -> Dict[str, Any]:
|
237 |
-
"""Process multi-view generation results"""
|
238 |
-
|
239 |
-
model_files = {}
|
240 |
-
|
241 |
-
if isinstance(result, dict):
|
242 |
-
model_files = result
|
243 |
-
elif isinstance(result, tuple):
|
244 |
-
model_files["model"] = result[0]
|
245 |
-
if len(result) > 1:
|
246 |
-
model_files["texture"] = result[1]
|
247 |
-
else:
|
248 |
-
model_files["model"] = result
|
249 |
-
|
250 |
-
return {
|
251 |
-
"success": True,
|
252 |
-
"files": model_files,
|
253 |
-
"metadata": {
|
254 |
-
"generation_type": "multi_view",
|
255 |
-
"views_used": self.config.views
|
256 |
-
}
|
257 |
-
}
|
258 |
-
|
259 |
-
class Hunyuan3DProcessor:
|
260 |
-
"""Post-processing for Hunyuan3D outputs"""
|
261 |
-
|
262 |
-
def __init__(self, config: Hunyuan3DConfig):
|
263 |
-
self.config = config
|
264 |
-
|
265 |
-
async def process_model(self,
|
266 |
-
raw_model_path: Path,
|
267 |
-
creature_name: str) -> Dict[str, Any]:
|
268 |
-
"""Process and optimize raw 3D model"""
|
269 |
-
|
270 |
-
logger.info(f"Processing model: {raw_model_path}")
|
271 |
-
|
272 |
-
# Load model
|
273 |
-
mesh = trimesh.load(raw_model_path)
|
274 |
-
if isinstance(mesh, trimesh.Scene):
|
275 |
-
mesh = mesh.dump(concatenate=True)
|
276 |
-
|
277 |
-
original_stats = {
|
278 |
-
"vertices": len(mesh.vertices),
|
279 |
-
"faces": len(mesh.faces),
|
280 |
-
"bounds": mesh.bounds.tolist()
|
281 |
-
}
|
282 |
-
|
283 |
-
# Optimize if enabled
|
284 |
-
if self.config.enable_optimization:
|
285 |
-
mesh = self._optimize_mesh(mesh)
|
286 |
-
|
287 |
-
# Export in requested format
|
288 |
-
output_path = self.config.output_dir / f"{creature_name}_processed.{self.config.export_format.value}"
|
289 |
-
mesh.export(output_path)
|
290 |
-
|
291 |
-
# Generate preview
|
292 |
-
preview_path = await self._generate_preview(mesh, creature_name)
|
293 |
-
|
294 |
-
return {
|
295 |
-
"processed_model": output_path,
|
296 |
-
"preview": preview_path,
|
297 |
-
"stats": {
|
298 |
-
"original": original_stats,
|
299 |
-
"optimized": {
|
300 |
-
"vertices": len(mesh.vertices),
|
301 |
-
"faces": len(mesh.faces)
|
302 |
-
}
|
303 |
-
}
|
304 |
-
}
|
305 |
-
|
306 |
-
def _optimize_mesh(self, mesh: trimesh.Trimesh) -> trimesh.Trimesh:
|
307 |
-
"""Optimize mesh for game use"""
|
308 |
-
|
309 |
-
logger.info(f"Optimizing mesh: {len(mesh.faces)} faces -> {self.config.target_polycount}")
|
310 |
-
|
311 |
-
# Clean up mesh
|
312 |
-
mesh.remove_degenerate_faces()
|
313 |
-
mesh.remove_duplicate_faces()
|
314 |
-
mesh.remove_unreferenced_vertices()
|
315 |
-
|
316 |
-
# Simplify if needed
|
317 |
-
if len(mesh.faces) > self.config.target_polycount:
|
318 |
-
mesh = mesh.simplify_quadric_decimation(self.config.target_polycount)
|
319 |
-
|
320 |
-
# Ensure watertight
|
321 |
-
mesh.fill_holes()
|
322 |
-
|
323 |
-
# Smooth normals
|
324 |
-
mesh.vertex_normals
|
325 |
-
|
326 |
-
return mesh
|
327 |
-
|
328 |
-
async def _generate_preview(self, mesh: trimesh.Trimesh, name: str) -> Path:
|
329 |
-
"""Generate preview image of mesh"""
|
330 |
-
|
331 |
-
# Create scene
|
332 |
-
scene = mesh.scene()
|
333 |
-
|
334 |
-
# Set camera angle
|
335 |
-
angles = [(np.pi / 6, np.pi / 4)] # 30 degrees elevation, 45 degrees azimuth
|
336 |
-
|
337 |
-
for elevation, azimuth in angles:
|
338 |
-
camera_transform = scene.camera.look_at(
|
339 |
-
mesh.bounds.mean(axis=0),
|
340 |
-
distance=mesh.scale * 2.5,
|
341 |
-
elevation=elevation,
|
342 |
-
azimuth=azimuth
|
343 |
-
)
|
344 |
-
scene.camera_transform = camera_transform
|
345 |
-
|
346 |
-
# Render
|
347 |
-
try:
|
348 |
-
data = scene.save_image(resolution=[1024, 1024], visible=False)
|
349 |
-
preview_path = self.config.output_dir / f"{name}_preview.png"
|
350 |
-
|
351 |
-
if data:
|
352 |
-
with open(preview_path, 'wb') as f:
|
353 |
-
f.write(data.getvalue())
|
354 |
-
return preview_path
|
355 |
-
except:
|
356 |
-
pass
|
357 |
-
|
358 |
-
# Fallback preview
|
359 |
-
preview_path = self.config.output_dir / f"{name}_preview.png"
|
360 |
-
img = Image.new('RGBA', (1024, 1024), (200, 200, 200, 255))
|
361 |
-
img.save(preview_path)
|
362 |
-
return preview_path
|
363 |
-
|
364 |
-
class Hunyuan3DPipeline:
|
365 |
-
"""
|
366 |
-
Complete Hunyuan3D integration pipeline for DigiPal
|
367 |
-
Philosophy: Leverage cutting-edge AI while maintaining simplicity
|
368 |
-
"""
|
369 |
-
|
370 |
-
def __init__(self, config: Optional[Hunyuan3DConfig] = None):
|
371 |
-
self.config = config or Hunyuan3DConfig()
|
372 |
-
|
373 |
-
# Initialize components
|
374 |
-
self.client = Hunyuan3DClient(self.config)
|
375 |
-
self.processor = Hunyuan3DProcessor(self.config)
|
376 |
-
|
377 |
-
# Create directories
|
378 |
-
self.config.output_dir.mkdir(parents=True, exist_ok=True)
|
379 |
-
self.config.cache_dir.mkdir(parents=True, exist_ok=True)
|
380 |
-
self.config.temp_dir.mkdir(parents=True, exist_ok=True)
|
381 |
-
|
382 |
-
# Cache for generated models
|
383 |
-
self.cache = self._load_cache()
|
384 |
-
|
385 |
-
logger.info("Hunyuan3D pipeline initialized")
|
386 |
-
|
387 |
-
def _load_cache(self) -> Dict[str, Any]:
|
388 |
-
"""Load generation cache"""
|
389 |
-
cache_file = self.config.cache_dir / "generation_cache.json"
|
390 |
-
if cache_file.exists():
|
391 |
-
with open(cache_file, 'r') as f:
|
392 |
-
return json.load(f)
|
393 |
-
return {}
|
394 |
-
|
395 |
-
def _save_cache(self):
|
396 |
-
"""Save generation cache"""
|
397 |
-
cache_file = self.config.cache_dir / "generation_cache.json"
|
398 |
-
with open(cache_file, 'w') as f:
|
399 |
-
json.dump(self.cache, f, indent=2)
|
400 |
-
|
401 |
-
async def generate_from_text(self,
|
402 |
-
prompt: str,
|
403 |
-
name: str,
|
404 |
-
mode: Optional[GenerationMode] = None,
|
405 |
-
style: Optional[str] = None) -> Dict[str, Any]:
|
406 |
-
"""Generate 3D model from text prompt"""
|
407 |
-
|
408 |
-
start_time = time.time()
|
409 |
-
mode = mode or self.config.default_mode
|
410 |
-
|
411 |
-
# Check cache
|
412 |
-
cache_key = f"{prompt}_{mode.value}_{style or 'default'}"
|
413 |
-
if cache_key in self.cache:
|
414 |
-
logger.info(f"Using cached model for: {prompt}")
|
415 |
-
return self.cache[cache_key]
|
416 |
-
|
417 |
-
try:
|
418 |
-
# Step 1: Generate concept image from text
|
419 |
-
logger.info(f"Generating concept image for: {prompt}")
|
420 |
-
concept_image = await self._generate_concept_image(prompt, style)
|
421 |
-
|
422 |
-
# Step 2: Generate 3D from image
|
423 |
-
logger.info("Converting to 3D with Hunyuan3D...")
|
424 |
-
generation_result = await self.client.generate_from_image(
|
425 |
-
concept_image,
|
426 |
-
mode,
|
427 |
-
self.config.remove_bg,
|
428 |
-
self.config.foreground_ratio
|
429 |
-
)
|
430 |
-
|
431 |
-
if not generation_result["success"]:
|
432 |
-
raise Exception("3D generation failed")
|
433 |
-
|
434 |
-
# Step 3: Process and optimize
|
435 |
-
logger.info("Processing generated model...")
|
436 |
-
process_result = await self.processor.process_model(
|
437 |
-
Path(generation_result["files"]["model"]),
|
438 |
-
name
|
439 |
-
)
|
440 |
-
|
441 |
-
# Step 4: Handle textures
|
442 |
-
texture_paths = {}
|
443 |
-
if "texture" in generation_result["files"]:
|
444 |
-
texture_paths["diffuse"] = generation_result["files"]["texture"]
|
445 |
-
|
446 |
-
# Compile results
|
447 |
-
result = {
|
448 |
-
"success": True,
|
449 |
-
"name": name,
|
450 |
-
"prompt": prompt,
|
451 |
-
"mode": mode.value,
|
452 |
-
"style": style,
|
453 |
-
"pipeline": "hunyuan3d",
|
454 |
-
"paths": {
|
455 |
-
"concept_image": str(concept_image),
|
456 |
-
"raw_model": generation_result["files"]["model"],
|
457 |
-
"processed_model": str(process_result["processed_model"]),
|
458 |
-
"preview": str(process_result["preview"])
|
459 |
-
},
|
460 |
-
"textures": texture_paths,
|
461 |
-
"stats": {
|
462 |
-
"generation_time": time.time() - start_time,
|
463 |
-
**process_result["stats"]
|
464 |
-
},
|
465 |
-
"metadata": generation_result["metadata"]
|
466 |
-
}
|
467 |
-
|
468 |
-
# Cache result
|
469 |
-
self.cache[cache_key] = result
|
470 |
-
self._save_cache()
|
471 |
-
|
472 |
-
# Save metadata
|
473 |
-
metadata_path = self.config.output_dir / f"{name}_metadata.json"
|
474 |
-
with open(metadata_path, 'w') as f:
|
475 |
-
json.dump(result, f, indent=2)
|
476 |
-
|
477 |
-
logger.info(f"Generation completed in {result['stats']['generation_time']:.2f}s")
|
478 |
-
return result
|
479 |
-
|
480 |
-
except Exception as e:
|
481 |
-
logger.error(f"Pipeline failed: {e}")
|
482 |
-
return {
|
483 |
-
"success": False,
|
484 |
-
"error": str(e),
|
485 |
-
"name": name,
|
486 |
-
"prompt": prompt
|
487 |
-
}
|
488 |
-
|
489 |
-
async def generate_from_image(self,
|
490 |
-
image_path: Union[str, Path],
|
491 |
-
name: str,
|
492 |
-
mode: Optional[GenerationMode] = None) -> Dict[str, Any]:
|
493 |
-
"""Generate 3D model from single image"""
|
494 |
-
|
495 |
-
start_time = time.time()
|
496 |
-
mode = mode or self.config.default_mode
|
497 |
-
|
498 |
-
try:
|
499 |
-
# Generate 3D from image
|
500 |
-
logger.info(f"Generating 3D from image: {image_path}")
|
501 |
-
generation_result = await self.client.generate_from_image(
|
502 |
-
image_path,
|
503 |
-
mode,
|
504 |
-
self.config.remove_bg,
|
505 |
-
self.config.foreground_ratio
|
506 |
-
)
|
507 |
-
|
508 |
-
if not generation_result["success"]:
|
509 |
-
raise Exception("3D generation failed")
|
510 |
-
|
511 |
-
# Process model
|
512 |
-
process_result = await self.processor.process_model(
|
513 |
-
Path(generation_result["files"]["model"]),
|
514 |
-
name
|
515 |
-
)
|
516 |
-
|
517 |
-
# Compile results
|
518 |
-
result = {
|
519 |
-
"success": True,
|
520 |
-
"name": name,
|
521 |
-
"input_image": str(image_path),
|
522 |
-
"mode": mode.value,
|
523 |
-
"pipeline": "hunyuan3d",
|
524 |
-
"paths": {
|
525 |
-
"input_image": str(image_path),
|
526 |
-
"raw_model": generation_result["files"]["model"],
|
527 |
-
"processed_model": str(process_result["processed_model"]),
|
528 |
-
"preview": str(process_result["preview"])
|
529 |
-
},
|
530 |
-
"stats": {
|
531 |
-
"generation_time": time.time() - start_time,
|
532 |
-
**process_result["stats"]
|
533 |
-
},
|
534 |
-
"metadata": generation_result["metadata"]
|
535 |
-
}
|
536 |
-
|
537 |
-
return result
|
538 |
-
|
539 |
-
except Exception as e:
|
540 |
-
logger.error(f"Pipeline failed: {e}")
|
541 |
-
return {
|
542 |
-
"success": False,
|
543 |
-
"error": str(e),
|
544 |
-
"name": name,
|
545 |
-
"input_image": str(image_path)
|
546 |
-
}
|
547 |
-
|
548 |
-
async def generate_from_multi_view(self,
|
549 |
-
image_paths: Dict[str, Union[str, Path]],
|
550 |
-
name: str,
|
551 |
-
mode: Optional[GenerationMode] = None) -> Dict[str, Any]:
|
552 |
-
"""Generate 3D model from multiple view images"""
|
553 |
-
|
554 |
-
start_time = time.time()
|
555 |
-
mode = mode or self.config.default_mode
|
556 |
-
|
557 |
-
try:
|
558 |
-
# Generate 3D from multi-view
|
559 |
-
logger.info(f"Generating 3D from {len(image_paths)} views")
|
560 |
-
generation_result = await self.client.generate_from_multi_view(
|
561 |
-
image_paths,
|
562 |
-
mode
|
563 |
-
)
|
564 |
-
|
565 |
-
if not generation_result["success"]:
|
566 |
-
raise Exception("Multi-view generation failed")
|
567 |
-
|
568 |
-
# Process model
|
569 |
-
process_result = await self.processor.process_model(
|
570 |
-
Path(generation_result["files"]["model"]),
|
571 |
-
name
|
572 |
-
)
|
573 |
-
|
574 |
-
# Compile results
|
575 |
-
result = {
|
576 |
-
"success": True,
|
577 |
-
"name": name,
|
578 |
-
"input_views": {k: str(v) for k, v in image_paths.items()},
|
579 |
-
"mode": mode.value,
|
580 |
-
"pipeline": "hunyuan3d_multiview",
|
581 |
-
"paths": {
|
582 |
-
"raw_model": generation_result["files"]["model"],
|
583 |
-
"processed_model": str(process_result["processed_model"]),
|
584 |
-
"preview": str(process_result["preview"])
|
585 |
-
},
|
586 |
-
"stats": {
|
587 |
-
"generation_time": time.time() - start_time,
|
588 |
-
**process_result["stats"]
|
589 |
-
},
|
590 |
-
"metadata": generation_result["metadata"]
|
591 |
-
}
|
592 |
-
|
593 |
-
return result
|
594 |
-
|
595 |
-
except Exception as e:
|
596 |
-
logger.error(f"Multi-view pipeline failed: {e}")
|
597 |
-
return {
|
598 |
-
"success": False,
|
599 |
-
"error": str(e),
|
600 |
-
"name": name,
|
601 |
-
"input_views": {k: str(v) for k, v in image_paths.items()}
|
602 |
-
}
|
603 |
-
|
604 |
-
async def _generate_concept_image(self, prompt: str, style: Optional[str] = None) -> Path:
|
605 |
-
"""Generate concept image from text (placeholder - integrate with your image gen)"""
|
606 |
-
|
607 |
-
# This is a placeholder - integrate with your preferred text-to-image model
|
608 |
-
# For example, could use SDXL, Flux, or other models
|
609 |
-
|
610 |
-
logger.info("Generating concept image...")
|
611 |
-
|
612 |
-
# For now, create a placeholder image
|
613 |
-
img = Image.new('RGB', (1024, 1024), (100, 100, 100))
|
614 |
-
|
615 |
-
# Add some text to indicate it's a placeholder
|
616 |
-
from PIL import ImageDraw, ImageFont
|
617 |
-
draw = ImageDraw.Draw(img)
|
618 |
-
text = f"Concept: {prompt[:50]}..."
|
619 |
-
|
620 |
-
try:
|
621 |
-
# Try to use a font
|
622 |
-
font = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 40)
|
623 |
-
except:
|
624 |
-
font = ImageFont.load_default()
|
625 |
-
|
626 |
-
# Draw text
|
627 |
-
bbox = draw.textbbox((0, 0), text, font=font)
|
628 |
-
text_width = bbox[2] - bbox[0]
|
629 |
-
text_height = bbox[3] - bbox[1]
|
630 |
-
position = ((1024 - text_width) // 2, (1024 - text_height) // 2)
|
631 |
-
draw.text(position, text, fill='white', font=font)
|
632 |
-
|
633 |
-
# Save image
|
634 |
-
concept_path = self.config.temp_dir / f"concept_{hash(prompt)}.png"
|
635 |
-
img.save(concept_path)
|
636 |
-
|
637 |
-
return concept_path
|
638 |
-
|
639 |
-
async def generate_monster_for_digipal(self,
|
640 |
-
monster_data: Dict[str, Any],
|
641 |
-
mode: Optional[GenerationMode] = None) -> Dict[str, Any]:
|
642 |
-
"""Generate 3D model specifically for DigiPal monster"""
|
643 |
-
|
644 |
-
# Extract monster characteristics
|
645 |
-
name = monster_data.get("name", "Monster")
|
646 |
-
stage = monster_data.get("stage", "child")
|
647 |
-
personality = monster_data.get("personality", "friendly")
|
648 |
-
species_type = monster_data.get("species_type", "data")
|
649 |
-
|
650 |
-
# Create detailed prompt
|
651 |
-
prompt = self._create_monster_prompt(monster_data)
|
652 |
-
|
653 |
-
# Determine generation mode based on stage
|
654 |
-
if mode is None:
|
655 |
-
stage_modes = {
|
656 |
-
"baby": GenerationMode.TURBO,
|
657 |
-
"child": GenerationMode.FAST,
|
658 |
-
"adult": GenerationMode.STANDARD,
|
659 |
-
"perfect": GenerationMode.STANDARD,
|
660 |
-
"ultimate": GenerationMode.STANDARD
|
661 |
-
}
|
662 |
-
mode = stage_modes.get(stage, GenerationMode.FAST)
|
663 |
-
|
664 |
-
# Generate model
|
665 |
-
result = await self.generate_from_text(
|
666 |
-
prompt=prompt,
|
667 |
-
name=f"{name}_{stage}",
|
668 |
-
mode=mode,
|
669 |
-
style=personality
|
670 |
-
)
|
671 |
-
|
672 |
-
# Add monster-specific metadata
|
673 |
-
if result["success"]:
|
674 |
-
result["monster_metadata"] = monster_data
|
675 |
-
|
676 |
-
return result
|
677 |
-
|
678 |
-
def _create_monster_prompt(self, monster_data: Dict[str, Any]) -> str:
|
679 |
-
"""Create detailed prompt from monster data with consistent style"""
|
680 |
-
|
681 |
-
# Core visual style - consistent across all stages
|
682 |
-
base_style = "AAA 3D render, well lit, white void background"
|
683 |
-
|
684 |
-
# Size descriptors that maintain similar adult proportions
|
685 |
-
stage_size = {
|
686 |
-
"egg": "small egg-shaped",
|
687 |
-
"baby": "compact young",
|
688 |
-
"child": "medium-sized adolescent",
|
689 |
-
"adult": "full-sized mature",
|
690 |
-
"perfect": "large powerful",
|
691 |
-
"ultimate": "imposing legendary",
|
692 |
-
"mega": "colossal mythical"
|
693 |
-
}
|
694 |
-
|
695 |
-
# Stage modifiers - subtle changes while maintaining core design
|
696 |
-
stage_modifier = {
|
697 |
-
"egg": "dormant crystalline",
|
698 |
-
"baby": "newly formed",
|
699 |
-
"child": "developing",
|
700 |
-
"adult": "fully formed",
|
701 |
-
"perfect": "enhanced",
|
702 |
-
"ultimate": "transcendent",
|
703 |
-
"mega": "apex"
|
704 |
-
}
|
705 |
-
|
706 |
-
# Personality traits - affects posture and expression
|
707 |
-
personality_traits = {
|
708 |
-
"brave": "fierce stance, determined expression",
|
709 |
-
"calm": "serene posture, wise countenance",
|
710 |
-
"energetic": "dynamic pose, alert expression",
|
711 |
-
"clever": "calculating gaze, strategic stance",
|
712 |
-
"friendly": "approachable posture, gentle expression"
|
713 |
-
}
|
714 |
-
|
715 |
-
# Species visual themes - consistent design language
|
716 |
-
species_theme = {
|
717 |
-
"data": "digital creature with crystalline segments, circuit patterns, holographic accents",
|
718 |
-
"vaccine": "angelic creature with white armor plating, golden trim, light aura",
|
719 |
-
"virus": "dark creature with shadowy armor, purple energy, corrupted data streams",
|
720 |
-
"free": "natural creature with organic armor, earth tones, elemental energy"
|
721 |
-
}
|
722 |
-
|
723 |
-
# Extract attributes
|
724 |
-
stage = monster_data.get("stage", "child")
|
725 |
-
personality = monster_data.get("personality", "friendly")
|
726 |
-
species = monster_data.get("species_type", "data")
|
727 |
-
name = monster_data.get("name", "DigiPal")
|
728 |
-
|
729 |
-
# Build consistent prompt
|
730 |
-
size = stage_size.get(stage, "medium-sized")
|
731 |
-
modifier = stage_modifier.get(stage, "")
|
732 |
-
traits = personality_traits.get(personality, "neutral stance")
|
733 |
-
theme = species_theme.get(species, "digital creature")
|
734 |
-
|
735 |
-
# Stats-based features - subtle additions
|
736 |
-
features = []
|
737 |
-
stats = monster_data.get("stats", {})
|
738 |
-
if stats.get("offense", 0) > 70:
|
739 |
-
features.append("reinforced claws")
|
740 |
-
if stats.get("defense", 0) > 70:
|
741 |
-
features.append("enhanced armor")
|
742 |
-
if stats.get("speed", 0) > 70:
|
743 |
-
features.append("streamlined form")
|
744 |
-
if stats.get("brains", 0) > 70:
|
745 |
-
features.append("glowing intelligence core")
|
746 |
-
|
747 |
-
feature_str = f", {', '.join(features)}" if features else ""
|
748 |
-
|
749 |
-
# Construct final prompt with consistent style
|
750 |
-
prompt = (
|
751 |
-
f"{base_style}, {size} {modifier} {theme}, "
|
752 |
-
f"{traits}{feature_str}, "
|
753 |
-
f"T-pose, game character model, consistent design language, "
|
754 |
-
f"professional 3D game asset, pbr materials, clean topology"
|
755 |
-
)
|
756 |
-
|
757 |
-
# Add stage-specific quality hints
|
758 |
-
if stage in ["ultimate", "mega", "perfect"]:
|
759 |
-
prompt += ", hero character quality, highly detailed"
|
760 |
-
elif stage in ["baby", "egg"]:
|
761 |
-
prompt += ", simplified design, cute proportions"
|
762 |
-
|
763 |
-
return prompt
|
764 |
-
|
765 |
-
async def batch_generate(self,
|
766 |
-
generation_tasks: List[Dict[str, Any]],
|
767 |
-
max_concurrent: int = 2) -> List[Dict[str, Any]]:
|
768 |
-
"""Batch generate multiple models"""
|
769 |
-
|
770 |
-
semaphore = asyncio.Semaphore(max_concurrent)
|
771 |
-
|
772 |
-
async def generate_with_limit(task: Dict[str, Any]):
|
773 |
-
async with semaphore:
|
774 |
-
if "prompt" in task:
|
775 |
-
return await self.generate_from_text(
|
776 |
-
prompt=task["prompt"],
|
777 |
-
name=task["name"],
|
778 |
-
mode=task.get("mode"),
|
779 |
-
style=task.get("style")
|
780 |
-
)
|
781 |
-
elif "image" in task:
|
782 |
-
return await self.generate_from_image(
|
783 |
-
image_path=task["image"],
|
784 |
-
name=task["name"],
|
785 |
-
mode=task.get("mode")
|
786 |
-
)
|
787 |
-
elif "images" in task:
|
788 |
-
return await self.generate_from_multi_view(
|
789 |
-
image_paths=task["images"],
|
790 |
-
name=task["name"],
|
791 |
-
mode=task.get("mode")
|
792 |
-
)
|
793 |
-
else:
|
794 |
-
return {
|
795 |
-
"success": False,
|
796 |
-
"error": "Invalid task format",
|
797 |
-
"task": task
|
798 |
-
}
|
799 |
-
|
800 |
-
tasks = [generate_with_limit(task) for task in generation_tasks]
|
801 |
-
results = await asyncio.gather(*tasks)
|
802 |
-
|
803 |
-
return results
|
804 |
-
|
805 |
-
# Integration with DigiPal's monster system
|
806 |
-
class DigiPalHunyuan3DIntegration:
|
807 |
-
"""Integration layer for DigiPal monster system"""
|
808 |
-
|
809 |
-
def __init__(self, config: Optional[Hunyuan3DConfig] = None):
|
810 |
-
self.pipeline = Hunyuan3DPipeline(config)
|
811 |
-
|
812 |
-
async def generate_monster_model(self,
|
813 |
-
monster: Any, # DW1Monster type
|
814 |
-
force_regenerate: bool = False) -> Dict[str, Any]:
|
815 |
-
"""Generate 3D model for DigiPal monster"""
|
816 |
-
|
817 |
-
# Convert monster to data dict
|
818 |
-
monster_data = {
|
819 |
-
"name": monster.name,
|
820 |
-
"stage": monster.evolution.current_stage.value,
|
821 |
-
"personality": monster.personality.value,
|
822 |
-
"species_type": monster.species_type.value,
|
823 |
-
"stats": {
|
824 |
-
"offense": monster.battle_stats.offense,
|
825 |
-
"defense": monster.battle_stats.defense,
|
826 |
-
"speed": monster.battle_stats.speed,
|
827 |
-
"brains": monster.battle_stats.brains
|
828 |
-
}
|
829 |
-
}
|
830 |
-
|
831 |
-
# Check cache unless forced
|
832 |
-
cache_key = f"{monster.id}_{monster.evolution.current_stage.value}"
|
833 |
-
if not force_regenerate and cache_key in self.pipeline.cache:
|
834 |
-
logger.info(f"Using cached model for {monster.name}")
|
835 |
-
return self.pipeline.cache[cache_key]
|
836 |
-
|
837 |
-
# Generate model
|
838 |
-
result = await self.pipeline.generate_monster_for_digipal(monster_data)
|
839 |
-
|
840 |
-
# Cache if successful
|
841 |
-
if result["success"]:
|
842 |
-
self.pipeline.cache[cache_key] = result
|
843 |
-
self.pipeline._save_cache()
|
844 |
-
|
845 |
-
return result
|
846 |
-
|
847 |
-
async def generate_evolution_sequence(self,
|
848 |
-
monster: Any,
|
849 |
-
stages: List[str]) -> List[Dict[str, Any]]:
|
850 |
-
"""Generate models for evolution sequence"""
|
851 |
-
|
852 |
-
tasks = []
|
853 |
-
for stage in stages:
|
854 |
-
monster_data = {
|
855 |
-
"name": monster.name,
|
856 |
-
"stage": stage,
|
857 |
-
"personality": monster.personality.value,
|
858 |
-
"species_type": monster.species_type.value,
|
859 |
-
"stats": {
|
860 |
-
"offense": monster.battle_stats.offense,
|
861 |
-
"defense": monster.battle_stats.defense,
|
862 |
-
"speed": monster.battle_stats.speed,
|
863 |
-
"brains": monster.battle_stats.brains
|
864 |
-
}
|
865 |
-
}
|
866 |
-
|
867 |
-
tasks.append({
|
868 |
-
"monster_data": monster_data,
|
869 |
-
"mode": GenerationMode.FAST if stage in ["baby", "child"] else GenerationMode.STANDARD
|
870 |
-
})
|
871 |
-
|
872 |
-
# Generate all stages
|
873 |
-
results = []
|
874 |
-
for task in tasks:
|
875 |
-
result = await self.pipeline.generate_monster_for_digipal(
|
876 |
-
task["monster_data"],
|
877 |
-
task["mode"]
|
878 |
-
)
|
879 |
-
results.append(result)
|
880 |
-
|
881 |
-
return results
|
882 |
-
|
883 |
-
# Convenience functions
|
884 |
-
def create_hunyuan3d_config(output_path: str = "hunyuan3d_config.json"):
|
885 |
-
"""Create configuration template"""
|
886 |
-
|
887 |
-
config = {
|
888 |
-
"space_id": "Tencent/Hunyuan3D-2",
|
889 |
-
"use_auth_token": None,
|
890 |
-
"max_retries": 3,
|
891 |
-
"timeout": 600,
|
892 |
-
"default_mode": "Fast",
|
893 |
-
"texture_method": "RGB",
|
894 |
-
"export_format": "glb",
|
895 |
-
"shape_resolution": 256,
|
896 |
-
"texture_resolution": 1024,
|
897 |
-
"remove_bg": True,
|
898 |
-
"foreground_ratio": 0.85,
|
899 |
-
"enable_optimization": True,
|
900 |
-
"target_polycount": 30000,
|
901 |
-
"simplify_ratio": 0.5,
|
902 |
-
"output_dir": "./hunyuan3d_output",
|
903 |
-
"cache_dir": "./hunyuan3d_cache",
|
904 |
-
"temp_dir": "./hunyuan3d_temp",
|
905 |
-
"keep_intermediates": False,
|
906 |
-
"enable_multi_view": True,
|
907 |
-
"views": ["front", "back", "left", "right"]
|
908 |
-
}
|
909 |
-
|
910 |
-
with open(output_path, 'w') as f:
|
911 |
-
json.dump(config, f, indent=2)
|
912 |
-
|
913 |
-
logger.info(f"Config template created at: {output_path}")
|
914 |
-
|
915 |
-
# Example usage
|
916 |
-
if __name__ == "__main__":
|
917 |
-
import asyncio
|
918 |
-
|
919 |
-
async def example():
|
920 |
-
# Initialize pipeline
|
921 |
-
config = Hunyuan3DConfig()
|
922 |
-
pipeline = Hunyuan3DPipeline(config)
|
923 |
-
|
924 |
-
# Example 1: Generate from text
|
925 |
-
result = await pipeline.generate_from_text(
|
926 |
-
prompt="cute blue dragon with big eyes, small wings, friendly expression",
|
927 |
-
name="BlueDragon",
|
928 |
-
mode=GenerationMode.FAST
|
929 |
-
)
|
930 |
-
|
931 |
-
if result["success"]:
|
932 |
-
print(f"Generated model: {result['paths']['processed_model']}")
|
933 |
-
print(f"Generation time: {result['stats']['generation_time']:.2f}s")
|
934 |
-
else:
|
935 |
-
print(f"Generation failed: {result['error']}")
|
936 |
-
|
937 |
-
# Example 2: Generate from image
|
938 |
-
# result = await pipeline.generate_from_image(
|
939 |
-
# image_path="dragon_concept.png",
|
940 |
-
# name="DragonFromImage",
|
941 |
-
# mode=GenerationMode.STANDARD
|
942 |
-
# )
|
943 |
-
|
944 |
-
# Example 3: Multi-view generation
|
945 |
-
# views = {
|
946 |
-
# "front": "dragon_front.png",
|
947 |
-
# "back": "dragon_back.png",
|
948 |
-
# "left": "dragon_left.png",
|
949 |
-
# "right": "dragon_right.png"
|
950 |
-
# }
|
951 |
-
# result = await pipeline.generate_from_multi_view(
|
952 |
-
# image_paths=views,
|
953 |
-
# name="DragonMultiView",
|
954 |
-
# mode=GenerationMode.STANDARD
|
955 |
-
# )
|
956 |
-
|
957 |
-
# Run example
|
958 |
-
# asyncio.run(example())
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/pipelines/opensource_3d_pipeline_v2.py
CHANGED
@@ -1,6 +1,6 @@
|
|
1 |
"""
|
2 |
Production-Ready Open-Source Text-to-Rigged-3D Pipeline
|
3 |
-
Uses HuggingFace Spaces API for Flux,
|
4 |
Rick Rubin philosophy: Strip complexity, amplify creativity
|
5 |
"""
|
6 |
|
@@ -33,14 +33,14 @@ logger = logging.getLogger(__name__)
|
|
33 |
class ProductionConfig:
|
34 |
"""Production configuration for open-source pipeline"""
|
35 |
|
36 |
-
# Text-to-image model options
|
37 |
-
text_to_image_model: str = "omnigen2" # omnigen2
|
38 |
-
omnigen2_repo: str = "
|
39 |
flux_space: str = "black-forest-labs/FLUX.1-dev" # Kept for fallback
|
40 |
-
sparc3d_space: Optional[str] = "lizhihao6/Sparc3D" # If available as Space
|
41 |
|
42 |
-
#
|
43 |
-
|
|
|
44 |
unirig_repo: str = "https://github.com/VAST-AI-Research/UniRig"
|
45 |
unirig_hf_model: str = "VAST-AI/UniRig"
|
46 |
|
@@ -51,7 +51,7 @@ class ProductionConfig:
|
|
51 |
inference_steps: int = 28
|
52 |
|
53 |
# 3D settings
|
54 |
-
|
55 |
target_polycount: int = 30000
|
56 |
texture_resolution: int = 2048
|
57 |
|
@@ -71,35 +71,56 @@ class ProductionConfig:
|
|
71 |
enable_cpu_offload: bool = True # For VRAM optimization
|
72 |
|
73 |
class OmniGen2MultiViewGenerator:
|
74 |
-
"""Generate multi-view images using
|
75 |
|
76 |
def __init__(self, config: ProductionConfig):
|
77 |
self.config = config
|
78 |
self.model = None
|
79 |
self.tokenizer = None
|
80 |
-
logger.info(f"Initializing
|
81 |
|
82 |
def _load_model(self):
|
83 |
-
"""Lazy load the model to save memory"""
|
84 |
if self.model is None:
|
85 |
try:
|
86 |
-
#
|
87 |
-
|
88 |
-
|
89 |
-
|
90 |
-
|
91 |
-
self.model = AutoModel.from_pretrained(
|
92 |
self.config.omnigen2_repo,
|
93 |
trust_remote_code=True,
|
94 |
torch_dtype=torch.float16 if self.config.device == "cuda" else torch.float32,
|
95 |
device_map="auto" if self.config.enable_cpu_offload else None
|
96 |
)
|
|
|
97 |
if not self.config.enable_cpu_offload:
|
98 |
self.model = self.model.to(self.config.device)
|
99 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
100 |
except Exception as e:
|
101 |
-
logger.error(f"Failed to load
|
102 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
103 |
|
104 |
async def generate_creature_views(self, base_prompt: str,
|
105 |
creature_name: str) -> Dict[str, Path]:
|
@@ -153,34 +174,59 @@ class OmniGen2MultiViewGenerator:
|
|
153 |
f"consistent lighting, monster character design"
|
154 |
)
|
155 |
|
156 |
-
# Generate image using
|
157 |
-
inputs = self.tokenizer(full_prompt, return_tensors="pt", truncation=True)
|
158 |
-
|
159 |
with torch.no_grad():
|
160 |
-
|
161 |
-
|
162 |
-
|
163 |
-
|
164 |
-
|
165 |
-
|
166 |
-
|
167 |
-
|
168 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
169 |
|
170 |
# Save generated image
|
171 |
output_path = self.config.output_dir / f"{creature_name}_{view_name}_view.png"
|
172 |
output_path.parent.mkdir(parents=True, exist_ok=True)
|
173 |
|
174 |
-
#
|
175 |
-
if isinstance(
|
176 |
-
|
177 |
-
|
178 |
-
|
179 |
-
|
180 |
-
|
181 |
-
image = Image.fromarray(image_array)
|
182 |
-
else:
|
183 |
-
image = outputs # Assume it's already a PIL Image
|
184 |
|
185 |
image.save(output_path)
|
186 |
output_paths[view_name] = output_path
|
@@ -400,316 +446,312 @@ class FluxMultiViewGenerator:
|
|
400 |
|
401 |
return output_path
|
402 |
|
403 |
-
class
|
404 |
-
"""
|
405 |
|
406 |
def __init__(self, config: ProductionConfig):
|
407 |
self.config = config
|
408 |
-
self.
|
409 |
-
|
410 |
-
|
411 |
-
def _setup_sparc3d(self):
|
412 |
-
"""Setup Sparc3D from GitHub repository"""
|
413 |
-
|
414 |
-
sparc3d_path = Path("./Sparc3D")
|
415 |
-
|
416 |
-
if not sparc3d_path.exists():
|
417 |
-
logger.info("Cloning Sparc3D repository...")
|
418 |
-
subprocess.run([
|
419 |
-
"git", "clone", self.config.sparc3d_repo, str(sparc3d_path)
|
420 |
-
], check=True)
|
421 |
-
|
422 |
-
# Install requirements
|
423 |
-
requirements_file = sparc3d_path / "requirements.txt"
|
424 |
-
if requirements_file.exists():
|
425 |
-
subprocess.run([
|
426 |
-
"pip", "install", "-r", str(requirements_file)
|
427 |
-
], check=True)
|
428 |
-
|
429 |
-
# Add to Python path
|
430 |
-
import sys
|
431 |
-
sys.path.insert(0, str(sparc3d_path))
|
432 |
|
433 |
-
|
434 |
-
|
435 |
-
|
436 |
-
|
437 |
-
|
438 |
-
|
439 |
-
|
440 |
-
|
441 |
-
|
442 |
-
|
443 |
-
|
444 |
-
|
445 |
-
|
446 |
-
|
447 |
-
|
|
|
|
|
|
|
448 |
|
449 |
async def generate_3d_from_views(self, view_paths: Dict[str, Path],
|
450 |
creature_name: str) -> Dict[str, Any]:
|
451 |
-
"""Generate 3D model from
|
452 |
-
|
453 |
-
logger.info(f"Generating 3D model from {len(view_paths)} views")
|
454 |
|
455 |
-
|
456 |
-
processed_views = self._preprocess_views(view_paths)
|
457 |
|
458 |
-
|
459 |
start_time = time.time()
|
460 |
|
461 |
try:
|
462 |
-
#
|
463 |
-
|
|
|
|
|
|
|
|
|
464 |
|
465 |
-
|
466 |
-
|
467 |
|
468 |
-
|
469 |
-
|
470 |
|
471 |
-
#
|
472 |
-
|
473 |
-
|
474 |
-
|
475 |
-
|
476 |
-
|
477 |
-
|
478 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
479 |
|
480 |
-
|
481 |
-
|
482 |
-
|
483 |
-
"texture_path": texture_data["path"],
|
484 |
-
"statistics": {
|
485 |
-
"vertices": len(mesh.vertices),
|
486 |
-
"faces": len(mesh.faces),
|
487 |
-
"generation_time": generation_time,
|
488 |
-
"resolution": self.config.sparc3d_resolution,
|
489 |
-
"texture_size": texture_data["size"]
|
490 |
-
}
|
491 |
-
}
|
492 |
|
493 |
except Exception as e:
|
494 |
-
logger.error(f"
|
495 |
return {
|
496 |
"success": False,
|
497 |
"error": str(e)
|
498 |
}
|
499 |
|
500 |
-
def
|
501 |
-
|
502 |
-
|
503 |
-
processed = {}
|
504 |
-
|
505 |
-
for view_name, path in view_paths.items():
|
506 |
-
if view_name == "concept_sheet":
|
507 |
-
continue # Skip concept sheet
|
508 |
-
|
509 |
-
# Load image
|
510 |
-
img = Image.open(path)
|
511 |
-
|
512 |
-
# Ensure square aspect ratio
|
513 |
-
if img.width != img.height:
|
514 |
-
size = min(img.width, img.height)
|
515 |
-
img = img.crop((
|
516 |
-
(img.width - size) // 2,
|
517 |
-
(img.height - size) // 2,
|
518 |
-
(img.width + size) // 2,
|
519 |
-
(img.height + size) // 2
|
520 |
-
))
|
521 |
-
|
522 |
-
# Resize to expected resolution
|
523 |
-
img = img.resize((512, 512), Image.Resampling.LANCZOS)
|
524 |
-
|
525 |
-
# Convert to array and normalize
|
526 |
-
img_array = np.array(img).astype(np.float32) / 255.0
|
527 |
-
|
528 |
-
processed[view_name] = img_array
|
529 |
-
|
530 |
-
return processed
|
531 |
-
|
532 |
-
async def _run_sparc3d_pipeline(self, views: Dict[str, np.ndarray]) -> Dict[str, Any]:
|
533 |
-
"""Run Sparc3D pipeline on preprocessed views"""
|
534 |
-
|
535 |
-
if hasattr(self.model, 'generate_3d'):
|
536 |
-
# Use actual Sparc3D implementation
|
537 |
-
return await self.model.generate_3d(views)
|
538 |
-
else:
|
539 |
-
# Use simplified implementation
|
540 |
-
return self.model.generate_mesh(views)
|
541 |
-
|
542 |
-
def _create_trimesh(self, mesh_data: Dict[str, Any]) -> trimesh.Trimesh:
|
543 |
-
"""Convert Sparc3D output to trimesh object"""
|
544 |
-
|
545 |
-
vertices = np.array(mesh_data["vertices"])
|
546 |
-
faces = np.array(mesh_data["faces"])
|
547 |
-
|
548 |
-
# Create mesh
|
549 |
-
mesh = trimesh.Trimesh(vertices=vertices, faces=faces)
|
550 |
-
|
551 |
-
# Center and normalize scale
|
552 |
-
mesh.vertices -= mesh.centroid
|
553 |
-
max_extent = np.max(mesh.extents)
|
554 |
-
mesh.vertices *= 2.0 / max_extent
|
555 |
-
|
556 |
-
return mesh
|
557 |
-
|
558 |
-
def _optimize_mesh(self, mesh: trimesh.Trimesh) -> trimesh.Trimesh:
|
559 |
-
"""Optimize mesh for game use"""
|
560 |
|
561 |
-
logger.info(
|
562 |
|
563 |
-
#
|
564 |
-
|
565 |
-
mesh.remove_unreferenced_vertices()
|
566 |
-
mesh.fill_holes()
|
567 |
|
568 |
-
#
|
569 |
-
|
570 |
-
target = self.config.target_polycount
|
571 |
-
logger.info(f"Simplifying to {target} faces...")
|
572 |
-
mesh = mesh.simplify_quadric_decimation(target)
|
573 |
|
574 |
-
#
|
575 |
-
|
576 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
577 |
|
578 |
-
#
|
579 |
-
|
|
|
|
|
580 |
|
581 |
-
|
582 |
-
|
583 |
-
async def _generate_textures(self, mesh: trimesh.Trimesh,
|
584 |
-
views: Dict[str, np.ndarray],
|
585 |
-
creature_name: str) -> Dict[str, Any]:
|
586 |
-
"""Generate PBR textures from input views"""
|
587 |
-
|
588 |
-
# Generate UV mapping
|
589 |
-
if not hasattr(mesh.visual, 'uv') or mesh.visual.uv is None:
|
590 |
-
logger.info("Generating UV mapping...")
|
591 |
-
# Use xatlas or similar for UV unwrapping
|
592 |
-
uv_coords = self._generate_smart_uvs(mesh)
|
593 |
-
mesh.visual = trimesh.visual.TextureVisuals(uv=uv_coords)
|
594 |
-
|
595 |
-
# Create texture from views
|
596 |
-
texture_size = self.config.texture_resolution
|
597 |
-
|
598 |
-
# Initialize texture maps
|
599 |
-
albedo = np.ones((texture_size, texture_size, 3), dtype=np.uint8) * 200
|
600 |
-
normal = np.ones((texture_size, texture_size, 3), dtype=np.uint8) * 128
|
601 |
-
normal[:, :, 2] = 255 # Default normal pointing up
|
602 |
-
|
603 |
-
# Project views onto texture
|
604 |
-
# In production, use proper texture projection algorithms
|
605 |
-
if "front" in views:
|
606 |
-
front_img = (views["front"] * 255).astype(np.uint8)
|
607 |
-
front_img = Image.fromarray(front_img)
|
608 |
-
front_img = front_img.resize((texture_size, texture_size), Image.Resampling.LANCZOS)
|
609 |
-
albedo = np.array(front_img)
|
610 |
-
|
611 |
-
# Save textures
|
612 |
-
texture_path = self.config.output_dir / f"{creature_name}_albedo.png"
|
613 |
-
Image.fromarray(albedo).save(texture_path)
|
614 |
-
|
615 |
-
normal_path = self.config.output_dir / f"{creature_name}_normal.png"
|
616 |
-
Image.fromarray(normal).save(normal_path)
|
617 |
-
|
618 |
-
# Apply to mesh
|
619 |
-
mesh.visual.material.image = Image.fromarray(albedo)
|
620 |
|
621 |
return {
|
622 |
-
"
|
623 |
-
"
|
624 |
-
"
|
625 |
-
|
626 |
-
"
|
|
|
|
|
|
|
|
|
|
|
627 |
}
|
628 |
}
|
629 |
-
|
630 |
-
def _generate_smart_uvs(self, mesh: trimesh.Trimesh) -> np.ndarray:
|
631 |
-
"""Generate smart UV coordinates"""
|
632 |
-
|
633 |
-
# Try using xatlas if available
|
634 |
-
try:
|
635 |
-
import xatlas
|
636 |
-
vmapping, indices, uvs = xatlas.parametrize(mesh.vertices, mesh.faces)
|
637 |
-
return uvs
|
638 |
-
except ImportError:
|
639 |
-
# Fallback to simple cylindrical mapping
|
640 |
-
vertices = mesh.vertices
|
641 |
-
theta = np.arctan2(vertices[:, 0], vertices[:, 2])
|
642 |
-
height = vertices[:, 1]
|
643 |
-
|
644 |
-
u = (theta + np.pi) / (2 * np.pi)
|
645 |
-
v = (height - height.min()) / (height.max() - height.min())
|
646 |
-
|
647 |
-
return np.column_stack([u, v])
|
648 |
|
649 |
class UniRigProcessor:
|
650 |
-
"""UniRig integration using HuggingFace models"""
|
651 |
|
652 |
def __init__(self, config: ProductionConfig):
|
653 |
self.config = config
|
654 |
self.model_path = None
|
655 |
-
self.
|
656 |
-
|
657 |
-
def _setup_unirig(self):
|
658 |
-
"""Setup UniRig from HuggingFace and GitHub"""
|
659 |
|
660 |
-
|
661 |
-
|
662 |
|
663 |
try:
|
664 |
-
|
665 |
-
|
666 |
-
cache_dir=self.config.cache_dir,
|
667 |
-
token=self.config.hf_token
|
668 |
-
)
|
669 |
-
logger.info(f"UniRig models downloaded to: {self.model_path}")
|
670 |
|
671 |
-
|
672 |
-
|
673 |
-
|
674 |
-
|
675 |
-
|
676 |
-
|
677 |
-
|
678 |
-
|
679 |
-
|
680 |
-
|
|
|
681 |
|
682 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
683 |
|
684 |
async def auto_rig_creature(self, mesh_path: Path, creature_name: str,
|
685 |
creature_type: str = "biped") -> Dict[str, Any]:
|
686 |
-
"""Apply automatic rigging using UniRig"""
|
687 |
|
688 |
-
logger.info(f"Auto-rigging {creature_name} as {creature_type}")
|
|
|
|
|
|
|
|
|
689 |
|
690 |
try:
|
691 |
-
#
|
692 |
-
if
|
693 |
-
|
694 |
-
elif creature_type == "quadruped":
|
695 |
-
skeleton_config = "configs/skeleton_quadruped.json"
|
696 |
-
else:
|
697 |
-
skeleton_config = "configs/skeleton_generic.json"
|
698 |
|
699 |
-
#
|
700 |
-
|
701 |
-
mesh_path,
|
702 |
-
skeleton_config,
|
703 |
-
creature_name
|
704 |
-
)
|
705 |
|
706 |
-
|
|
|
|
|
|
|
707 |
|
708 |
except Exception as e:
|
709 |
logger.error(f"UniRig failed: {e}")
|
710 |
# Fallback to procedural rigging
|
711 |
return await self._procedural_rigging_fallback(mesh_path, creature_name, creature_type)
|
712 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
713 |
async def _run_unirig_pipeline(self, mesh_path: Path,
|
714 |
skeleton_config: str,
|
715 |
creature_name: str) -> Dict[str, Any]:
|
@@ -999,8 +1041,8 @@ class UniRigProcessor:
|
|
999 |
"method": "procedural_fallback"
|
1000 |
}
|
1001 |
|
1002 |
-
class
|
1003 |
-
"""Simplified 3D generation when
|
1004 |
|
1005 |
def __init__(self, config: ProductionConfig):
|
1006 |
self.config = config
|
@@ -1076,7 +1118,7 @@ class SimplifiedSparc3D:
|
|
1076 |
class ProductionPipeline:
|
1077 |
"""
|
1078 |
Complete production-ready open-source pipeline
|
1079 |
-
Flux ->
|
1080 |
"""
|
1081 |
|
1082 |
def __init__(self, config: Optional[ProductionConfig] = None):
|
@@ -1101,7 +1143,7 @@ class ProductionPipeline:
|
|
1101 |
else:
|
1102 |
self.image_generator = FluxMultiViewGenerator(self.config)
|
1103 |
|
1104 |
-
self.
|
1105 |
self.unirig = UniRigProcessor(self.config)
|
1106 |
|
1107 |
logger.info("Production pipeline ready!")
|
@@ -1141,14 +1183,14 @@ class ProductionPipeline:
|
|
1141 |
"outputs": {k: str(v) for k, v in views.items()}
|
1142 |
}
|
1143 |
|
1144 |
-
# Stage 2: 3D generation with
|
1145 |
logger.info("=" * 50)
|
1146 |
-
logger.info("Stage 2: Generating 3D model with
|
1147 |
stage_start = time.time()
|
1148 |
|
1149 |
-
model_data = await self.
|
1150 |
|
1151 |
-
results["pipeline_stages"]["
|
1152 |
"success": model_data["success"],
|
1153 |
"duration": time.time() - stage_start,
|
1154 |
"outputs": model_data
|
@@ -1182,13 +1224,14 @@ class ProductionPipeline:
|
|
1182 |
results["final_outputs"] = {
|
1183 |
"concept_sheet": str(views.get("concept_sheet", "")),
|
1184 |
"mesh": str(model_data["mesh_path"]),
|
1185 |
-
"texture": str(model_data
|
1186 |
"rigged_model": str(rig_data["rigged_path"]),
|
1187 |
"statistics": {
|
1188 |
-
"vertices": model_data
|
1189 |
-
"faces": model_data
|
1190 |
-
"bones": rig_data
|
1191 |
-
"
|
|
|
1192 |
}
|
1193 |
}
|
1194 |
|
@@ -1224,7 +1267,7 @@ def create_production_config():
|
|
1224 |
"num_views": 6,
|
1225 |
"guidance_scale": 3.5,
|
1226 |
"inference_steps": 28,
|
1227 |
-
"
|
1228 |
"target_polycount": 30000,
|
1229 |
"texture_resolution": 2048,
|
1230 |
"output_dir": "./digipal_3d_output",
|
|
|
1 |
"""
|
2 |
Production-Ready Open-Source Text-to-Rigged-3D Pipeline
|
3 |
+
Uses HuggingFace Spaces API for Flux, Hunyuan3D implementation, and UniRig models
|
4 |
Rick Rubin philosophy: Strip complexity, amplify creativity
|
5 |
"""
|
6 |
|
|
|
33 |
class ProductionConfig:
|
34 |
"""Production configuration for open-source pipeline"""
|
35 |
|
36 |
+
# Text-to-image model options
|
37 |
+
text_to_image_model: str = "omnigen2" # omnigen2 is primary, flux as fallback
|
38 |
+
omnigen2_repo: str = "shitao/OmniGen-v1" # Updated to working repo
|
39 |
flux_space: str = "black-forest-labs/FLUX.1-dev" # Kept for fallback
|
|
|
40 |
|
41 |
+
# 3D Generation models
|
42 |
+
hunyuan3d_model: str = "tencent/Hunyuan3D-2.1" # Updated to latest version
|
43 |
+
hunyuan3d_space: str = "tencent/Hunyuan3D-2.1" # Official Gradio Space
|
44 |
unirig_repo: str = "https://github.com/VAST-AI-Research/UniRig"
|
45 |
unirig_hf_model: str = "VAST-AI/UniRig"
|
46 |
|
|
|
51 |
inference_steps: int = 28
|
52 |
|
53 |
# 3D settings
|
54 |
+
hunyuan3d_resolution: int = 1024
|
55 |
target_polycount: int = 30000
|
56 |
texture_resolution: int = 2048
|
57 |
|
|
|
71 |
enable_cpu_offload: bool = True # For VRAM optimization
|
72 |
|
73 |
class OmniGen2MultiViewGenerator:
|
74 |
+
"""Generate multi-view images using OmniGen"""
|
75 |
|
76 |
def __init__(self, config: ProductionConfig):
|
77 |
self.config = config
|
78 |
self.model = None
|
79 |
self.tokenizer = None
|
80 |
+
logger.info(f"Initializing OmniGen from: {config.omnigen2_repo}")
|
81 |
|
82 |
def _load_model(self):
|
83 |
+
"""Lazy load the OmniGen model to save memory"""
|
84 |
if self.model is None:
|
85 |
try:
|
86 |
+
# Import OmniGen specific modules
|
87 |
+
from diffusers import DiffusionPipeline
|
88 |
+
|
89 |
+
# Load OmniGen model using diffusers pipeline
|
90 |
+
self.model = DiffusionPipeline.from_pretrained(
|
|
|
91 |
self.config.omnigen2_repo,
|
92 |
trust_remote_code=True,
|
93 |
torch_dtype=torch.float16 if self.config.device == "cuda" else torch.float32,
|
94 |
device_map="auto" if self.config.enable_cpu_offload else None
|
95 |
)
|
96 |
+
|
97 |
if not self.config.enable_cpu_offload:
|
98 |
self.model = self.model.to(self.config.device)
|
99 |
+
|
100 |
+
# Enable memory efficient attention if available
|
101 |
+
if hasattr(self.model, 'enable_attention_slicing'):
|
102 |
+
self.model.enable_attention_slicing()
|
103 |
+
if hasattr(self.model, 'enable_vae_slicing'):
|
104 |
+
self.model.enable_vae_slicing()
|
105 |
+
|
106 |
+
logger.info("OmniGen model loaded successfully")
|
107 |
except Exception as e:
|
108 |
+
logger.error(f"Failed to load OmniGen model: {str(e)}")
|
109 |
+
logger.info("Trying alternative loading method...")
|
110 |
+
try:
|
111 |
+
# Fallback: try loading as a generic model
|
112 |
+
self.model = AutoModel.from_pretrained(
|
113 |
+
self.config.omnigen2_repo,
|
114 |
+
trust_remote_code=True,
|
115 |
+
torch_dtype=torch.float16 if self.config.device == "cuda" else torch.float32,
|
116 |
+
device_map="auto" if self.config.enable_cpu_offload else None
|
117 |
+
)
|
118 |
+
if not self.config.enable_cpu_offload:
|
119 |
+
self.model = self.model.to(self.config.device)
|
120 |
+
logger.info("OmniGen model loaded via fallback method")
|
121 |
+
except Exception as fallback_e:
|
122 |
+
logger.error(f"Fallback loading also failed: {fallback_e}")
|
123 |
+
raise
|
124 |
|
125 |
async def generate_creature_views(self, base_prompt: str,
|
126 |
creature_name: str) -> Dict[str, Path]:
|
|
|
174 |
f"consistent lighting, monster character design"
|
175 |
)
|
176 |
|
177 |
+
# Generate image using OmniGen
|
|
|
|
|
178 |
with torch.no_grad():
|
179 |
+
if hasattr(self.model, '__call__'):
|
180 |
+
# Standard diffusers pipeline call
|
181 |
+
result = self.model(
|
182 |
+
prompt=full_prompt,
|
183 |
+
width=self.config.image_resolution,
|
184 |
+
height=self.config.image_resolution,
|
185 |
+
guidance_scale=self.config.text_guidance_scale,
|
186 |
+
num_inference_steps=self.config.inference_steps,
|
187 |
+
generator=torch.Generator(device=self.config.device).manual_seed(42 + len(output_paths))
|
188 |
+
)
|
189 |
+
|
190 |
+
# Extract the image from the result
|
191 |
+
if hasattr(result, 'images'):
|
192 |
+
image = result.images[0]
|
193 |
+
elif isinstance(result, list):
|
194 |
+
image = result[0]
|
195 |
+
else:
|
196 |
+
image = result
|
197 |
+
|
198 |
+
elif hasattr(self.model, 'generate'):
|
199 |
+
# Alternative generation method for different model types
|
200 |
+
result = self.model.generate(
|
201 |
+
prompt=full_prompt,
|
202 |
+
image_size=(self.config.image_resolution, self.config.image_resolution),
|
203 |
+
guidance_scale=self.config.text_guidance_scale,
|
204 |
+
num_inference_steps=self.config.inference_steps
|
205 |
+
)
|
206 |
+
|
207 |
+
if isinstance(result, torch.Tensor):
|
208 |
+
# Convert tensor to PIL Image
|
209 |
+
image_array = result.cpu().numpy().squeeze()
|
210 |
+
if image_array.ndim == 3 and image_array.shape[0] == 3:
|
211 |
+
image_array = np.transpose(image_array, (1, 2, 0))
|
212 |
+
image_array = (image_array * 255).astype(np.uint8)
|
213 |
+
image = Image.fromarray(image_array)
|
214 |
+
else:
|
215 |
+
image = result
|
216 |
+
else:
|
217 |
+
raise ValueError("Unknown model interface - cannot generate image")
|
218 |
|
219 |
# Save generated image
|
220 |
output_path = self.config.output_dir / f"{creature_name}_{view_name}_view.png"
|
221 |
output_path.parent.mkdir(parents=True, exist_ok=True)
|
222 |
|
223 |
+
# Ensure image is a PIL Image and save
|
224 |
+
if not isinstance(image, Image.Image):
|
225 |
+
if isinstance(image, np.ndarray):
|
226 |
+
image = Image.fromarray((image * 255).astype(np.uint8))
|
227 |
+
else:
|
228 |
+
logger.warning(f"Unexpected image type: {type(image)}")
|
229 |
+
continue
|
|
|
|
|
|
|
230 |
|
231 |
image.save(output_path)
|
232 |
output_paths[view_name] = output_path
|
|
|
446 |
|
447 |
return output_path
|
448 |
|
449 |
+
class Hunyuan3DProcessor:
|
450 |
+
"""Hunyuan3D-2.1 implementation using official Gradio Space API"""
|
451 |
|
452 |
def __init__(self, config: ProductionConfig):
|
453 |
self.config = config
|
454 |
+
self.client = None
|
455 |
+
logger.info(f"Initializing Hunyuan3D-2.1 from space: {config.hunyuan3d_space}")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
456 |
|
457 |
+
def _initialize_client(self):
|
458 |
+
"""Initialize Gradio client for Hunyuan3D Space"""
|
459 |
+
if self.client is None:
|
460 |
+
try:
|
461 |
+
from gradio_client import Client
|
462 |
+
|
463 |
+
# Connect to the official Hunyuan3D Space
|
464 |
+
self.client = Client(
|
465 |
+
src=self.config.hunyuan3d_space,
|
466 |
+
hf_token=self.config.hf_token
|
467 |
+
)
|
468 |
+
logger.info(f"Connected to Hunyuan3D-2.1 Space: {self.config.hunyuan3d_space}")
|
469 |
+
|
470 |
+
except Exception as e:
|
471 |
+
logger.error(f"Failed to connect to Hunyuan3D Space: {e}")
|
472 |
+
logger.info("Will try local fallback if available")
|
473 |
+
# Don't raise here, let the generation method handle fallback
|
474 |
+
self.client = None
|
475 |
|
476 |
async def generate_3d_from_views(self, view_paths: Dict[str, Path],
|
477 |
creature_name: str) -> Dict[str, Any]:
|
478 |
+
"""Generate 3D model from image using Hunyuan3D-2.1 Space API"""
|
|
|
|
|
479 |
|
480 |
+
self._initialize_client()
|
|
|
481 |
|
482 |
+
logger.info(f"Generating 3D model from {len(view_paths)} views using Hunyuan3D-2.1")
|
483 |
start_time = time.time()
|
484 |
|
485 |
try:
|
486 |
+
# Use the front view as primary input for Hunyuan3D
|
487 |
+
primary_view = None
|
488 |
+
if "front" in view_paths:
|
489 |
+
primary_view = view_paths["front"]
|
490 |
+
elif view_paths:
|
491 |
+
primary_view = next(iter(view_paths.values()))
|
492 |
|
493 |
+
if not primary_view:
|
494 |
+
raise ValueError("No input images provided")
|
495 |
|
496 |
+
if not primary_view.exists():
|
497 |
+
raise ValueError(f"Input image not found: {primary_view}")
|
498 |
|
499 |
+
# Try using the official Hunyuan3D Space API
|
500 |
+
if self.client:
|
501 |
+
try:
|
502 |
+
logger.info("Using official Hunyuan3D-2.1 Space API...")
|
503 |
+
|
504 |
+
# Call the Hunyuan3D Space API
|
505 |
+
# Based on the official interface, it typically takes an image input
|
506 |
+
result = self.client.predict(
|
507 |
+
image=str(primary_view), # Input image path
|
508 |
+
api_name="/generate_3d" # API endpoint name (may vary)
|
509 |
+
)
|
510 |
+
|
511 |
+
# Handle the result - typically returns file paths or URLs
|
512 |
+
if isinstance(result, (list, tuple)) and len(result) > 0:
|
513 |
+
# Extract the 3D model file
|
514 |
+
model_result = result[0] if isinstance(result[0], str) else result
|
515 |
+
|
516 |
+
# Download or copy the result to our output directory
|
517 |
+
mesh_path = self.config.output_dir / f"{creature_name}_hunyuan3d.glb"
|
518 |
+
mesh_path.parent.mkdir(parents=True, exist_ok=True)
|
519 |
+
|
520 |
+
if isinstance(model_result, str) and os.path.exists(model_result):
|
521 |
+
# Copy from local path
|
522 |
+
shutil.copy(model_result, mesh_path)
|
523 |
+
elif isinstance(model_result, str) and model_result.startswith('http'):
|
524 |
+
# Download from URL
|
525 |
+
response = requests.get(model_result)
|
526 |
+
with open(mesh_path, 'wb') as f:
|
527 |
+
f.write(response.content)
|
528 |
+
else:
|
529 |
+
raise ValueError(f"Unexpected result format: {type(model_result)}")
|
530 |
+
|
531 |
+
generation_time = time.time() - start_time
|
532 |
+
|
533 |
+
# Get basic file statistics
|
534 |
+
file_size = mesh_path.stat().st_size if mesh_path.exists() else 0
|
535 |
+
|
536 |
+
return {
|
537 |
+
"success": True,
|
538 |
+
"mesh_path": mesh_path,
|
539 |
+
"texture_path": mesh_path, # Same file for GLB
|
540 |
+
"statistics": {
|
541 |
+
"file_size_mb": file_size / (1024 * 1024),
|
542 |
+
"generation_time": generation_time,
|
543 |
+
"model": "Hunyuan3D-2.1",
|
544 |
+
"input_views": len(view_paths),
|
545 |
+
"method": "official_space_api"
|
546 |
+
}
|
547 |
+
}
|
548 |
+
|
549 |
+
else:
|
550 |
+
raise ValueError("Invalid result from Hunyuan3D Space API")
|
551 |
+
|
552 |
+
except Exception as api_error:
|
553 |
+
logger.error(f"Hunyuan3D Space API failed: {api_error}")
|
554 |
+
logger.info("Falling back to alternative method...")
|
555 |
+
# Fall through to local fallback
|
556 |
|
557 |
+
# Fallback: Use local processing or placeholder
|
558 |
+
logger.info("Using local fallback for 3D generation...")
|
559 |
+
return await self._local_3d_fallback(primary_view, creature_name, start_time)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
560 |
|
561 |
except Exception as e:
|
562 |
+
logger.error(f"Hunyuan3D generation failed: {e}")
|
563 |
return {
|
564 |
"success": False,
|
565 |
"error": str(e)
|
566 |
}
|
567 |
|
568 |
+
async def _local_3d_fallback(self, image_path: Path, creature_name: str,
|
569 |
+
start_time: float) -> Dict[str, Any]:
|
570 |
+
"""Fallback method for 3D generation when Space API is unavailable"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
571 |
|
572 |
+
logger.info("Generating placeholder 3D model...")
|
573 |
|
574 |
+
# Create a simple cube mesh as placeholder
|
575 |
+
import trimesh
|
|
|
|
|
576 |
|
577 |
+
# Generate a basic cube mesh
|
578 |
+
mesh = trimesh.creation.box(extents=[1.0, 1.0, 1.0])
|
|
|
|
|
|
|
579 |
|
580 |
+
# Apply basic coloring based on input image
|
581 |
+
try:
|
582 |
+
input_image = Image.open(image_path).convert("RGB")
|
583 |
+
# Get dominant color from image
|
584 |
+
avg_color = np.array(input_image).mean(axis=(0, 1)) / 255.0
|
585 |
+
|
586 |
+
# Apply color to mesh
|
587 |
+
if hasattr(mesh.visual, 'vertex_colors'):
|
588 |
+
mesh.visual.vertex_colors = np.tile(
|
589 |
+
[*avg_color, 1.0], (len(mesh.vertices), 1)
|
590 |
+
).astype(np.uint8) * 255
|
591 |
+
|
592 |
+
except Exception as color_error:
|
593 |
+
logger.warning(f"Failed to apply coloring: {color_error}")
|
594 |
|
595 |
+
# Save the mesh
|
596 |
+
mesh_path = self.config.output_dir / f"{creature_name}_fallback_3d.glb"
|
597 |
+
mesh_path.parent.mkdir(parents=True, exist_ok=True)
|
598 |
+
mesh.export(str(mesh_path))
|
599 |
|
600 |
+
generation_time = time.time() - start_time
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
601 |
|
602 |
return {
|
603 |
+
"success": True,
|
604 |
+
"mesh_path": mesh_path,
|
605 |
+
"texture_path": mesh_path,
|
606 |
+
"statistics": {
|
607 |
+
"vertices": len(mesh.vertices),
|
608 |
+
"faces": len(mesh.faces),
|
609 |
+
"generation_time": generation_time,
|
610 |
+
"model": "fallback_cube",
|
611 |
+
"input_views": 1,
|
612 |
+
"method": "local_fallback"
|
613 |
}
|
614 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
615 |
|
616 |
class UniRigProcessor:
|
617 |
+
"""UniRig integration using HuggingFace models and inference API"""
|
618 |
|
619 |
def __init__(self, config: ProductionConfig):
|
620 |
self.config = config
|
621 |
self.model_path = None
|
622 |
+
self.client = None
|
623 |
+
logger.info(f"Initializing UniRig from HuggingFace: {config.unirig_hf_model}")
|
|
|
|
|
624 |
|
625 |
+
def _setup_unirig(self):
|
626 |
+
"""Setup UniRig using HuggingFace models and API"""
|
627 |
|
628 |
try:
|
629 |
+
# Try to use HuggingFace Inference API first
|
630 |
+
from gradio_client import Client
|
|
|
|
|
|
|
|
|
631 |
|
632 |
+
# Check if there's a UniRig Space available
|
633 |
+
try:
|
634 |
+
# This would be the ideal approach if there's a UniRig Space
|
635 |
+
self.client = Client(
|
636 |
+
src=self.config.unirig_hf_model, # or a specific space
|
637 |
+
hf_token=self.config.hf_token
|
638 |
+
)
|
639 |
+
logger.info("Connected to UniRig via HuggingFace Space/API")
|
640 |
+
return
|
641 |
+
except:
|
642 |
+
logger.info("No UniRig Space found, trying direct model download...")
|
643 |
|
644 |
+
# Fallback: Download models from HuggingFace
|
645 |
+
try:
|
646 |
+
self.model_path = snapshot_download(
|
647 |
+
repo_id=self.config.unirig_hf_model,
|
648 |
+
cache_dir=self.config.cache_dir,
|
649 |
+
token=self.config.hf_token,
|
650 |
+
allow_patterns=["*.py", "*.yaml", "*.json", "*.bin", "*.safetensors"]
|
651 |
+
)
|
652 |
+
logger.info(f"UniRig models downloaded to: {self.model_path}")
|
653 |
+
|
654 |
+
except Exception as download_error:
|
655 |
+
logger.warning(f"Could not download UniRig from HF: {download_error}")
|
656 |
+
logger.info("UniRig will use procedural fallback method")
|
657 |
+
self.model_path = None
|
658 |
+
|
659 |
+
except Exception as e:
|
660 |
+
logger.error(f"Failed to setup UniRig: {e}")
|
661 |
+
logger.info("UniRig will use procedural fallback method")
|
662 |
+
self.model_path = None
|
663 |
|
664 |
async def auto_rig_creature(self, mesh_path: Path, creature_name: str,
|
665 |
creature_type: str = "biped") -> Dict[str, Any]:
|
666 |
+
"""Apply automatic rigging using UniRig via HuggingFace"""
|
667 |
|
668 |
+
logger.info(f"Auto-rigging {creature_name} as {creature_type} using UniRig")
|
669 |
+
|
670 |
+
# Setup UniRig if not already done
|
671 |
+
if self.model_path is None and self.client is None:
|
672 |
+
self._setup_unirig()
|
673 |
|
674 |
try:
|
675 |
+
# Try using HuggingFace Space/API first
|
676 |
+
if self.client:
|
677 |
+
return await self._rig_via_hf_api(mesh_path, creature_name, creature_type)
|
|
|
|
|
|
|
|
|
678 |
|
679 |
+
# Try using downloaded models
|
680 |
+
elif self.model_path:
|
681 |
+
return await self._rig_via_local_models(mesh_path, creature_name, creature_type)
|
|
|
|
|
|
|
682 |
|
683 |
+
# Fallback to procedural rigging
|
684 |
+
else:
|
685 |
+
logger.info("No UniRig models available, using procedural fallback")
|
686 |
+
return await self._procedural_rigging_fallback(mesh_path, creature_name, creature_type)
|
687 |
|
688 |
except Exception as e:
|
689 |
logger.error(f"UniRig failed: {e}")
|
690 |
# Fallback to procedural rigging
|
691 |
return await self._procedural_rigging_fallback(mesh_path, creature_name, creature_type)
|
692 |
|
693 |
+
async def _rig_via_hf_api(self, mesh_path: Path, creature_name: str,
|
694 |
+
creature_type: str) -> Dict[str, Any]:
|
695 |
+
"""Rig using HuggingFace Space API"""
|
696 |
+
|
697 |
+
logger.info("Using UniRig HuggingFace Space API...")
|
698 |
+
|
699 |
+
try:
|
700 |
+
# Call the UniRig Space API
|
701 |
+
result = self.client.predict(
|
702 |
+
mesh_file=str(mesh_path),
|
703 |
+
creature_type=creature_type,
|
704 |
+
api_name="/auto_rig" # This would be the API endpoint
|
705 |
+
)
|
706 |
+
|
707 |
+
# Handle the result
|
708 |
+
if isinstance(result, (list, tuple)) and len(result) > 0:
|
709 |
+
rigged_file = result[0]
|
710 |
+
|
711 |
+
# Copy result to our output directory
|
712 |
+
output_dir = self.config.output_dir / "rigged"
|
713 |
+
output_dir.mkdir(exist_ok=True)
|
714 |
+
rigged_path = output_dir / f"{creature_name}_rigged.glb"
|
715 |
+
|
716 |
+
if isinstance(rigged_file, str) and os.path.exists(rigged_file):
|
717 |
+
shutil.copy(rigged_file, rigged_path)
|
718 |
+
elif isinstance(rigged_file, str) and rigged_file.startswith('http'):
|
719 |
+
# Download from URL
|
720 |
+
response = requests.get(rigged_file)
|
721 |
+
with open(rigged_path, 'wb') as f:
|
722 |
+
f.write(response.content)
|
723 |
+
else:
|
724 |
+
raise ValueError(f"Unexpected result format: {type(rigged_file)}")
|
725 |
+
|
726 |
+
return {
|
727 |
+
"success": True,
|
728 |
+
"rigged_path": rigged_path,
|
729 |
+
"bone_count": "unknown", # API doesn't provide this
|
730 |
+
"method": "hf_space_api"
|
731 |
+
}
|
732 |
+
else:
|
733 |
+
raise ValueError("Invalid result from UniRig Space API")
|
734 |
+
|
735 |
+
except Exception as api_error:
|
736 |
+
logger.error(f"UniRig Space API failed: {api_error}")
|
737 |
+
raise
|
738 |
+
|
739 |
+
async def _rig_via_local_models(self, mesh_path: Path, creature_name: str,
|
740 |
+
creature_type: str) -> Dict[str, Any]:
|
741 |
+
"""Rig using locally downloaded UniRig models"""
|
742 |
+
|
743 |
+
logger.info("Using local UniRig models...")
|
744 |
+
|
745 |
+
try:
|
746 |
+
# This would require implementing the UniRig inference pipeline
|
747 |
+
# For now, fall back to procedural method
|
748 |
+
logger.info("Local UniRig inference not yet implemented, using procedural fallback")
|
749 |
+
return await self._procedural_rigging_fallback(mesh_path, creature_name, creature_type)
|
750 |
+
|
751 |
+
except Exception as e:
|
752 |
+
logger.error(f"Local UniRig inference failed: {e}")
|
753 |
+
raise
|
754 |
+
|
755 |
async def _run_unirig_pipeline(self, mesh_path: Path,
|
756 |
skeleton_config: str,
|
757 |
creature_name: str) -> Dict[str, Any]:
|
|
|
1041 |
"method": "procedural_fallback"
|
1042 |
}
|
1043 |
|
1044 |
+
class SimplifiedHunyuan3D:
|
1045 |
+
"""Simplified 3D generation when Hunyuan3D is not available"""
|
1046 |
|
1047 |
def __init__(self, config: ProductionConfig):
|
1048 |
self.config = config
|
|
|
1118 |
class ProductionPipeline:
|
1119 |
"""
|
1120 |
Complete production-ready open-source pipeline
|
1121 |
+
OmniGen2/Flux -> Hunyuan3D -> UniRig
|
1122 |
"""
|
1123 |
|
1124 |
def __init__(self, config: Optional[ProductionConfig] = None):
|
|
|
1143 |
else:
|
1144 |
self.image_generator = FluxMultiViewGenerator(self.config)
|
1145 |
|
1146 |
+
self.hunyuan3d = Hunyuan3DProcessor(self.config)
|
1147 |
self.unirig = UniRigProcessor(self.config)
|
1148 |
|
1149 |
logger.info("Production pipeline ready!")
|
|
|
1183 |
"outputs": {k: str(v) for k, v in views.items()}
|
1184 |
}
|
1185 |
|
1186 |
+
# Stage 2: 3D generation with Hunyuan3D
|
1187 |
logger.info("=" * 50)
|
1188 |
+
logger.info("Stage 2: Generating 3D model with Hunyuan3D")
|
1189 |
stage_start = time.time()
|
1190 |
|
1191 |
+
model_data = await self.hunyuan3d.generate_3d_from_views(views, name)
|
1192 |
|
1193 |
+
results["pipeline_stages"]["hunyuan3d"] = {
|
1194 |
"success": model_data["success"],
|
1195 |
"duration": time.time() - stage_start,
|
1196 |
"outputs": model_data
|
|
|
1224 |
results["final_outputs"] = {
|
1225 |
"concept_sheet": str(views.get("concept_sheet", "")),
|
1226 |
"mesh": str(model_data["mesh_path"]),
|
1227 |
+
"texture": str(model_data.get("texture_path", model_data["mesh_path"])),
|
1228 |
"rigged_model": str(rig_data["rigged_path"]),
|
1229 |
"statistics": {
|
1230 |
+
"vertices": model_data.get("statistics", {}).get("vertices", "unknown"),
|
1231 |
+
"faces": model_data.get("statistics", {}).get("faces", "unknown"),
|
1232 |
+
"bones": rig_data.get("bone_count", "unknown"),
|
1233 |
+
"generation_time": model_data.get("statistics", {}).get("generation_time", 0),
|
1234 |
+
"model_method": model_data.get("statistics", {}).get("method", "unknown")
|
1235 |
}
|
1236 |
}
|
1237 |
|
|
|
1267 |
"num_views": 6,
|
1268 |
"guidance_scale": 3.5,
|
1269 |
"inference_steps": 28,
|
1270 |
+
"hunyuan3d_resolution": 512,
|
1271 |
"target_polycount": 30000,
|
1272 |
"texture_resolution": 2048,
|
1273 |
"output_dir": "./digipal_3d_output",
|
src/ui/gradio_interface.py
DELETED
@@ -1,1263 +0,0 @@
|
|
1 |
-
import gradio as gr
|
2 |
-
import asyncio
|
3 |
-
import logging
|
4 |
-
import json
|
5 |
-
import time
|
6 |
-
from typing import Dict, List, Optional, Any, Tuple
|
7 |
-
from datetime import datetime, timedelta
|
8 |
-
import numpy as np
|
9 |
-
|
10 |
-
# Core imports
|
11 |
-
from ..core.monster_engine import Monster, MonsterPersonalityType, EmotionalState
|
12 |
-
from ..ai.qwen_processor import QwenProcessor, ModelConfig
|
13 |
-
from ..ai.speech_engine import AdvancedSpeechEngine, SpeechConfig
|
14 |
-
from ..utils.performance_tracker import PerformanceTracker
|
15 |
-
from .state_manager import AdvancedStateManager
|
16 |
-
from ..deployment.zero_gpu_optimizer import ZeroGPUOptimizer
|
17 |
-
import spaces
|
18 |
-
|
19 |
-
def create_interface():
|
20 |
-
"""Create and return the Gradio interface"""
|
21 |
-
interface = ModernDigiPalInterface()
|
22 |
-
return interface.create_interface()
|
23 |
-
|
24 |
-
class StreamingComponents:
|
25 |
-
"""Helper class for streaming components"""
|
26 |
-
def __init__(self):
|
27 |
-
self.logger = logging.getLogger(__name__)
|
28 |
-
|
29 |
-
class ModernDigiPalInterface:
|
30 |
-
def __init__(self):
|
31 |
-
self.logger = logging.getLogger(__name__)
|
32 |
-
|
33 |
-
# Initialize core systems
|
34 |
-
self.state_manager = AdvancedStateManager()
|
35 |
-
self.streaming = StreamingComponents()
|
36 |
-
self.gpu_optimizer = ZeroGPUOptimizer()
|
37 |
-
|
38 |
-
# AI Systems (will be initialized based on available resources)
|
39 |
-
self.qwen_processor = None
|
40 |
-
self.speech_engine = None
|
41 |
-
|
42 |
-
# Performance tracking
|
43 |
-
self.performance_metrics = {
|
44 |
-
"total_interactions": 0,
|
45 |
-
"average_response_time": 0.0,
|
46 |
-
"user_satisfaction": 0.0
|
47 |
-
}
|
48 |
-
|
49 |
-
# UI State
|
50 |
-
self.current_monster = None
|
51 |
-
self.ui_theme = "soft"
|
52 |
-
|
53 |
-
async def initialize(self):
|
54 |
-
"""Initialize the interface with optimized configurations"""
|
55 |
-
try:
|
56 |
-
# Detect available resources
|
57 |
-
resources = await self.gpu_optimizer.detect_available_resources()
|
58 |
-
|
59 |
-
# Initialize AI processors based on resources
|
60 |
-
await self._initialize_ai_systems(resources)
|
61 |
-
|
62 |
-
# Initialize state management
|
63 |
-
await self.state_manager.initialize()
|
64 |
-
|
65 |
-
self.logger.info("DigiPal interface initialized successfully")
|
66 |
-
|
67 |
-
except Exception as e:
|
68 |
-
self.logger.error(f"Failed to initialize interface: {e}")
|
69 |
-
raise
|
70 |
-
|
71 |
-
async def _initialize_ai_systems(self, resources: Dict[str, Any]):
|
72 |
-
"""Initialize AI systems based on available resources"""
|
73 |
-
|
74 |
-
# Initialize Qwen processor with fallback handling
|
75 |
-
try:
|
76 |
-
# Configure Qwen processor
|
77 |
-
if resources["gpu_memory_gb"] >= 8:
|
78 |
-
model_config = ModelConfig(
|
79 |
-
model_name="Qwen/Qwen2.5-3B-Instruct",
|
80 |
-
max_memory_gb=resources["gpu_memory_gb"],
|
81 |
-
inference_speed="quality"
|
82 |
-
)
|
83 |
-
elif resources["gpu_memory_gb"] >= 4:
|
84 |
-
model_config = ModelConfig(
|
85 |
-
model_name="Qwen/Qwen2.5-1.5B-Instruct",
|
86 |
-
max_memory_gb=resources["gpu_memory_gb"],
|
87 |
-
inference_speed="balanced"
|
88 |
-
)
|
89 |
-
else:
|
90 |
-
model_config = ModelConfig(
|
91 |
-
model_name="Qwen/Qwen2.5-0.5B-Instruct",
|
92 |
-
max_memory_gb=resources["gpu_memory_gb"],
|
93 |
-
inference_speed="fast"
|
94 |
-
)
|
95 |
-
|
96 |
-
self.qwen_processor = QwenProcessor(model_config)
|
97 |
-
await self.qwen_processor.initialize()
|
98 |
-
self.logger.info("Qwen processor initialized successfully")
|
99 |
-
|
100 |
-
except Exception as e:
|
101 |
-
self.logger.error(f"Failed to initialize Qwen processor: {e}")
|
102 |
-
self.logger.info("Continuing with fallback responses")
|
103 |
-
self.qwen_processor = None
|
104 |
-
|
105 |
-
# Initialize speech engine with fallback handling
|
106 |
-
try:
|
107 |
-
speech_config = SpeechConfig()
|
108 |
-
if resources["gpu_memory_gb"] >= 6:
|
109 |
-
speech_config.model_size = "medium"
|
110 |
-
speech_config.device = "cuda"
|
111 |
-
elif resources["gpu_memory_gb"] >= 3:
|
112 |
-
speech_config.model_size = "small"
|
113 |
-
speech_config.device = "cuda"
|
114 |
-
else:
|
115 |
-
speech_config.model_size = "base"
|
116 |
-
speech_config.device = "cpu"
|
117 |
-
|
118 |
-
self.speech_engine = AdvancedSpeechEngine(speech_config)
|
119 |
-
await self.speech_engine.initialize()
|
120 |
-
self.logger.info("Speech engine initialized successfully")
|
121 |
-
|
122 |
-
except Exception as e:
|
123 |
-
self.logger.error(f"Failed to initialize speech engine: {e}")
|
124 |
-
self.logger.info("Continuing without speech capabilities")
|
125 |
-
self.speech_engine = None
|
126 |
-
|
127 |
-
def create_interface(self) -> gr.Blocks:
|
128 |
-
"""Create the main Gradio interface"""
|
129 |
-
|
130 |
-
# Custom CSS for modern monster game UI
|
131 |
-
custom_css = """
|
132 |
-
/* Modern Dark Theme */
|
133 |
-
.gradio-container {
|
134 |
-
background: linear-gradient(135deg, #1a1a2e 0%, #16213e 50%, #0f3460 100%);
|
135 |
-
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
136 |
-
}
|
137 |
-
|
138 |
-
/* Monster Display */
|
139 |
-
.monster-display {
|
140 |
-
background: linear-gradient(145deg, #2a2a4e, #1e1e3c);
|
141 |
-
border: 3px solid #4a9eff;
|
142 |
-
border-radius: 20px;
|
143 |
-
padding: 20px;
|
144 |
-
text-align: center;
|
145 |
-
box-shadow: 0 10px 30px rgba(74, 158, 255, 0.3);
|
146 |
-
backdrop-filter: blur(10px);
|
147 |
-
min-height: 400px;
|
148 |
-
}
|
149 |
-
|
150 |
-
/* Stat Bars */
|
151 |
-
.stat-bar {
|
152 |
-
background: #1e1e3c;
|
153 |
-
border-radius: 15px;
|
154 |
-
overflow: hidden;
|
155 |
-
margin: 8px 0;
|
156 |
-
height: 25px;
|
157 |
-
border: 2px solid #333;
|
158 |
-
}
|
159 |
-
|
160 |
-
.stat-fill {
|
161 |
-
height: 100%;
|
162 |
-
border-radius: 12px;
|
163 |
-
transition: width 0.8s ease-in-out;
|
164 |
-
background: linear-gradient(90deg, #ff6b6b, #4ecdc4, #45b7d1);
|
165 |
-
}
|
166 |
-
|
167 |
-
/* Care Action Buttons */
|
168 |
-
.care-button {
|
169 |
-
background: linear-gradient(145deg, #4a9eff, #357abd);
|
170 |
-
border: none;
|
171 |
-
color: white;
|
172 |
-
padding: 12px 24px;
|
173 |
-
border-radius: 12px;
|
174 |
-
font-weight: bold;
|
175 |
-
transition: all 0.3s ease;
|
176 |
-
box-shadow: 0 4px 15px rgba(74, 158, 255, 0.4);
|
177 |
-
}
|
178 |
-
|
179 |
-
.care-button:hover {
|
180 |
-
transform: translateY(-3px);
|
181 |
-
box-shadow: 0 8px 25px rgba(74, 158, 255, 0.6);
|
182 |
-
background: linear-gradient(145deg, #5aa7ff, #4a9eff);
|
183 |
-
}
|
184 |
-
|
185 |
-
/* Conversation Area */
|
186 |
-
.conversation-container {
|
187 |
-
background: rgba(30, 30, 60, 0.8);
|
188 |
-
border: 2px solid #4a9eff;
|
189 |
-
border-radius: 15px;
|
190 |
-
backdrop-filter: blur(10px);
|
191 |
-
}
|
192 |
-
|
193 |
-
/* Mini-game Container */
|
194 |
-
.mini-game-area {
|
195 |
-
background: linear-gradient(145deg, #2d1b69, #1a1a2e);
|
196 |
-
border: 2px solid #8b5cf6;
|
197 |
-
border-radius: 15px;
|
198 |
-
padding: 20px;
|
199 |
-
margin: 10px 0;
|
200 |
-
}
|
201 |
-
|
202 |
-
/* Status Indicators */
|
203 |
-
.status-indicator {
|
204 |
-
display: inline-block;
|
205 |
-
width: 12px;
|
206 |
-
height: 12px;
|
207 |
-
border-radius: 50%;
|
208 |
-
margin-right: 8px;
|
209 |
-
}
|
210 |
-
|
211 |
-
.status-healthy { background: #4ade80; }
|
212 |
-
.status-warning { background: #fbbf24; }
|
213 |
-
.status-critical { background: #ef4444; }
|
214 |
-
|
215 |
-
/* Responsive Design */
|
216 |
-
@media (max-width: 768px) {
|
217 |
-
.monster-display {
|
218 |
-
padding: 15px;
|
219 |
-
margin: 10px;
|
220 |
-
}
|
221 |
-
|
222 |
-
.care-button {
|
223 |
-
padding: 10px 20px;
|
224 |
-
margin: 5px;
|
225 |
-
}
|
226 |
-
}
|
227 |
-
"""
|
228 |
-
|
229 |
-
with gr.Blocks(
|
230 |
-
css=custom_css,
|
231 |
-
title="DigiPal - Advanced Monster Companion",
|
232 |
-
theme=gr.themes.Soft()
|
233 |
-
) as interface:
|
234 |
-
|
235 |
-
# Header
|
236 |
-
gr.HTML("""
|
237 |
-
<div style="text-align: center; padding: 20px;">
|
238 |
-
<h1 style="color: #4a9eff; font-size: 2.5em; margin: 0;">🐾 DigiPal</h1>
|
239 |
-
<p style="color: #8b5cf6; font-size: 1.2em;">Advanced AI Monster Companion</p>
|
240 |
-
</div>
|
241 |
-
""")
|
242 |
-
|
243 |
-
# State Management - Modern Gradio 5.34.2 patterns
|
244 |
-
with gr.Row():
|
245 |
-
# Session State for current monster
|
246 |
-
current_monster_state = gr.State(None)
|
247 |
-
|
248 |
-
# Conversation State
|
249 |
-
conversation_state = gr.State([])
|
250 |
-
|
251 |
-
# UI State
|
252 |
-
ui_state = gr.State({
|
253 |
-
"last_action": None,
|
254 |
-
"current_tab": "care",
|
255 |
-
"mini_game_active": False
|
256 |
-
})
|
257 |
-
|
258 |
-
# Main Interface Layout
|
259 |
-
with gr.Row(equal_height=True):
|
260 |
-
|
261 |
-
# Left Column - Monster Display and Stats
|
262 |
-
with gr.Column(scale=3):
|
263 |
-
|
264 |
-
# Monster Display Area
|
265 |
-
monster_display = gr.HTML(
|
266 |
-
value=self._get_default_monster_display(),
|
267 |
-
elem_classes="monster-display"
|
268 |
-
)
|
269 |
-
|
270 |
-
# Monster Management Controls
|
271 |
-
with gr.Row():
|
272 |
-
create_monster_btn = gr.Button(
|
273 |
-
"🥚 Create New Monster",
|
274 |
-
variant="primary",
|
275 |
-
elem_classes="care-button"
|
276 |
-
)
|
277 |
-
load_monster_btn = gr.Button(
|
278 |
-
"📂 Load Monster",
|
279 |
-
elem_classes="care-button"
|
280 |
-
)
|
281 |
-
save_progress_btn = gr.Button(
|
282 |
-
"💾 Save Progress",
|
283 |
-
elem_classes="care-button"
|
284 |
-
)
|
285 |
-
|
286 |
-
# New Monster Creation
|
287 |
-
with gr.Group(visible=False) as monster_creation_group:
|
288 |
-
monster_name_input = gr.Textbox(
|
289 |
-
label="Monster Name",
|
290 |
-
placeholder="Enter your monster's name...",
|
291 |
-
max_lines=1
|
292 |
-
)
|
293 |
-
|
294 |
-
personality_type = gr.Dropdown(
|
295 |
-
choices=[p.value for p in MonsterPersonalityType],
|
296 |
-
label="Personality Type",
|
297 |
-
value="playful"
|
298 |
-
)
|
299 |
-
|
300 |
-
confirm_creation_btn = gr.Button(
|
301 |
-
"✨ Create Monster",
|
302 |
-
variant="primary"
|
303 |
-
)
|
304 |
-
|
305 |
-
# Middle Column - Care Actions and Training
|
306 |
-
with gr.Column(scale=2):
|
307 |
-
|
308 |
-
with gr.Tabs() as care_tabs:
|
309 |
-
|
310 |
-
# Care Tab
|
311 |
-
with gr.TabItem("🍼 Care", id=0):
|
312 |
-
|
313 |
-
# Feeding Section
|
314 |
-
with gr.Group():
|
315 |
-
gr.Markdown("### 🍽️ Feeding")
|
316 |
-
|
317 |
-
food_type = gr.Dropdown(
|
318 |
-
choices=[
|
319 |
-
"meat", "fish", "fruit", "vegetables",
|
320 |
-
"medicine", "supplement", "treat"
|
321 |
-
],
|
322 |
-
value="meat",
|
323 |
-
label="Food Type"
|
324 |
-
)
|
325 |
-
|
326 |
-
feed_btn = gr.Button(
|
327 |
-
"🍖 Feed Monster",
|
328 |
-
elem_classes="care-button"
|
329 |
-
)
|
330 |
-
|
331 |
-
# Training Section
|
332 |
-
with gr.Group():
|
333 |
-
gr.Markdown("### 💪 Training")
|
334 |
-
|
335 |
-
training_type = gr.Dropdown(
|
336 |
-
choices=[
|
337 |
-
"strength", "endurance", "intelligence",
|
338 |
-
"dexterity", "spirit", "technique"
|
339 |
-
],
|
340 |
-
value="strength",
|
341 |
-
label="Training Focus"
|
342 |
-
)
|
343 |
-
|
344 |
-
training_intensity = gr.Slider(
|
345 |
-
minimum=1,
|
346 |
-
maximum=5,
|
347 |
-
value=3,
|
348 |
-
step=1,
|
349 |
-
label="Training Intensity"
|
350 |
-
)
|
351 |
-
|
352 |
-
train_btn = gr.Button(
|
353 |
-
"🏋️ Start Training",
|
354 |
-
elem_classes="care-button"
|
355 |
-
)
|
356 |
-
|
357 |
-
# Care Actions
|
358 |
-
with gr.Group():
|
359 |
-
gr.Markdown("### 🧼 Care Actions")
|
360 |
-
|
361 |
-
with gr.Row():
|
362 |
-
clean_btn = gr.Button("🚿 Clean", elem_classes="care-button")
|
363 |
-
play_btn = gr.Button("🎮 Play", elem_classes="care-button")
|
364 |
-
rest_btn = gr.Button("😴 Rest", elem_classes="care-button")
|
365 |
-
discipline_btn = gr.Button("📚 Discipline", elem_classes="care-button")
|
366 |
-
|
367 |
-
# Evolution Tab
|
368 |
-
with gr.TabItem("🦋 Evolution", id=1):
|
369 |
-
|
370 |
-
evolution_status = gr.HTML(
|
371 |
-
value="<p>No monster loaded</p>"
|
372 |
-
)
|
373 |
-
|
374 |
-
evolution_requirements = gr.JSON(
|
375 |
-
label="Evolution Requirements",
|
376 |
-
value={}
|
377 |
-
)
|
378 |
-
|
379 |
-
trigger_evolution_btn = gr.Button(
|
380 |
-
"🌟 Trigger Evolution",
|
381 |
-
variant="primary",
|
382 |
-
interactive=False
|
383 |
-
)
|
384 |
-
|
385 |
-
# Breeding Tab
|
386 |
-
with gr.TabItem("💕 Breeding", id=2):
|
387 |
-
|
388 |
-
gr.Markdown("### Find a Breeding Partner")
|
389 |
-
|
390 |
-
partner_search = gr.Dropdown(
|
391 |
-
choices=[],
|
392 |
-
label="Available Partners",
|
393 |
-
interactive=False
|
394 |
-
)
|
395 |
-
|
396 |
-
breeding_compatibility = gr.HTML(
|
397 |
-
value="<p>Select a partner to see compatibility</p>"
|
398 |
-
)
|
399 |
-
|
400 |
-
start_breeding_btn = gr.Button(
|
401 |
-
"💖 Start Breeding",
|
402 |
-
variant="primary",
|
403 |
-
interactive=False
|
404 |
-
)
|
405 |
-
|
406 |
-
# Right Column - Conversation and Mini-games
|
407 |
-
with gr.Column(scale=3):
|
408 |
-
|
409 |
-
with gr.Tabs():
|
410 |
-
|
411 |
-
# Conversation Tab
|
412 |
-
with gr.TabItem("💬 Talk", id=0):
|
413 |
-
|
414 |
-
# Conversation Display
|
415 |
-
chatbot = gr.Chatbot(
|
416 |
-
value=[],
|
417 |
-
height=350,
|
418 |
-
label="Conversation with your Monster",
|
419 |
-
elem_classes="conversation-container",
|
420 |
-
avatar_images=("👤", "🐾"),
|
421 |
-
type="messages"
|
422 |
-
)
|
423 |
-
|
424 |
-
# Text Input
|
425 |
-
with gr.Row():
|
426 |
-
text_input = gr.Textbox(
|
427 |
-
label="Message",
|
428 |
-
placeholder="Talk to your monster...",
|
429 |
-
scale=4,
|
430 |
-
max_lines=3
|
431 |
-
)
|
432 |
-
send_btn = gr.Button("💬", scale=1)
|
433 |
-
|
434 |
-
# Voice Input
|
435 |
-
with gr.Group():
|
436 |
-
gr.Markdown("### 🎤 Voice Chat")
|
437 |
-
|
438 |
-
with gr.Row():
|
439 |
-
audio_input = gr.Audio(
|
440 |
-
sources=["microphone"],
|
441 |
-
type="numpy",
|
442 |
-
label="Voice Input",
|
443 |
-
streaming=False
|
444 |
-
)
|
445 |
-
|
446 |
-
voice_btn = gr.Button("🗣️ Send Voice")
|
447 |
-
|
448 |
-
# Real-time audio streaming (Gradio 5.34.2 feature)
|
449 |
-
with gr.Row():
|
450 |
-
start_stream_btn = gr.Button("🎙️ Start Live Chat")
|
451 |
-
stop_stream_btn = gr.Button("⏹️ Stop", interactive=False)
|
452 |
-
|
453 |
-
# Mini-games Tab
|
454 |
-
with gr.TabItem("🎯 Games", id=1):
|
455 |
-
|
456 |
-
mini_game_display = gr.HTML(
|
457 |
-
value=self._get_mini_game_display(),
|
458 |
-
elem_classes="mini-game-area"
|
459 |
-
)
|
460 |
-
|
461 |
-
with gr.Row():
|
462 |
-
reaction_game_btn = gr.Button("⚡ Reaction Training")
|
463 |
-
memory_game_btn = gr.Button("🧠 Memory Challenge")
|
464 |
-
rhythm_game_btn = gr.Button("🎵 Rhythm Game")
|
465 |
-
puzzle_game_btn = gr.Button("🧩 Logic Puzzle")
|
466 |
-
|
467 |
-
game_score_display = gr.JSON(
|
468 |
-
label="Game Statistics",
|
469 |
-
value={}
|
470 |
-
)
|
471 |
-
|
472 |
-
# Stats Tab
|
473 |
-
with gr.TabItem("📊 Statistics", id=2):
|
474 |
-
|
475 |
-
detailed_stats = gr.JSON(
|
476 |
-
label="Detailed Monster Statistics",
|
477 |
-
value={}
|
478 |
-
)
|
479 |
-
|
480 |
-
performance_charts = gr.Plot(
|
481 |
-
label="Performance Over Time"
|
482 |
-
)
|
483 |
-
|
484 |
-
achievement_display = gr.HTML(
|
485 |
-
value="<p>No achievements yet</p>"
|
486 |
-
)
|
487 |
-
|
488 |
-
# Global Status Bar
|
489 |
-
with gr.Row():
|
490 |
-
status_display = gr.HTML(
|
491 |
-
value="<p>Ready to start your monster care journey!</p>",
|
492 |
-
elem_id="status-bar"
|
493 |
-
)
|
494 |
-
|
495 |
-
auto_save_indicator = gr.HTML(
|
496 |
-
value="<span style='color: green;'>● Auto-save: ON</span>",
|
497 |
-
elem_id="auto-save-status"
|
498 |
-
)
|
499 |
-
|
500 |
-
# Hidden components for data flow
|
501 |
-
action_result = gr.Textbox(visible=False)
|
502 |
-
background_timer = gr.Timer(value=30, active=True) # 30-second updates
|
503 |
-
|
504 |
-
# Event Handlers with Modern Async Patterns
|
505 |
-
|
506 |
-
# Monster Creation Flow
|
507 |
-
create_monster_btn.click(
|
508 |
-
fn=lambda: gr.update(visible=True),
|
509 |
-
outputs=monster_creation_group
|
510 |
-
)
|
511 |
-
|
512 |
-
confirm_creation_btn.click(
|
513 |
-
fn=lambda name, personality: safe_create_monster(self, name, personality),
|
514 |
-
inputs=[monster_name_input, personality_type],
|
515 |
-
outputs=[current_monster_state, monster_display, monster_creation_group]
|
516 |
-
)
|
517 |
-
|
518 |
-
# Feeding handlers
|
519 |
-
feed_btn.click(
|
520 |
-
fn=lambda monster_state, food: asyncio.run(self.feed_monster(monster_state, food)),
|
521 |
-
inputs=[current_monster_state, food_type],
|
522 |
-
outputs=[current_monster_state, monster_display, action_result, chatbot]
|
523 |
-
)
|
524 |
-
|
525 |
-
# Training handlers
|
526 |
-
train_btn.click(
|
527 |
-
fn=lambda monster_state, training, intensity: asyncio.run(self.train_monster(monster_state, training, intensity)),
|
528 |
-
inputs=[current_monster_state, training_type, training_intensity],
|
529 |
-
outputs=[current_monster_state, monster_display, action_result]
|
530 |
-
)
|
531 |
-
|
532 |
-
# Conversation handlers
|
533 |
-
send_btn.click(
|
534 |
-
fn=lambda monster_state, message, history: safe_handle_conversation(self, monster_state, message, history),
|
535 |
-
inputs=[current_monster_state, text_input, conversation_state],
|
536 |
-
outputs=[chatbot, text_input, conversation_state, current_monster_state]
|
537 |
-
)
|
538 |
-
|
539 |
-
text_input.submit(
|
540 |
-
fn=lambda monster_state, message, history: safe_handle_conversation(self, monster_state, message, history),
|
541 |
-
inputs=[current_monster_state, text_input, conversation_state],
|
542 |
-
outputs=[chatbot, text_input, conversation_state, current_monster_state]
|
543 |
-
)
|
544 |
-
|
545 |
-
# Voice handlers
|
546 |
-
voice_btn.click(
|
547 |
-
fn=lambda monster_state, audio, history: safe_handle_voice(self, monster_state, audio, history),
|
548 |
-
inputs=[current_monster_state, audio_input, conversation_state],
|
549 |
-
outputs=[chatbot, conversation_state, current_monster_state, action_result]
|
550 |
-
)
|
551 |
-
|
552 |
-
# Streaming handlers
|
553 |
-
start_stream_btn.click(
|
554 |
-
fn=self.start_voice_streaming,
|
555 |
-
outputs=[start_stream_btn, stop_stream_btn]
|
556 |
-
)
|
557 |
-
|
558 |
-
# Care Actions
|
559 |
-
feed_btn.click(
|
560 |
-
fn=self.feed_monster,
|
561 |
-
inputs=[current_monster_state, food_type],
|
562 |
-
outputs=[current_monster_state, monster_display, action_result, chatbot]
|
563 |
-
)
|
564 |
-
|
565 |
-
train_btn.click(
|
566 |
-
fn=self.train_monster,
|
567 |
-
inputs=[current_monster_state, training_type, training_intensity],
|
568 |
-
outputs=[current_monster_state, monster_display, action_result]
|
569 |
-
)
|
570 |
-
|
571 |
-
# Conversation Handlers
|
572 |
-
send_btn.click(
|
573 |
-
fn=lambda monster_state, message, history: safe_handle_conversation(self, monster_state, message, history),
|
574 |
-
inputs=[current_monster_state, text_input, conversation_state],
|
575 |
-
outputs=[chatbot, text_input, conversation_state, current_monster_state]
|
576 |
-
)
|
577 |
-
|
578 |
-
text_input.submit(
|
579 |
-
fn=lambda monster_state, message, history: safe_handle_conversation(self, monster_state, message, history),
|
580 |
-
inputs=[current_monster_state, text_input, conversation_state],
|
581 |
-
outputs=[chatbot, text_input, conversation_state, current_monster_state]
|
582 |
-
)
|
583 |
-
|
584 |
-
voice_btn.click(
|
585 |
-
fn=lambda monster_state, audio, history: safe_handle_voice(self, monster_state, audio, history),
|
586 |
-
inputs=[current_monster_state, audio_input, conversation_state],
|
587 |
-
outputs=[chatbot, conversation_state, current_monster_state, action_result]
|
588 |
-
)
|
589 |
-
|
590 |
-
# Real-time streaming (Gradio 5.34.2)
|
591 |
-
start_stream_btn.click(
|
592 |
-
fn=self.start_voice_streaming,
|
593 |
-
outputs=[start_stream_btn, stop_stream_btn]
|
594 |
-
)
|
595 |
-
|
596 |
-
stop_stream_btn.click(
|
597 |
-
fn=self.stop_voice_streaming,
|
598 |
-
outputs=[start_stream_btn, stop_stream_btn]
|
599 |
-
)
|
600 |
-
|
601 |
-
# Background Updates
|
602 |
-
background_timer.tick(
|
603 |
-
fn=self.background_update,
|
604 |
-
inputs=[current_monster_state],
|
605 |
-
outputs=[current_monster_state, monster_display, auto_save_indicator]
|
606 |
-
)
|
607 |
-
|
608 |
-
# Care action handlers
|
609 |
-
def clean_action(monster_state):
|
610 |
-
return asyncio.run(self.perform_care_action(monster_state, "clean"))
|
611 |
-
|
612 |
-
def play_action(monster_state):
|
613 |
-
return asyncio.run(self.perform_care_action(monster_state, "play"))
|
614 |
-
|
615 |
-
def rest_action(monster_state):
|
616 |
-
return asyncio.run(self.perform_care_action(monster_state, "rest"))
|
617 |
-
|
618 |
-
def discipline_action(monster_state):
|
619 |
-
return asyncio.run(self.perform_care_action(monster_state, "discipline"))
|
620 |
-
|
621 |
-
clean_btn.click(
|
622 |
-
fn=clean_action,
|
623 |
-
inputs=[current_monster_state],
|
624 |
-
outputs=[current_monster_state, monster_display, action_result]
|
625 |
-
)
|
626 |
-
|
627 |
-
play_btn.click(
|
628 |
-
fn=play_action,
|
629 |
-
inputs=[current_monster_state],
|
630 |
-
outputs=[current_monster_state, monster_display, action_result]
|
631 |
-
)
|
632 |
-
|
633 |
-
rest_btn.click(
|
634 |
-
fn=rest_action,
|
635 |
-
inputs=[current_monster_state],
|
636 |
-
outputs=[current_monster_state, monster_display, action_result]
|
637 |
-
)
|
638 |
-
|
639 |
-
discipline_btn.click(
|
640 |
-
fn=discipline_action,
|
641 |
-
inputs=[current_monster_state],
|
642 |
-
outputs=[current_monster_state, monster_display, action_result]
|
643 |
-
)
|
644 |
-
|
645 |
-
# Mini-game handlers
|
646 |
-
for btn, game in [(reaction_game_btn, "reaction"), (memory_game_btn, "memory"),
|
647 |
-
(rhythm_game_btn, "rhythm"), (puzzle_game_btn, "puzzle")]:
|
648 |
-
btn.click(
|
649 |
-
fn=lambda monster_state, game=game: self.start_mini_game(monster_state, game),
|
650 |
-
inputs=[current_monster_state],
|
651 |
-
outputs=[mini_game_display, game_score_display]
|
652 |
-
)
|
653 |
-
|
654 |
-
return interface
|
655 |
-
|
656 |
-
# Implementation methods continue...
|
657 |
-
|
658 |
-
async def create_new_monster(self, name: str, personality: str) -> Tuple:
|
659 |
-
"""Create a new monster with specified parameters"""
|
660 |
-
try:
|
661 |
-
if not name.strip():
|
662 |
-
return None, self._get_default_monster_display(), gr.update(visible=True)
|
663 |
-
|
664 |
-
# Create monster with personality
|
665 |
-
monster = Monster(
|
666 |
-
name=name.strip(),
|
667 |
-
species="Botamon" # Starting species
|
668 |
-
)
|
669 |
-
|
670 |
-
# Set personality
|
671 |
-
monster.personality.primary_type = MonsterPersonalityType(personality)
|
672 |
-
|
673 |
-
# Randomize personality traits based on type
|
674 |
-
trait_modifiers = {
|
675 |
-
"playful": {"extraversion": 0.8, "openness": 0.7, "agreeableness": 0.6},
|
676 |
-
"serious": {"conscientiousness": 0.8, "neuroticism": 0.3, "extraversion": 0.4},
|
677 |
-
"curious": {"openness": 0.9, "extraversion": 0.6, "conscientiousness": 0.5},
|
678 |
-
"gentle": {"agreeableness": 0.9, "neuroticism": 0.2, "extraversion": 0.5},
|
679 |
-
"energetic": {"extraversion": 0.9, "openness": 0.6, "neuroticism": 0.3},
|
680 |
-
"calm": {"neuroticism": 0.1, "conscientiousness": 0.7, "agreeableness": 0.7},
|
681 |
-
"mischievous": {"openness": 0.8, "extraversion": 0.7, "conscientiousness": 0.3},
|
682 |
-
"loyal": {"agreeableness": 0.8, "conscientiousness": 0.9, "neuroticism": 0.2}
|
683 |
-
}
|
684 |
-
|
685 |
-
modifiers = trait_modifiers.get(personality, {})
|
686 |
-
for trait, value in modifiers.items():
|
687 |
-
if hasattr(monster.personality, trait):
|
688 |
-
setattr(monster.personality, trait, value)
|
689 |
-
|
690 |
-
# Save monster
|
691 |
-
await self.state_manager.save_monster(monster)
|
692 |
-
|
693 |
-
# Generate display
|
694 |
-
display_html = self._generate_monster_display(monster)
|
695 |
-
|
696 |
-
self.current_monster = monster
|
697 |
-
|
698 |
-
return (
|
699 |
-
monster.dict(),
|
700 |
-
display_html,
|
701 |
-
gr.update(visible=False)
|
702 |
-
)
|
703 |
-
|
704 |
-
except Exception as e:
|
705 |
-
self.logger.error(f"Monster creation failed: {e}")
|
706 |
-
return None, self._get_error_display(str(e)), gr.update(visible=True)
|
707 |
-
|
708 |
-
def _get_default_monster_display(self) -> str:
|
709 |
-
"""Get default monster display when no monster is loaded"""
|
710 |
-
return """
|
711 |
-
<div style="text-align: center; padding: 40px;">
|
712 |
-
<div style="font-size: 4em; margin-bottom: 20px;">🥚</div>
|
713 |
-
<h2 style="color: #4a9eff;">No Monster Loaded</h2>
|
714 |
-
<p style="color: #8b5cf6;">Create a new monster to begin your journey!</p>
|
715 |
-
</div>
|
716 |
-
"""
|
717 |
-
|
718 |
-
def _generate_monster_display(self, monster: Monster) -> str:
|
719 |
-
"""Generate HTML display for the monster"""
|
720 |
-
# Monster sprite based on species and stage
|
721 |
-
sprite_map = {
|
722 |
-
"Botamon": {"egg": "🥚", "baby": "🐣", "child": "🐾", "adult": "🐲"},
|
723 |
-
# Add more species...
|
724 |
-
}
|
725 |
-
|
726 |
-
sprite = sprite_map.get(monster.species, {}).get(monster.lifecycle.stage.value, "🐾")
|
727 |
-
|
728 |
-
# Emotional state emoji
|
729 |
-
emotion_emojis = {
|
730 |
-
"ecstatic": "🤩", "happy": "😊", "content": "😌", "neutral": "😐",
|
731 |
-
"melancholy": "😔", "sad": "😢", "angry": "😠", "sick": "🤒",
|
732 |
-
"excited": "😆", "tired": "😴"
|
733 |
-
}
|
734 |
-
|
735 |
-
emotion_emoji = emotion_emojis.get(monster.emotional_state.value, "😐")
|
736 |
-
|
737 |
-
# Calculate stat colors
|
738 |
-
def get_stat_color(value: int) -> str:
|
739 |
-
if value >= 80: return "#4ade80" # Green
|
740 |
-
elif value >= 60: return "#fbbf24" # Yellow
|
741 |
-
elif value >= 40: return "#fb923c" # Orange
|
742 |
-
else: return "#ef4444" # Red
|
743 |
-
|
744 |
-
# Age display
|
745 |
-
age_days = monster.lifecycle.age_minutes / 1440
|
746 |
-
age_display = f"{age_days:.1f} days"
|
747 |
-
|
748 |
-
return f"""
|
749 |
-
<div style="text-align: center; padding: 20px;">
|
750 |
-
|
751 |
-
<!-- Monster Sprite -->
|
752 |
-
<div style="font-size: 6em; margin: 20px 0;">{sprite}</div>
|
753 |
-
|
754 |
-
<!-- Monster Info -->
|
755 |
-
<h2 style="color: #4a9eff; margin: 10px 0;">{monster.name} {emotion_emoji}</h2>
|
756 |
-
<p style="color: #8b5cf6; margin: 5px 0;">
|
757 |
-
<strong>{monster.species}</strong> | {monster.lifecycle.stage.value.title()} | {age_display}
|
758 |
-
</p>
|
759 |
-
|
760 |
-
<!-- Mood and Activity -->
|
761 |
-
<p style="color: #a78bfa; margin: 10px 0;">
|
762 |
-
Feeling {monster.emotional_state.value} while {monster.current_activity}
|
763 |
-
</p>
|
764 |
-
|
765 |
-
<!-- Care Stats -->
|
766 |
-
<div style="margin: 20px 0;">
|
767 |
-
<h3 style="color: #4a9eff;">Care Status</h3>
|
768 |
-
|
769 |
-
<div style="text-align: left; max-width: 300px; margin: 0 auto;">
|
770 |
-
<div style="margin: 8px 0;">
|
771 |
-
<span style="color: white;">Health</span>
|
772 |
-
<div class="stat-bar">
|
773 |
-
<div class="stat-fill" style="width: {monster.stats.health}%; background: {get_stat_color(monster.stats.health)};"></div>
|
774 |
-
</div>
|
775 |
-
<span style="color: #888; font-size: 0.9em;">{monster.stats.health}/100</span>
|
776 |
-
</div>
|
777 |
-
|
778 |
-
<div style="margin: 8px 0;">
|
779 |
-
<span style="color: white;">Happiness</span>
|
780 |
-
<div class="stat-bar">
|
781 |
-
<div class="stat-fill" style="width: {monster.stats.happiness}%; background: {get_stat_color(monster.stats.happiness)};"></div>
|
782 |
-
</div>
|
783 |
-
<span style="color: #888; font-size: 0.9em;">{monster.stats.happiness}/100</span>
|
784 |
-
</div>
|
785 |
-
|
786 |
-
<div style="margin: 8px 0;">
|
787 |
-
<span style="color: white;">Hunger</span>
|
788 |
-
<div class="stat-bar">
|
789 |
-
<div class="stat-fill" style="width: {monster.stats.hunger}%; background: {get_stat_color(monster.stats.hunger)};"></div>
|
790 |
-
</div>
|
791 |
-
<span style="color: #888; font-size: 0.9em;">{monster.stats.hunger}/100</span>
|
792 |
-
</div>
|
793 |
-
|
794 |
-
<div style="margin: 8px 0;">
|
795 |
-
<span style="color: white;">Energy</span>
|
796 |
-
<div class="stat-bar">
|
797 |
-
<div class="stat-fill" style="width: {monster.stats.energy}%; background: {get_stat_color(monster.stats.energy)};"></div>
|
798 |
-
</div>
|
799 |
-
<span style="color: #888; font-size: 0.9em;">{monster.stats.energy}/100</span>
|
800 |
-
</div>
|
801 |
-
</div>
|
802 |
-
</div>
|
803 |
-
|
804 |
-
<!-- Battle Stats -->
|
805 |
-
<div style="margin: 20px 0;">
|
806 |
-
<h3 style="color: #8b5cf6;">Battle Power</h3>
|
807 |
-
<div style="display: grid; grid-template-columns: 1fr 1fr; gap: 10px; max-width: 300px; margin: 0 auto;">
|
808 |
-
<div>Life: <strong style="color: #4ade80;">{monster.stats.life}</strong></div>
|
809 |
-
<div>MP: <strong style="color: #60a5fa;">{monster.stats.mp}</strong></div>
|
810 |
-
<div>Offense: <strong style="color: #f87171;">{monster.stats.offense}</strong></div>
|
811 |
-
<div>Defense: <strong style="color: #34d399;">{monster.stats.defense}</strong></div>
|
812 |
-
<div>Speed: <strong style="color: #fbbf24;">{monster.stats.speed}</strong></div>
|
813 |
-
<div>Brains: <strong style="color: #a78bfa;">{monster.stats.brains}</strong></div>
|
814 |
-
</div>
|
815 |
-
</div>
|
816 |
-
|
817 |
-
<!-- Generation and Care Info -->
|
818 |
-
<div style="margin: 15px 0; font-size: 0.9em; color: #888;">
|
819 |
-
Generation {monster.lifecycle.generation} |
|
820 |
-
Care Mistakes: {monster.lifecycle.care_mistakes} |
|
821 |
-
Relationship: {monster.personality.relationship_level}/100
|
822 |
-
</div>
|
823 |
-
|
824 |
-
</div>
|
825 |
-
"""
|
826 |
-
|
827 |
-
def _get_mini_game_display(self) -> str:
|
828 |
-
"""Get mini-game display HTML"""
|
829 |
-
return """
|
830 |
-
<div style="text-align: center; padding: 20px;">
|
831 |
-
<h3 style="color: #8b5cf6;">Mini-Games Training Center</h3>
|
832 |
-
<p style="color: #a78bfa;">Select a mini-game to train your monster!</p>
|
833 |
-
<div style="margin-top: 20px;">
|
834 |
-
<p>⚡ Reaction: Improve Speed & Reflexes</p>
|
835 |
-
<p>🧠 Memory: Enhance Intelligence</p>
|
836 |
-
<p>🎵 Rhythm: Boost Spirit & Happiness</p>
|
837 |
-
<p>🧩 Logic: Develop Problem-Solving</p>
|
838 |
-
</div>
|
839 |
-
</div>
|
840 |
-
"""
|
841 |
-
|
842 |
-
def _get_error_display(self, error: str) -> str:
|
843 |
-
"""Get error display HTML"""
|
844 |
-
return f"""
|
845 |
-
<div style="text-align: center; padding: 40px;">
|
846 |
-
<div style="font-size: 3em; margin-bottom: 20px;">❌</div>
|
847 |
-
<h2 style="color: #ef4444;">Error Occurred</h2>
|
848 |
-
<p style="color: #f87171;">{error}</p>
|
849 |
-
</div>
|
850 |
-
"""
|
851 |
-
|
852 |
-
async def feed_monster(self, monster_state: Dict, food_type: str) -> Tuple:
|
853 |
-
"""Feed the monster"""
|
854 |
-
if not monster_state:
|
855 |
-
return monster_state, self._get_default_monster_display(), "No monster loaded!", []
|
856 |
-
|
857 |
-
try:
|
858 |
-
monster = Monster(**monster_state)
|
859 |
-
|
860 |
-
# Food effects
|
861 |
-
food_effects = {
|
862 |
-
"meat": {"hunger": 30, "happiness": 10},
|
863 |
-
"fish": {"hunger": 25, "happiness": 15, "health": 5},
|
864 |
-
"fruit": {"hunger": 20, "happiness": 20},
|
865 |
-
"vegetables": {"hunger": 25, "happiness": 5, "health": 10},
|
866 |
-
"medicine": {"health": 50, "happiness": -10},
|
867 |
-
"supplement": {"energy": 20, "happiness": 5},
|
868 |
-
"treat": {"happiness": 30, "hunger": 10}
|
869 |
-
}
|
870 |
-
|
871 |
-
effects = food_effects.get(food_type, food_effects["meat"])
|
872 |
-
|
873 |
-
# Apply effects
|
874 |
-
for stat, value in effects.items():
|
875 |
-
current = getattr(monster.stats, stat)
|
876 |
-
setattr(monster.stats, stat, max(0, min(100, current + value)))
|
877 |
-
|
878 |
-
# Update emotional state
|
879 |
-
monster.emotional_state = monster.calculate_emotional_state()
|
880 |
-
|
881 |
-
# Save monster
|
882 |
-
await self.state_manager.save_monster(monster)
|
883 |
-
|
884 |
-
# Generate response
|
885 |
-
response = f"{monster.name} enjoyed the {food_type}! 😋"
|
886 |
-
|
887 |
-
# Create message format for chatbot
|
888 |
-
messages = [
|
889 |
-
{"role": "user", "content": f"Fed {food_type}"},
|
890 |
-
{"role": "assistant", "content": response}
|
891 |
-
]
|
892 |
-
|
893 |
-
return (
|
894 |
-
monster.dict(),
|
895 |
-
self._generate_monster_display(monster),
|
896 |
-
response,
|
897 |
-
messages
|
898 |
-
)
|
899 |
-
|
900 |
-
except Exception as e:
|
901 |
-
self.logger.error(f"Feeding failed: {e}")
|
902 |
-
return monster_state, self._get_error_display(str(e)), str(e), []
|
903 |
-
|
904 |
-
async def train_monster(self, monster_state: Dict, training_type: str, intensity: int) -> Tuple:
|
905 |
-
"""Train the monster"""
|
906 |
-
if not monster_state:
|
907 |
-
return monster_state, self._get_default_monster_display(), "No monster loaded!"
|
908 |
-
|
909 |
-
try:
|
910 |
-
monster = Monster(**monster_state)
|
911 |
-
|
912 |
-
# Check if monster can train
|
913 |
-
if monster.stats.energy < 20:
|
914 |
-
return monster_state, self._generate_monster_display(monster), f"{monster.name} is too tired to train! 😴"
|
915 |
-
|
916 |
-
# Training effects
|
917 |
-
training_effects = {
|
918 |
-
"strength": {"offense": 5 * intensity, "life": 20 * intensity},
|
919 |
-
"endurance": {"defense": 5 * intensity, "life": 30 * intensity},
|
920 |
-
"intelligence": {"brains": 8 * intensity, "mp": 10 * intensity},
|
921 |
-
"dexterity": {"speed": 6 * intensity},
|
922 |
-
"spirit": {"mp": 15 * intensity, "happiness": 5},
|
923 |
-
"technique": {"offense": 3 * intensity, "defense": 3 * intensity}
|
924 |
-
}
|
925 |
-
|
926 |
-
effects = training_effects.get(training_type, {})
|
927 |
-
|
928 |
-
# Apply stat increases
|
929 |
-
for stat, increase in effects.items():
|
930 |
-
if hasattr(monster.stats, stat):
|
931 |
-
current = getattr(monster.stats, stat)
|
932 |
-
setattr(monster.stats, stat, current + increase)
|
933 |
-
|
934 |
-
# Update training progress
|
935 |
-
if training_type in monster.stats.training_progress:
|
936 |
-
monster.stats.training_progress[training_type] += 10 * intensity
|
937 |
-
|
938 |
-
# Training costs
|
939 |
-
monster.stats.energy = max(0, monster.stats.energy - (15 * intensity))
|
940 |
-
monster.stats.hunger = max(0, monster.stats.hunger - (10 * intensity))
|
941 |
-
|
942 |
-
# Update emotional state
|
943 |
-
monster.emotional_state = monster.calculate_emotional_state()
|
944 |
-
monster.current_activity = "training"
|
945 |
-
|
946 |
-
# Save monster
|
947 |
-
await self.state_manager.save_monster(monster)
|
948 |
-
|
949 |
-
response = f"{monster.name} completed {training_type} training! 💪"
|
950 |
-
|
951 |
-
return (
|
952 |
-
monster.dict(),
|
953 |
-
self._generate_monster_display(monster),
|
954 |
-
response
|
955 |
-
)
|
956 |
-
|
957 |
-
except Exception as e:
|
958 |
-
self.logger.error(f"Training failed: {e}")
|
959 |
-
return monster_state, self._get_error_display(str(e)), str(e)
|
960 |
-
|
961 |
-
async def handle_text_conversation(self, monster_state: Dict, message: str, conversation_history: List) -> Tuple:
|
962 |
-
"""Handle text conversation with monster"""
|
963 |
-
if not monster_state or not message.strip():
|
964 |
-
return conversation_history, "", conversation_history, monster_state if monster_state else {}
|
965 |
-
|
966 |
-
try:
|
967 |
-
monster = Monster(**monster_state)
|
968 |
-
|
969 |
-
# Generate AI response with fallback
|
970 |
-
if self.qwen_processor:
|
971 |
-
response_data = await self.qwen_processor.generate_monster_response(
|
972 |
-
monster.dict(),
|
973 |
-
message,
|
974 |
-
conversation_history
|
975 |
-
)
|
976 |
-
response = response_data["response"]
|
977 |
-
else:
|
978 |
-
# Fallback response when AI is not available
|
979 |
-
response = self._get_fallback_response(monster, message)
|
980 |
-
response_data = {
|
981 |
-
"response": response,
|
982 |
-
"emotional_impact": {"happiness": 0.1, "bonding": 0.02},
|
983 |
-
"inference_time": 0.0
|
984 |
-
}
|
985 |
-
|
986 |
-
# Update conversation history
|
987 |
-
conversation_history.append([message, response])
|
988 |
-
|
989 |
-
# Convert to messages format for Gradio chatbot with type='messages'
|
990 |
-
messages_format = []
|
991 |
-
for msg in conversation_history:
|
992 |
-
if isinstance(msg, list) and len(msg) == 2:
|
993 |
-
messages_format.append({"role": "user", "content": msg[0]})
|
994 |
-
messages_format.append({"role": "assistant", "content": msg[1]})
|
995 |
-
|
996 |
-
# Update monster state based on interaction
|
997 |
-
monster.conversation.total_conversations += 1
|
998 |
-
monster.conversation.last_interaction = datetime.now()
|
999 |
-
monster.stats.happiness = min(100, monster.stats.happiness + 2)
|
1000 |
-
monster.personality.relationship_level = min(100, monster.personality.relationship_level + 1)
|
1001 |
-
|
1002 |
-
# Apply emotional impact
|
1003 |
-
emotional_impact = response_data.get("emotional_impact", {})
|
1004 |
-
for emotion, value in emotional_impact.items():
|
1005 |
-
if emotion == "happiness":
|
1006 |
-
monster.stats.happiness = max(0, min(100, monster.stats.happiness + int(value * 10)))
|
1007 |
-
elif emotion == "bonding":
|
1008 |
-
monster.personality.relationship_level = min(100, monster.personality.relationship_level + int(value * 5))
|
1009 |
-
|
1010 |
-
# Save monster
|
1011 |
-
await self.state_manager.save_monster(monster)
|
1012 |
-
|
1013 |
-
return messages_format, "", conversation_history, monster.dict()
|
1014 |
-
|
1015 |
-
except Exception as e:
|
1016 |
-
self.logger.error(f"Conversation failed: {e}")
|
1017 |
-
return conversation_history, "", conversation_history, monster_state if monster_state else {}
|
1018 |
-
|
1019 |
-
async def handle_voice_input(self, monster_state: Dict, audio_data, conversation_history: List) -> Tuple:
|
1020 |
-
"""Handle voice input"""
|
1021 |
-
if not monster_state or audio_data is None:
|
1022 |
-
return conversation_history, conversation_history, monster_state, ""
|
1023 |
-
|
1024 |
-
try:
|
1025 |
-
# Process speech with fallback
|
1026 |
-
if self.speech_engine:
|
1027 |
-
speech_result = await self.speech_engine.process_audio_stream(audio_data[1])
|
1028 |
-
|
1029 |
-
if not speech_result["success"]:
|
1030 |
-
return conversation_history, conversation_history, monster_state, "Speech processing failed"
|
1031 |
-
|
1032 |
-
transcribed_text = speech_result["transcription"]
|
1033 |
-
if not transcribed_text.strip():
|
1034 |
-
return conversation_history, conversation_history, monster_state, "No speech detected"
|
1035 |
-
else:
|
1036 |
-
return conversation_history, conversation_history, monster_state, "Speech processing not available"
|
1037 |
-
|
1038 |
-
# Process as text conversation
|
1039 |
-
new_history, _, updated_history, updated_monster = await self.handle_text_conversation(
|
1040 |
-
monster_state, transcribed_text, conversation_history
|
1041 |
-
)
|
1042 |
-
|
1043 |
-
return new_history, updated_history, updated_monster, f"Heard: \"{transcribed_text}\""
|
1044 |
-
|
1045 |
-
except Exception as e:
|
1046 |
-
self.logger.error(f"Voice input failed: {e}")
|
1047 |
-
return conversation_history, conversation_history, monster_state, str(e)
|
1048 |
-
|
1049 |
-
async def perform_care_action(self, monster_state: Dict, action: str) -> Tuple:
|
1050 |
-
"""Perform care action on monster"""
|
1051 |
-
if not monster_state:
|
1052 |
-
return monster_state, self._get_default_monster_display(), "No monster loaded!"
|
1053 |
-
|
1054 |
-
try:
|
1055 |
-
monster = Monster(**monster_state)
|
1056 |
-
|
1057 |
-
care_effects = {
|
1058 |
-
"clean": {"cleanliness": 50, "happiness": 10},
|
1059 |
-
"play": {"happiness": 25, "energy": -15, "relationship": 5},
|
1060 |
-
"rest": {"energy": 40, "happiness": 5},
|
1061 |
-
"discipline": {"discipline": 20, "happiness": -10}
|
1062 |
-
}
|
1063 |
-
|
1064 |
-
effects = care_effects.get(action, {})
|
1065 |
-
|
1066 |
-
# Apply effects
|
1067 |
-
for stat, value in effects.items():
|
1068 |
-
if stat == "relationship":
|
1069 |
-
monster.personality.relationship_level = min(100, monster.personality.relationship_level + value)
|
1070 |
-
elif hasattr(monster.stats, stat):
|
1071 |
-
current = getattr(monster.stats, stat)
|
1072 |
-
setattr(monster.stats, stat, max(0, min(100, current + value)))
|
1073 |
-
|
1074 |
-
# Update activity
|
1075 |
-
monster.current_activity = action
|
1076 |
-
monster.emotional_state = monster.calculate_emotional_state()
|
1077 |
-
|
1078 |
-
# Save monster
|
1079 |
-
await self.state_manager.save_monster(monster)
|
1080 |
-
|
1081 |
-
response = f"{monster.name} is now {action}ing! ✨"
|
1082 |
-
|
1083 |
-
return (
|
1084 |
-
monster.dict(),
|
1085 |
-
self._generate_monster_display(monster),
|
1086 |
-
response
|
1087 |
-
)
|
1088 |
-
|
1089 |
-
except Exception as e:
|
1090 |
-
self.logger.error(f"Care action failed: {e}")
|
1091 |
-
return monster_state, self._get_error_display(str(e)), str(e)
|
1092 |
-
|
1093 |
-
async def background_update(self, monster_state: Dict) -> Tuple:
|
1094 |
-
"""Background update for time-based effects"""
|
1095 |
-
if not monster_state:
|
1096 |
-
return monster_state, self._get_default_monster_display(), gr.update()
|
1097 |
-
|
1098 |
-
try:
|
1099 |
-
monster = Monster(**monster_state)
|
1100 |
-
|
1101 |
-
# Calculate time elapsed
|
1102 |
-
time_elapsed = (datetime.now() - monster.last_update).total_seconds() / 60 # minutes
|
1103 |
-
|
1104 |
-
# Apply time effects
|
1105 |
-
monster.apply_time_effects(time_elapsed)
|
1106 |
-
|
1107 |
-
# Save monster
|
1108 |
-
await self.state_manager.save_monster(monster)
|
1109 |
-
|
1110 |
-
# Update save indicator
|
1111 |
-
save_indicator = f"<span style='color: green;'>● Auto-saved at {datetime.now().strftime('%H:%M:%S')}</span>"
|
1112 |
-
|
1113 |
-
return (
|
1114 |
-
monster.dict(),
|
1115 |
-
self._generate_monster_display(monster),
|
1116 |
-
save_indicator
|
1117 |
-
)
|
1118 |
-
|
1119 |
-
except Exception as e:
|
1120 |
-
self.logger.error(f"Background update failed: {e}")
|
1121 |
-
return monster_state, self._get_error_display(str(e)), gr.update()
|
1122 |
-
|
1123 |
-
def start_mini_game(self, monster_state: Dict, game_type: str) -> Tuple:
|
1124 |
-
"""Start a mini-game"""
|
1125 |
-
if not monster_state:
|
1126 |
-
return self._get_mini_game_display(), {}
|
1127 |
-
|
1128 |
-
# Placeholder for mini-game implementation
|
1129 |
-
game_display = f"""
|
1130 |
-
<div style="text-align: center; padding: 20px;">
|
1131 |
-
<h3 style="color: #8b5cf6;">{game_type.title()} Training</h3>
|
1132 |
-
<p>Mini-game implementation coming soon!</p>
|
1133 |
-
</div>
|
1134 |
-
"""
|
1135 |
-
|
1136 |
-
game_stats = {
|
1137 |
-
"game_type": game_type,
|
1138 |
-
"status": "not_implemented"
|
1139 |
-
}
|
1140 |
-
|
1141 |
-
return game_display, game_stats
|
1142 |
-
|
1143 |
-
def start_voice_streaming(self) -> Tuple:
|
1144 |
-
"""Start voice streaming"""
|
1145 |
-
return gr.update(interactive=False), gr.update(interactive=True)
|
1146 |
-
|
1147 |
-
def stop_voice_streaming(self) -> Tuple:
|
1148 |
-
"""Stop voice streaming"""
|
1149 |
-
return gr.update(interactive=True), gr.update(interactive=False)
|
1150 |
-
|
1151 |
-
def _get_fallback_response(self, monster: 'Monster', message: str) -> str:
|
1152 |
-
"""Get fallback response when AI is not available"""
|
1153 |
-
import random
|
1154 |
-
|
1155 |
-
# Simple rule-based responses
|
1156 |
-
message_lower = message.lower()
|
1157 |
-
|
1158 |
-
greetings = ["hello", "hi", "hey", "good morning", "good afternoon", "good evening"]
|
1159 |
-
questions = ["how", "what", "when", "where", "why", "who"]
|
1160 |
-
positive_words = ["good", "great", "love", "happy", "fun", "wonderful", "amazing"]
|
1161 |
-
negative_words = ["bad", "sad", "angry", "hate", "terrible", "awful"]
|
1162 |
-
|
1163 |
-
responses = []
|
1164 |
-
|
1165 |
-
# Greeting responses
|
1166 |
-
if any(greeting in message_lower for greeting in greetings):
|
1167 |
-
responses = [
|
1168 |
-
f"Hello there! {monster.name} is happy to see you! 😊",
|
1169 |
-
f"Hi! {monster.name} waves excitedly! 👋",
|
1170 |
-
f"Hey! {monster.name} is ready to play! 🎮"
|
1171 |
-
]
|
1172 |
-
|
1173 |
-
# Question responses
|
1174 |
-
elif any(q in message_lower for q in questions):
|
1175 |
-
responses = [
|
1176 |
-
f"*{monster.name} tilts head thoughtfully* 🤔",
|
1177 |
-
f"That's a great question! {monster.name} is thinking... 💭",
|
1178 |
-
f"*{monster.name} looks curious* Tell me more! 👀"
|
1179 |
-
]
|
1180 |
-
|
1181 |
-
# Positive responses
|
1182 |
-
elif any(word in message_lower for word in positive_words):
|
1183 |
-
responses = [
|
1184 |
-
f"{monster.name} is so happy! 😄",
|
1185 |
-
f"*{monster.name} does a little dance* 💃",
|
1186 |
-
f"That makes {monster.name} feel great! ✨"
|
1187 |
-
]
|
1188 |
-
|
1189 |
-
# Negative responses
|
1190 |
-
elif any(word in message_lower for word in negative_words):
|
1191 |
-
responses = [
|
1192 |
-
f"*{monster.name} looks concerned* 😟",
|
1193 |
-
f"{monster.name} wants to help make things better! 💗",
|
1194 |
-
f"*{monster.name} offers a gentle hug* 🤗"
|
1195 |
-
]
|
1196 |
-
|
1197 |
-
# Default responses
|
1198 |
-
else:
|
1199 |
-
responses = [
|
1200 |
-
f"*{monster.name} makes a happy sound* 😊",
|
1201 |
-
f"{monster.name} is listening! 👂",
|
1202 |
-
f"*{monster.name} nods understandingly* 🙂",
|
1203 |
-
f"Tell {monster.name} more! 💬"
|
1204 |
-
]
|
1205 |
-
|
1206 |
-
return random.choice(responses)
|
1207 |
-
|
1208 |
-
def launch(self, **kwargs):
|
1209 |
-
"""Launch the Gradio interface with optimized settings"""
|
1210 |
-
loop = asyncio.new_event_loop()
|
1211 |
-
asyncio.set_event_loop(loop)
|
1212 |
-
|
1213 |
-
# Initialize async components
|
1214 |
-
loop.run_until_complete(self.initialize())
|
1215 |
-
|
1216 |
-
# Create interface
|
1217 |
-
interface = self.create_interface()
|
1218 |
-
|
1219 |
-
# Launch with production settings
|
1220 |
-
launch_config = {
|
1221 |
-
"server_name": "0.0.0.0",
|
1222 |
-
"server_port": 7860,
|
1223 |
-
"share": False,
|
1224 |
-
"debug": False,
|
1225 |
-
"show_error": True,
|
1226 |
-
"quiet": False,
|
1227 |
-
"favicon_path": None,
|
1228 |
-
"ssl_keyfile": None,
|
1229 |
-
"ssl_certfile": None,
|
1230 |
-
"ssl_keyfile_password": None,
|
1231 |
-
"max_threads": 40,
|
1232 |
-
**kwargs
|
1233 |
-
}
|
1234 |
-
|
1235 |
-
self.logger.info("Launching DigiPal interface...")
|
1236 |
-
return interface.launch(**launch_config)
|
1237 |
-
|
1238 |
-
# ZeroGPU wrapper functions for CPU-intensive operations
|
1239 |
-
def safe_create_monster(interface, name: str, personality: str):
|
1240 |
-
"""Safe wrapper for monster creation"""
|
1241 |
-
import asyncio
|
1242 |
-
return asyncio.run(interface.create_new_monster(name, personality))
|
1243 |
-
|
1244 |
-
def safe_handle_conversation(interface, monster_state: Dict, message: str, conversation_history: List):
|
1245 |
-
"""Safe wrapper for conversation handling"""
|
1246 |
-
import asyncio
|
1247 |
-
return asyncio.run(interface.handle_text_conversation(monster_state, message, conversation_history))
|
1248 |
-
|
1249 |
-
def safe_handle_voice(interface, monster_state: Dict, audio_data, conversation_history: List):
|
1250 |
-
"""Safe wrapper for voice input handling"""
|
1251 |
-
import asyncio
|
1252 |
-
return asyncio.run(interface.handle_voice_input(monster_state, audio_data, conversation_history))
|
1253 |
-
|
1254 |
-
# Apply GPU decorators only in Spaces environment for specific operations
|
1255 |
-
try:
|
1256 |
-
import os
|
1257 |
-
if os.getenv("SPACE_ID") is not None:
|
1258 |
-
# Only decorate the most GPU-intensive operations
|
1259 |
-
safe_handle_conversation = spaces.GPU(duration=120)(safe_handle_conversation)
|
1260 |
-
safe_handle_voice = spaces.GPU(duration=120)(safe_handle_voice)
|
1261 |
-
except (ImportError, NotImplementedError, AttributeError) as e:
|
1262 |
-
# GPU decorator not available or failed, continue without it
|
1263 |
-
pass
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
src/ui/streamlit_interface.py
ADDED
@@ -0,0 +1,565 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
"""
|
2 |
+
Streamlit Interface for DigiPal - Modern AI Monster Companion
|
3 |
+
Replaces Gradio with Streamlit for better user experience
|
4 |
+
"""
|
5 |
+
|
6 |
+
import streamlit as st
|
7 |
+
import asyncio
|
8 |
+
import logging
|
9 |
+
import json
|
10 |
+
import time
|
11 |
+
import requests
|
12 |
+
import io
|
13 |
+
from typing import Dict, List, Optional, Any, Tuple
|
14 |
+
from datetime import datetime, timedelta
|
15 |
+
import numpy as np
|
16 |
+
from PIL import Image
|
17 |
+
import threading
|
18 |
+
|
19 |
+
# Configure logging
|
20 |
+
logging.basicConfig(level=logging.INFO)
|
21 |
+
logger = logging.getLogger(__name__)
|
22 |
+
|
23 |
+
# API Configuration
|
24 |
+
API_BASE_URL = "http://localhost:7861" # FastAPI backend
|
25 |
+
|
26 |
+
class StreamlitDigiPalInterface:
|
27 |
+
"""Modern Streamlit interface for DigiPal"""
|
28 |
+
|
29 |
+
def __init__(self):
|
30 |
+
self.logger = logging.getLogger(__name__)
|
31 |
+
|
32 |
+
# Initialize session state
|
33 |
+
if 'current_monster' not in st.session_state:
|
34 |
+
st.session_state.current_monster = None
|
35 |
+
if 'monster_stats' not in st.session_state:
|
36 |
+
st.session_state.monster_stats = {}
|
37 |
+
if 'conversation_history' not in st.session_state:
|
38 |
+
st.session_state.conversation_history = []
|
39 |
+
if 'available_monsters' not in st.session_state:
|
40 |
+
st.session_state.available_monsters = []
|
41 |
+
|
42 |
+
def run(self):
|
43 |
+
"""Main Streamlit application"""
|
44 |
+
|
45 |
+
# Page configuration
|
46 |
+
st.set_page_config(
|
47 |
+
page_title="DigiPal - AI Monster Companion",
|
48 |
+
page_icon="🐉",
|
49 |
+
layout="wide",
|
50 |
+
initial_sidebar_state="expanded"
|
51 |
+
)
|
52 |
+
|
53 |
+
# Custom CSS for cyberpunk theme
|
54 |
+
self._apply_custom_css()
|
55 |
+
|
56 |
+
# Header with cyberpunk styling
|
57 |
+
st.markdown('<h1 class="digipal-title">🐉 DigiPal</h1>', unsafe_allow_html=True)
|
58 |
+
st.markdown('<p style="text-align: center; font-family: Rajdhani, sans-serif; font-size: 1.2rem; color: #00ffff; margin-top: -1rem;">Advanced AI Monster Companion with 3D Generation</p>', unsafe_allow_html=True)
|
59 |
+
|
60 |
+
# Sidebar for monster management
|
61 |
+
self._render_sidebar()
|
62 |
+
|
63 |
+
# Main content area
|
64 |
+
if st.session_state.current_monster:
|
65 |
+
self._render_monster_interface()
|
66 |
+
else:
|
67 |
+
self._render_welcome_screen()
|
68 |
+
|
69 |
+
def _apply_custom_css(self):
|
70 |
+
"""Apply custom CSS for cyberpunk theme"""
|
71 |
+
st.markdown("""
|
72 |
+
<style>
|
73 |
+
/* Import Google Fonts */
|
74 |
+
@import url('https://fonts.googleapis.com/css2?family=Orbitron:wght@400;700;900&family=Rajdhani:wght@300;400;500;600;700&display=swap');
|
75 |
+
|
76 |
+
/* Main background */
|
77 |
+
.main {
|
78 |
+
background: linear-gradient(135deg, #0a0a0a 0%, #1a0d2e 25%, #16213e 50%, #0f3460 75%, #0e4b99 100%);
|
79 |
+
color: #e0e0e0;
|
80 |
+
font-family: 'Rajdhani', sans-serif;
|
81 |
+
}
|
82 |
+
|
83 |
+
/* Headers */
|
84 |
+
h1, h2, h3 {
|
85 |
+
font-family: 'Orbitron', monospace;
|
86 |
+
color: #00ffff;
|
87 |
+
text-shadow: 0 0 20px #00ffff40;
|
88 |
+
}
|
89 |
+
|
90 |
+
/* Sidebar */
|
91 |
+
.css-1d391kg {
|
92 |
+
background: linear-gradient(180deg, #1a1a2e 0%, #16213e 100%);
|
93 |
+
border-right: 2px solid #00ffff40;
|
94 |
+
}
|
95 |
+
|
96 |
+
/* Buttons */
|
97 |
+
.stButton > button {
|
98 |
+
background: linear-gradient(45deg, #ff0080, #00ffff);
|
99 |
+
color: #0a0a0a;
|
100 |
+
border: none;
|
101 |
+
border-radius: 25px;
|
102 |
+
padding: 0.75rem 1.5rem;
|
103 |
+
font-weight: bold;
|
104 |
+
font-family: 'Orbitron', monospace;
|
105 |
+
font-size: 0.9rem;
|
106 |
+
box-shadow: 0 0 20px rgba(255, 0, 128, 0.3);
|
107 |
+
transition: all 0.3s ease;
|
108 |
+
text-transform: uppercase;
|
109 |
+
letter-spacing: 1px;
|
110 |
+
}
|
111 |
+
.stButton > button:hover {
|
112 |
+
transform: translateY(-3px) scale(1.05);
|
113 |
+
box-shadow: 0 0 30px rgba(0, 255, 255, 0.6);
|
114 |
+
background: linear-gradient(45deg, #00ffff, #ff0080);
|
115 |
+
}
|
116 |
+
|
117 |
+
/* Input fields */
|
118 |
+
.stTextInput > div > div > input, .stTextArea > div > div > textarea, .stSelectbox > div > div > select {
|
119 |
+
background: rgba(0, 0, 0, 0.8);
|
120 |
+
border: 2px solid #00ffff40;
|
121 |
+
border-radius: 10px;
|
122 |
+
color: #e0e0e0;
|
123 |
+
font-family: 'Rajdhani', sans-serif;
|
124 |
+
}
|
125 |
+
.stTextInput > div > div > input:focus, .stTextArea > div > div > textarea:focus {
|
126 |
+
border-color: #00ffff;
|
127 |
+
box-shadow: 0 0 15px #00ffff40;
|
128 |
+
}
|
129 |
+
|
130 |
+
/* Monster stats container */
|
131 |
+
.monster-stats {
|
132 |
+
background: linear-gradient(135deg, rgba(0, 255, 255, 0.1) 0%, rgba(255, 0, 128, 0.1) 100%);
|
133 |
+
border-radius: 15px;
|
134 |
+
padding: 1.5rem;
|
135 |
+
backdrop-filter: blur(10px);
|
136 |
+
border: 2px solid rgba(0, 255, 255, 0.3);
|
137 |
+
box-shadow: 0 0 30px rgba(0, 255, 255, 0.2);
|
138 |
+
}
|
139 |
+
|
140 |
+
/* Progress bars */
|
141 |
+
.stProgress > div > div > div {
|
142 |
+
background: linear-gradient(90deg, #ff0080 0%, #00ffff 100%);
|
143 |
+
}
|
144 |
+
|
145 |
+
/* Metrics */
|
146 |
+
[data-testid="metric-container"] {
|
147 |
+
background: rgba(0, 0, 0, 0.6);
|
148 |
+
border: 1px solid #00ffff40;
|
149 |
+
border-radius: 10px;
|
150 |
+
padding: 1rem;
|
151 |
+
box-shadow: 0 0 15px rgba(0, 255, 255, 0.1);
|
152 |
+
}
|
153 |
+
|
154 |
+
/* Chat messages */
|
155 |
+
.stChatMessage {
|
156 |
+
background: rgba(0, 0, 0, 0.7);
|
157 |
+
border-radius: 15px;
|
158 |
+
border-left: 4px solid #00ffff;
|
159 |
+
margin: 0.5rem 0;
|
160 |
+
}
|
161 |
+
|
162 |
+
/* Success/Error messages */
|
163 |
+
.stSuccess {
|
164 |
+
background: rgba(0, 255, 0, 0.1);
|
165 |
+
border: 1px solid #00ff00;
|
166 |
+
color: #00ff00;
|
167 |
+
}
|
168 |
+
.stError {
|
169 |
+
background: rgba(255, 0, 0, 0.1);
|
170 |
+
border: 1px solid #ff0000;
|
171 |
+
color: #ff6666;
|
172 |
+
}
|
173 |
+
|
174 |
+
/* Neon text */
|
175 |
+
.neon-text {
|
176 |
+
color: #00ffff;
|
177 |
+
text-shadow: 0 0 10px #00ffff, 0 0 20px #00ffff, 0 0 30px #00ffff;
|
178 |
+
font-family: 'Orbitron', monospace;
|
179 |
+
font-weight: 700;
|
180 |
+
}
|
181 |
+
|
182 |
+
/* DigiPal title effect */
|
183 |
+
.digipal-title {
|
184 |
+
background: linear-gradient(45deg, #ff0080, #00ffff, #ff0080);
|
185 |
+
background-size: 200% 200%;
|
186 |
+
-webkit-background-clip: text;
|
187 |
+
-webkit-text-fill-color: transparent;
|
188 |
+
animation: neon-glow 2s ease-in-out infinite alternate;
|
189 |
+
font-family: 'Orbitron', monospace;
|
190 |
+
font-weight: 900;
|
191 |
+
font-size: 3rem;
|
192 |
+
text-align: center;
|
193 |
+
margin: 1rem 0;
|
194 |
+
}
|
195 |
+
|
196 |
+
@keyframes neon-glow {
|
197 |
+
from { background-position: 0% 50%; }
|
198 |
+
to { background-position: 100% 50%; }
|
199 |
+
}
|
200 |
+
|
201 |
+
/* Holographic effect for containers */
|
202 |
+
.holo-container {
|
203 |
+
background: linear-gradient(135deg,
|
204 |
+
rgba(0, 255, 255, 0.1) 0%,
|
205 |
+
rgba(0, 255, 255, 0.05) 25%,
|
206 |
+
rgba(255, 0, 128, 0.05) 50%,
|
207 |
+
rgba(255, 0, 128, 0.1) 75%,
|
208 |
+
rgba(0, 255, 255, 0.1) 100%);
|
209 |
+
border: 2px solid;
|
210 |
+
border-image: linear-gradient(45deg, #00ffff, #ff0080, #00ffff) 1;
|
211 |
+
border-radius: 15px;
|
212 |
+
padding: 1.5rem;
|
213 |
+
backdrop-filter: blur(15px);
|
214 |
+
box-shadow:
|
215 |
+
0 0 20px rgba(0, 255, 255, 0.3),
|
216 |
+
inset 0 0 20px rgba(255, 0, 128, 0.1);
|
217 |
+
}
|
218 |
+
</style>
|
219 |
+
""", unsafe_allow_html=True)
|
220 |
+
|
221 |
+
def _render_sidebar(self):
|
222 |
+
"""Render sidebar with monster management"""
|
223 |
+
with st.sidebar:
|
224 |
+
st.header("🎮 Monster Management")
|
225 |
+
|
226 |
+
# Load available monsters
|
227 |
+
if st.button("🔄 Refresh Monsters"):
|
228 |
+
self._load_available_monsters()
|
229 |
+
|
230 |
+
# Monster selection
|
231 |
+
if st.session_state.available_monsters:
|
232 |
+
selected_monster = st.selectbox(
|
233 |
+
"Select Monster:",
|
234 |
+
options=["None"] + [m["name"] for m in st.session_state.available_monsters],
|
235 |
+
index=0
|
236 |
+
)
|
237 |
+
|
238 |
+
if selected_monster != "None":
|
239 |
+
if st.button("🐾 Load Monster"):
|
240 |
+
self._load_monster(selected_monster)
|
241 |
+
|
242 |
+
# Create new monster
|
243 |
+
st.subheader("🆕 Create New Monster")
|
244 |
+
with st.form("create_monster"):
|
245 |
+
new_name = st.text_input("Monster Name:")
|
246 |
+
personality = st.selectbox(
|
247 |
+
"Personality:",
|
248 |
+
["FRIENDLY", "ENERGETIC", "CALM", "CURIOUS", "BRAVE"]
|
249 |
+
)
|
250 |
+
|
251 |
+
if st.form_submit_button("🥚 Create Monster"):
|
252 |
+
self._create_monster(new_name, personality)
|
253 |
+
|
254 |
+
# Current monster info
|
255 |
+
if st.session_state.current_monster:
|
256 |
+
st.subheader(f"🐉 {st.session_state.current_monster['name']}")
|
257 |
+
st.write(f"**Stage:** {st.session_state.current_monster.get('stage', 'Unknown')}")
|
258 |
+
st.write(f"**Personality:** {st.session_state.current_monster.get('personality', 'Unknown')}")
|
259 |
+
|
260 |
+
def _render_welcome_screen(self):
|
261 |
+
"""Render welcome screen when no monster is selected"""
|
262 |
+
col1, col2 = st.columns([2, 1])
|
263 |
+
|
264 |
+
with col1:
|
265 |
+
st.markdown("""
|
266 |
+
<div class="holo-container">
|
267 |
+
<h2 class="neon-text">Welcome to DigiPal! 🐉</h2>
|
268 |
+
|
269 |
+
<p style="font-size: 1.3rem; color: #e0e0e0; margin-bottom: 1.5rem;">
|
270 |
+
<strong>The most advanced AI monster companion experience</strong>
|
271 |
+
</p>
|
272 |
+
|
273 |
+
<h3 style="color: #ff0080;">🚀 Revolutionary Features:</h3>
|
274 |
+
<ul style="font-size: 1.1rem; line-height: 1.8;">
|
275 |
+
<li>🤖 <strong style="color: #00ffff;">Advanced AI Conversations</strong> with Qwen 2.5</li>
|
276 |
+
<li>🎤 <strong style="color: #00ffff;">Voice Interaction</strong> with Kyutai STT-2.6b</li>
|
277 |
+
<li>🎨 <strong style="color: #00ffff;">3D Model Generation</strong> with OmniGen2 → Hunyuan3D → UniRig</li>
|
278 |
+
<li>📊 <strong style="color: #00ffff;">Complex Care System</strong> inspired by Digimon World</li>
|
279 |
+
<li>🧬 <strong style="color: #00ffff;">Dynamic Evolution</strong> based on care quality</li>
|
280 |
+
<li>💬 <strong style="color: #00ffff;">Personality-driven Responses</strong></li>
|
281 |
+
</ul>
|
282 |
+
|
283 |
+
<h3 style="color: #ff0080; margin-top: 2rem;">⚡ Getting Started:</h3>
|
284 |
+
<ol style="font-size: 1.1rem; line-height: 1.8;">
|
285 |
+
<li><span style="color: #00ffff;">Create a new monster</span> in the sidebar</li>
|
286 |
+
<li><span style="color: #00ffff;">Talk to your monster</span> and watch it grow</li>
|
287 |
+
<li><span style="color: #00ffff;">Generate a unique 3D model</span></li>
|
288 |
+
<li><span style="color: #00ffff;">Care for your digital companion!</span></li>
|
289 |
+
</ol>
|
290 |
+
</div>
|
291 |
+
""", unsafe_allow_html=True)
|
292 |
+
|
293 |
+
with col2:
|
294 |
+
st.markdown("""
|
295 |
+
<div class="holo-container" style="text-align: center;">
|
296 |
+
<h3 class="neon-text">🔮 Your AI Companion Awaits</h3>
|
297 |
+
<div style="background: linear-gradient(45deg, #ff0080, #00ffff);
|
298 |
+
border-radius: 20px;
|
299 |
+
padding: 2rem;
|
300 |
+
margin: 1rem 0;
|
301 |
+
box-shadow: 0 0 30px rgba(0, 255, 255, 0.5);">
|
302 |
+
<p style="font-size: 4rem; margin: 0; animation: neon-glow 2s ease-in-out infinite alternate;">🐉</p>
|
303 |
+
<p style="font-size: 1.2rem; margin: 0.5rem 0; color: #0a0a0a; font-weight: bold;">DigiPal</p>
|
304 |
+
</div>
|
305 |
+
<p style="color: #e0e0e0; font-style: italic;">Ready to create your perfect digital companion?</p>
|
306 |
+
</div>
|
307 |
+
""", unsafe_allow_html=True)
|
308 |
+
|
309 |
+
def _render_monster_interface(self):
|
310 |
+
"""Render main monster interaction interface"""
|
311 |
+
monster = st.session_state.current_monster
|
312 |
+
|
313 |
+
# Main layout
|
314 |
+
col1, col2 = st.columns([2, 1])
|
315 |
+
|
316 |
+
with col1:
|
317 |
+
# Conversation area
|
318 |
+
self._render_conversation_area()
|
319 |
+
|
320 |
+
# Action buttons
|
321 |
+
self._render_action_buttons()
|
322 |
+
|
323 |
+
with col2:
|
324 |
+
# Monster stats and 3D model
|
325 |
+
self._render_monster_stats()
|
326 |
+
self._render_3d_model_section()
|
327 |
+
|
328 |
+
def _render_conversation_area(self):
|
329 |
+
"""Render conversation interface"""
|
330 |
+
st.subheader("💬 Talk to Your Monster")
|
331 |
+
|
332 |
+
# Chat history
|
333 |
+
chat_container = st.container()
|
334 |
+
with chat_container:
|
335 |
+
for message in st.session_state.conversation_history:
|
336 |
+
if message["role"] == "user":
|
337 |
+
st.chat_message("user").write(message["content"])
|
338 |
+
else:
|
339 |
+
st.chat_message("assistant").write(message["content"])
|
340 |
+
|
341 |
+
# Chat input
|
342 |
+
user_input = st.chat_input("Say something to your monster...")
|
343 |
+
|
344 |
+
if user_input:
|
345 |
+
self._send_message(user_input)
|
346 |
+
|
347 |
+
def _render_action_buttons(self):
|
348 |
+
"""Render care action buttons"""
|
349 |
+
st.subheader("🎮 Care Actions")
|
350 |
+
|
351 |
+
col1, col2, col3 = st.columns(3)
|
352 |
+
|
353 |
+
with col1:
|
354 |
+
if st.button("🍖 Feed"):
|
355 |
+
self._perform_action("feed")
|
356 |
+
if st.button("🏃 Train"):
|
357 |
+
self._perform_action("train")
|
358 |
+
|
359 |
+
with col2:
|
360 |
+
if st.button("🎲 Play"):
|
361 |
+
self._perform_action("play")
|
362 |
+
if st.button("🧼 Clean"):
|
363 |
+
self._perform_action("clean")
|
364 |
+
|
365 |
+
with col3:
|
366 |
+
if st.button("💊 Heal"):
|
367 |
+
self._perform_action("heal")
|
368 |
+
if st.button("😴 Rest"):
|
369 |
+
self._perform_action("rest")
|
370 |
+
|
371 |
+
def _render_monster_stats(self):
|
372 |
+
"""Render monster statistics"""
|
373 |
+
st.subheader("📊 Monster Stats")
|
374 |
+
|
375 |
+
if 'stats' in st.session_state.current_monster:
|
376 |
+
stats = st.session_state.current_monster['stats']
|
377 |
+
|
378 |
+
# Create visual stat bars
|
379 |
+
for stat_name, value in stats.items():
|
380 |
+
if isinstance(value, (int, float)):
|
381 |
+
# Normalize to 0-100 for progress bar
|
382 |
+
normalized_value = min(100, max(0, value))
|
383 |
+
st.metric(
|
384 |
+
label=stat_name.title(),
|
385 |
+
value=f"{value:.1f}",
|
386 |
+
delta=None
|
387 |
+
)
|
388 |
+
st.progress(normalized_value / 100)
|
389 |
+
else:
|
390 |
+
st.info("Load a monster to see stats")
|
391 |
+
|
392 |
+
def _render_3d_model_section(self):
|
393 |
+
"""Render 3D model generation section"""
|
394 |
+
st.subheader("🎨 3D Model Generation")
|
395 |
+
|
396 |
+
# Model display area
|
397 |
+
if st.session_state.current_monster and st.session_state.current_monster.get('model_url'):
|
398 |
+
st.success("3D Model Ready!")
|
399 |
+
st.write(f"Model: {st.session_state.current_monster['model_url']}")
|
400 |
+
else:
|
401 |
+
st.info("No 3D model generated yet")
|
402 |
+
|
403 |
+
# Generation controls
|
404 |
+
with st.form("generate_3d"):
|
405 |
+
description = st.text_area(
|
406 |
+
"Custom Description (optional):",
|
407 |
+
placeholder="A cute dragon with blue scales and friendly eyes..."
|
408 |
+
)
|
409 |
+
|
410 |
+
if st.form_submit_button("🎨 Generate 3D Model"):
|
411 |
+
self._generate_3d_model(description)
|
412 |
+
|
413 |
+
def _load_available_monsters(self):
|
414 |
+
"""Load list of available monsters from API"""
|
415 |
+
try:
|
416 |
+
response = requests.get(f"{API_BASE_URL}/api/monsters", timeout=5)
|
417 |
+
if response.status_code == 200:
|
418 |
+
data = response.json()
|
419 |
+
st.session_state.available_monsters = data.get("monsters", [])
|
420 |
+
st.success(f"Found {len(st.session_state.available_monsters)} monsters")
|
421 |
+
else:
|
422 |
+
st.error("Failed to load monsters")
|
423 |
+
except requests.exceptions.RequestException as e:
|
424 |
+
st.warning("🔧 Backend not connected")
|
425 |
+
st.info("💡 To enable full functionality, start backend: `python app.py`")
|
426 |
+
# Add some demo monsters for UI preview
|
427 |
+
st.session_state.available_monsters = [
|
428 |
+
{"name": "Demo Dragon", "id": "demo1", "stage": "Adult"},
|
429 |
+
{"name": "Cyber Wolf", "id": "demo2", "stage": "Champion"}
|
430 |
+
]
|
431 |
+
|
432 |
+
def _load_monster(self, monster_name: str):
|
433 |
+
"""Load a specific monster"""
|
434 |
+
try:
|
435 |
+
# Find monster by name
|
436 |
+
monster_data = None
|
437 |
+
for monster in st.session_state.available_monsters:
|
438 |
+
if monster["name"] == monster_name:
|
439 |
+
monster_data = monster
|
440 |
+
break
|
441 |
+
|
442 |
+
if monster_data:
|
443 |
+
# Load full monster data from API
|
444 |
+
response = requests.get(f"{API_BASE_URL}/api/monsters/{monster_data['id']}")
|
445 |
+
if response.status_code == 200:
|
446 |
+
st.session_state.current_monster = response.json()
|
447 |
+
st.session_state.conversation_history = st.session_state.current_monster.get('conversation_history', [])
|
448 |
+
st.success(f"Loaded {monster_name}!")
|
449 |
+
st.rerun()
|
450 |
+
else:
|
451 |
+
st.error("Failed to load monster details")
|
452 |
+
except Exception as e:
|
453 |
+
st.error(f"Error loading monster: {str(e)}")
|
454 |
+
|
455 |
+
def _create_monster(self, name: str, personality: str):
|
456 |
+
"""Create a new monster"""
|
457 |
+
if not name:
|
458 |
+
st.error("Please enter a monster name")
|
459 |
+
return
|
460 |
+
|
461 |
+
try:
|
462 |
+
response = requests.post(
|
463 |
+
f"{API_BASE_URL}/api/monsters",
|
464 |
+
json={"name": name, "personality": personality}
|
465 |
+
)
|
466 |
+
if response.status_code == 200:
|
467 |
+
monster_data = response.json()
|
468 |
+
st.session_state.current_monster = monster_data
|
469 |
+
st.session_state.conversation_history = []
|
470 |
+
st.success(f"Created {name}!")
|
471 |
+
st.rerun()
|
472 |
+
else:
|
473 |
+
st.error("Failed to create monster")
|
474 |
+
except Exception as e:
|
475 |
+
st.error(f"Error creating monster: {str(e)}")
|
476 |
+
|
477 |
+
def _send_message(self, message: str):
|
478 |
+
"""Send message to monster"""
|
479 |
+
if not st.session_state.current_monster:
|
480 |
+
return
|
481 |
+
|
482 |
+
try:
|
483 |
+
# Add user message to history
|
484 |
+
st.session_state.conversation_history.append({
|
485 |
+
"role": "user",
|
486 |
+
"content": message,
|
487 |
+
"timestamp": datetime.now().isoformat()
|
488 |
+
})
|
489 |
+
|
490 |
+
# Send to API
|
491 |
+
response = requests.post(
|
492 |
+
f"{API_BASE_URL}/api/monsters/{st.session_state.current_monster['id']}/talk",
|
493 |
+
json={"message": message}
|
494 |
+
)
|
495 |
+
|
496 |
+
if response.status_code == 200:
|
497 |
+
data = response.json()
|
498 |
+
# Add AI response to history
|
499 |
+
st.session_state.conversation_history.append({
|
500 |
+
"role": "assistant",
|
501 |
+
"content": data["response"],
|
502 |
+
"timestamp": datetime.now().isoformat()
|
503 |
+
})
|
504 |
+
|
505 |
+
# Update monster stats
|
506 |
+
st.session_state.current_monster['stats'] = data.get("stats", {})
|
507 |
+
st.rerun()
|
508 |
+
else:
|
509 |
+
st.error("Failed to send message")
|
510 |
+
except Exception as e:
|
511 |
+
st.error(f"Error sending message: {str(e)}")
|
512 |
+
|
513 |
+
def _perform_action(self, action: str):
|
514 |
+
"""Perform care action on monster"""
|
515 |
+
if not st.session_state.current_monster:
|
516 |
+
return
|
517 |
+
|
518 |
+
try:
|
519 |
+
response = requests.post(
|
520 |
+
f"{API_BASE_URL}/api/monsters/{st.session_state.current_monster['id']}/action",
|
521 |
+
json={"action": action}
|
522 |
+
)
|
523 |
+
|
524 |
+
if response.status_code == 200:
|
525 |
+
data = response.json()
|
526 |
+
st.session_state.current_monster['stats'] = data.get("stats", {})
|
527 |
+
st.success(f"Performed {action}!")
|
528 |
+
st.rerun()
|
529 |
+
else:
|
530 |
+
st.error(f"Failed to perform {action}")
|
531 |
+
except Exception as e:
|
532 |
+
st.error(f"Error performing {action}: {str(e)}")
|
533 |
+
|
534 |
+
def _generate_3d_model(self, description: str = ""):
|
535 |
+
"""Generate 3D model for monster"""
|
536 |
+
if not st.session_state.current_monster:
|
537 |
+
return
|
538 |
+
|
539 |
+
try:
|
540 |
+
with st.spinner("Generating 3D model... This may take a few minutes."):
|
541 |
+
response = requests.post(
|
542 |
+
f"{API_BASE_URL}/api/monsters/{st.session_state.current_monster['id']}/generate-3d",
|
543 |
+
json={"description": description}
|
544 |
+
)
|
545 |
+
|
546 |
+
if response.status_code == 200:
|
547 |
+
data = response.json()
|
548 |
+
if data["success"]:
|
549 |
+
st.session_state.current_monster['model_url'] = data["model_url"]
|
550 |
+
st.success("3D model generated successfully!")
|
551 |
+
st.rerun()
|
552 |
+
else:
|
553 |
+
st.error("3D generation failed")
|
554 |
+
else:
|
555 |
+
st.error("Failed to generate 3D model")
|
556 |
+
except Exception as e:
|
557 |
+
st.error(f"Error generating 3D model: {str(e)}")
|
558 |
+
|
559 |
+
def main():
|
560 |
+
"""Main entry point for Streamlit app"""
|
561 |
+
interface = StreamlitDigiPalInterface()
|
562 |
+
interface.run()
|
563 |
+
|
564 |
+
if __name__ == "__main__":
|
565 |
+
main()
|
streamlit_app.py
ADDED
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/usr/bin/env python3
|
2 |
+
"""
|
3 |
+
DigiPal Streamlit App - HuggingFace Spaces Entry Point
|
4 |
+
Unified Streamlit application that includes embedded FastAPI functionality
|
5 |
+
"""
|
6 |
+
|
7 |
+
import streamlit as st
|
8 |
+
import asyncio
|
9 |
+
import threading
|
10 |
+
import time
|
11 |
+
import logging
|
12 |
+
import sys
|
13 |
+
import os
|
14 |
+
from pathlib import Path
|
15 |
+
|
16 |
+
# Add src to path
|
17 |
+
sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'src'))
|
18 |
+
|
19 |
+
# Configure logging
|
20 |
+
logging.basicConfig(level=logging.INFO)
|
21 |
+
logger = logging.getLogger(__name__)
|
22 |
+
|
23 |
+
# Import our Streamlit interface
|
24 |
+
from src.ui.streamlit_interface import StreamlitDigiPalInterface
|
25 |
+
|
26 |
+
def start_background_services():
|
27 |
+
"""Start background services needed for DigiPal"""
|
28 |
+
try:
|
29 |
+
# Create necessary directories
|
30 |
+
os.makedirs("data/saves", exist_ok=True)
|
31 |
+
os.makedirs("data/models", exist_ok=True)
|
32 |
+
os.makedirs("data/cache", exist_ok=True)
|
33 |
+
os.makedirs("logs", exist_ok=True)
|
34 |
+
|
35 |
+
# For Spaces deployment, we'll run a simplified version
|
36 |
+
# that doesn't require separate FastAPI server
|
37 |
+
logger.info("DigiPal background services initialized")
|
38 |
+
|
39 |
+
except Exception as e:
|
40 |
+
logger.error(f"Failed to initialize background services: {e}")
|
41 |
+
|
42 |
+
def main():
|
43 |
+
"""Main Streamlit application entry point"""
|
44 |
+
|
45 |
+
# Initialize background services
|
46 |
+
if 'services_initialized' not in st.session_state:
|
47 |
+
start_background_services()
|
48 |
+
st.session_state.services_initialized = True
|
49 |
+
|
50 |
+
# Create and run the interface
|
51 |
+
interface = StreamlitDigiPalInterface()
|
52 |
+
interface.run()
|
53 |
+
|
54 |
+
if __name__ == "__main__":
|
55 |
+
main()
|
test_ui.py
ADDED
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/usr/bin/env python3
|
2 |
+
"""
|
3 |
+
Quick UI Test Script
|
4 |
+
Run this to see the new Streamlit UI without the full backend
|
5 |
+
"""
|
6 |
+
|
7 |
+
import subprocess
|
8 |
+
import sys
|
9 |
+
import os
|
10 |
+
import logging
|
11 |
+
|
12 |
+
# Configure logging
|
13 |
+
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
14 |
+
logger = logging.getLogger(__name__)
|
15 |
+
|
16 |
+
def main():
|
17 |
+
"""Test the Streamlit UI"""
|
18 |
+
logger.info("🎨 Testing DigiPal Streamlit UI")
|
19 |
+
logger.info("=" * 50)
|
20 |
+
logger.info("This will show you the new UI interface")
|
21 |
+
logger.info("Note: Backend features won't work without running the API")
|
22 |
+
logger.info("=" * 50)
|
23 |
+
|
24 |
+
# Create necessary directories
|
25 |
+
os.makedirs("data/saves", exist_ok=True)
|
26 |
+
os.makedirs("data/models", exist_ok=True)
|
27 |
+
os.makedirs("data/cache", exist_ok=True)
|
28 |
+
os.makedirs("logs", exist_ok=True)
|
29 |
+
|
30 |
+
try:
|
31 |
+
port = os.getenv("STREAMLIT_PORT", "8501")
|
32 |
+
logger.info(f"Starting Streamlit UI on port {port}...")
|
33 |
+
logger.info(f"Open your browser to: http://localhost:{port}")
|
34 |
+
|
35 |
+
subprocess.run([
|
36 |
+
sys.executable, "-m", "streamlit", "run",
|
37 |
+
"streamlit_app.py",
|
38 |
+
"--server.port", port,
|
39 |
+
"--server.address", "0.0.0.0",
|
40 |
+
"--server.headless", "false"
|
41 |
+
], check=True)
|
42 |
+
|
43 |
+
except subprocess.CalledProcessError as e:
|
44 |
+
logger.error(f"Failed to start Streamlit: {e}")
|
45 |
+
logger.info("Make sure you have streamlit installed: pip install streamlit")
|
46 |
+
except KeyboardInterrupt:
|
47 |
+
logger.info("UI test stopped")
|
48 |
+
|
49 |
+
if __name__ == "__main__":
|
50 |
+
main()
|