Spaces:
Sleeping
Sleeping
Create app.py
Browse files
app.py
ADDED
@@ -0,0 +1,706 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/usr/bin/env python3
|
2 |
+
"""
|
3 |
+
Hugging Face Space: GGUF Model Converter
|
4 |
+
A web interface for converting Hugging Face models to GGUF format
|
5 |
+
|
6 |
+
This Space provides:
|
7 |
+
1. Web interface for model conversion
|
8 |
+
2. Progress tracking and logging
|
9 |
+
3. Automatic upload to Hugging Face
|
10 |
+
4. Resource monitoring
|
11 |
+
"""
|
12 |
+
|
13 |
+
import os
|
14 |
+
import sys
|
15 |
+
import subprocess
|
16 |
+
import shutil
|
17 |
+
import logging
|
18 |
+
import tempfile
|
19 |
+
import threading
|
20 |
+
import queue
|
21 |
+
import time
|
22 |
+
import psutil
|
23 |
+
import gc
|
24 |
+
from pathlib import Path
|
25 |
+
from typing import Optional, List, Dict, Any
|
26 |
+
from datetime import datetime
|
27 |
+
|
28 |
+
import gradio as gr
|
29 |
+
import torch
|
30 |
+
|
31 |
+
# Try importing required packages
|
32 |
+
try:
|
33 |
+
from huggingface_hub import HfApi, login, create_repo, snapshot_download
|
34 |
+
from transformers import AutoConfig, AutoTokenizer
|
35 |
+
HF_HUB_AVAILABLE = True
|
36 |
+
except ImportError:
|
37 |
+
HF_HUB_AVAILABLE = False
|
38 |
+
|
39 |
+
# Set up logging
|
40 |
+
logging.basicConfig(
|
41 |
+
level=logging.INFO,
|
42 |
+
format='%(asctime)s - %(levelname)s - %(message)s'
|
43 |
+
)
|
44 |
+
logger = logging.getLogger(__name__)
|
45 |
+
|
46 |
+
# Global variables for progress tracking
|
47 |
+
conversion_progress = queue.Queue()
|
48 |
+
current_status = {"status": "idle", "progress": 0, "message": "Ready"}
|
49 |
+
|
50 |
+
class SpaceGGUFConverter:
|
51 |
+
def __init__(self):
|
52 |
+
"""Initialize the GGUF converter for Hugging Face Spaces"""
|
53 |
+
self.temp_dir = None
|
54 |
+
self.llama_cpp_dir = None
|
55 |
+
self.hf_token = None
|
56 |
+
|
57 |
+
def set_hf_token(self, token: str):
|
58 |
+
"""Set the Hugging Face token"""
|
59 |
+
self.hf_token = token
|
60 |
+
if token:
|
61 |
+
login(token=token)
|
62 |
+
return "β
HF Token set successfully!"
|
63 |
+
return "β Invalid token"
|
64 |
+
|
65 |
+
def update_progress(self, status: str, progress: int, message: str):
|
66 |
+
"""Update the global progress status"""
|
67 |
+
global current_status
|
68 |
+
current_status = {
|
69 |
+
"status": status,
|
70 |
+
"progress": progress,
|
71 |
+
"message": message,
|
72 |
+
"timestamp": datetime.now().strftime("%H:%M:%S")
|
73 |
+
}
|
74 |
+
conversion_progress.put(current_status.copy())
|
75 |
+
|
76 |
+
def check_resources(self) -> Dict[str, Any]:
|
77 |
+
"""Check available system resources"""
|
78 |
+
try:
|
79 |
+
memory = psutil.virtual_memory()
|
80 |
+
disk = psutil.disk_usage('/')
|
81 |
+
|
82 |
+
return {
|
83 |
+
"memory_total": f"{memory.total / (1024**3):.1f} GB",
|
84 |
+
"memory_available": f"{memory.available / (1024**3):.1f} GB",
|
85 |
+
"memory_percent": memory.percent,
|
86 |
+
"disk_total": f"{disk.total / (1024**3):.1f} GB",
|
87 |
+
"disk_free": f"{disk.free / (1024**3):.1f} GB",
|
88 |
+
"disk_percent": disk.percent,
|
89 |
+
"cpu_count": psutil.cpu_count(),
|
90 |
+
"gpu_available": torch.cuda.is_available(),
|
91 |
+
"gpu_memory": f"{torch.cuda.get_device_properties(0).total_memory / (1024**3):.1f} GB" if torch.cuda.is_available() else "N/A"
|
92 |
+
}
|
93 |
+
except Exception as e:
|
94 |
+
return {"error": str(e)}
|
95 |
+
|
96 |
+
def validate_model(self, model_id: str) -> tuple[bool, str]:
|
97 |
+
"""Validate if the model exists and get basic info"""
|
98 |
+
try:
|
99 |
+
if not HF_HUB_AVAILABLE:
|
100 |
+
return False, "β Required packages not available"
|
101 |
+
|
102 |
+
self.update_progress("validating", 10, f"Validating model: {model_id}")
|
103 |
+
|
104 |
+
# Try to get model config
|
105 |
+
config = AutoConfig.from_pretrained(model_id, trust_remote_code=False)
|
106 |
+
|
107 |
+
# Get approximate model size
|
108 |
+
try:
|
109 |
+
api = HfApi()
|
110 |
+
model_info = api.model_info(model_id)
|
111 |
+
|
112 |
+
# Calculate approximate size from number of parameters
|
113 |
+
if hasattr(config, 'num_parameters'):
|
114 |
+
params = config.num_parameters()
|
115 |
+
elif hasattr(config, 'n_params'):
|
116 |
+
params = config.n_params
|
117 |
+
else:
|
118 |
+
# Estimate from model files
|
119 |
+
params = "Unknown"
|
120 |
+
|
121 |
+
estimated_size = f"~{params/1e9:.1f}B parameters" if isinstance(params, (int, float)) else params
|
122 |
+
|
123 |
+
return True, f"β
Valid model found!\nParameters: {estimated_size}\nArchitecture: {config.model_type if hasattr(config, 'model_type') else 'Unknown'}"
|
124 |
+
|
125 |
+
except Exception as e:
|
126 |
+
return True, f"β
Model accessible (size estimation failed: {str(e)})"
|
127 |
+
|
128 |
+
except Exception as e:
|
129 |
+
return False, f"β Model validation failed: {str(e)}"
|
130 |
+
|
131 |
+
def setup_environment(self) -> bool:
|
132 |
+
"""Set up the environment for GGUF conversion"""
|
133 |
+
try:
|
134 |
+
self.update_progress("setup", 20, "Setting up conversion environment...")
|
135 |
+
|
136 |
+
# Create temporary directory
|
137 |
+
self.temp_dir = tempfile.mkdtemp(prefix="gguf_space_")
|
138 |
+
logger.info(f"Created temporary directory: {self.temp_dir}")
|
139 |
+
|
140 |
+
# Clone llama.cpp
|
141 |
+
self.llama_cpp_dir = os.path.join(self.temp_dir, "llama.cpp")
|
142 |
+
self.update_progress("setup", 30, "Downloading llama.cpp...")
|
143 |
+
|
144 |
+
result = subprocess.run([
|
145 |
+
"git", "clone", "--depth", "1",
|
146 |
+
"https://github.com/ggerganov/llama.cpp.git",
|
147 |
+
self.llama_cpp_dir
|
148 |
+
], capture_output=True, text=True)
|
149 |
+
|
150 |
+
if result.returncode != 0:
|
151 |
+
raise Exception(f"Failed to clone llama.cpp: {result.stderr}")
|
152 |
+
|
153 |
+
# Build llama.cpp
|
154 |
+
self.update_progress("setup", 50, "Building llama.cpp (this may take a few minutes)...")
|
155 |
+
|
156 |
+
original_dir = os.getcwd()
|
157 |
+
try:
|
158 |
+
os.chdir(self.llama_cpp_dir)
|
159 |
+
|
160 |
+
# Configure with CMake
|
161 |
+
configure_result = subprocess.run([
|
162 |
+
"cmake", "-S", ".", "-B", "build",
|
163 |
+
"-DCMAKE_BUILD_TYPE=Release",
|
164 |
+
"-DLLAMA_BUILD_TESTS=OFF",
|
165 |
+
"-DLLAMA_BUILD_EXAMPLES=ON"
|
166 |
+
], capture_output=True, text=True)
|
167 |
+
|
168 |
+
if configure_result.returncode != 0:
|
169 |
+
raise Exception(f"CMake configure failed: {configure_result.stderr}")
|
170 |
+
|
171 |
+
# Build
|
172 |
+
build_result = subprocess.run([
|
173 |
+
"cmake", "--build", "build", "--config", "Release", "-j"
|
174 |
+
], capture_output=True, text=True)
|
175 |
+
|
176 |
+
if build_result.returncode != 0:
|
177 |
+
raise Exception(f"CMake build failed: {build_result.stderr}")
|
178 |
+
|
179 |
+
finally:
|
180 |
+
os.chdir(original_dir)
|
181 |
+
|
182 |
+
self.update_progress("setup", 70, "Environment setup complete!")
|
183 |
+
return True
|
184 |
+
|
185 |
+
except Exception as e:
|
186 |
+
self.update_progress("error", 0, f"Setup failed: {str(e)}")
|
187 |
+
logger.error(f"Environment setup failed: {e}")
|
188 |
+
return False
|
189 |
+
|
190 |
+
def convert_model(
|
191 |
+
self,
|
192 |
+
model_id: str,
|
193 |
+
output_repo: str,
|
194 |
+
quantizations: List[str],
|
195 |
+
hf_token: str,
|
196 |
+
private_repo: bool = False
|
197 |
+
) -> tuple[bool, str]:
|
198 |
+
"""Convert model to GGUF format"""
|
199 |
+
try:
|
200 |
+
if not hf_token:
|
201 |
+
return False, "β Hugging Face token is required"
|
202 |
+
|
203 |
+
# Set token
|
204 |
+
self.set_hf_token(hf_token)
|
205 |
+
|
206 |
+
# Validate model first
|
207 |
+
valid, validation_msg = self.validate_model(model_id)
|
208 |
+
if not valid:
|
209 |
+
return False, validation_msg
|
210 |
+
|
211 |
+
# Check resources
|
212 |
+
resources = self.check_resources()
|
213 |
+
if resources.get("memory_percent", 100) > 90:
|
214 |
+
return False, "β Insufficient memory available (>90% used)"
|
215 |
+
|
216 |
+
# Setup environment
|
217 |
+
if not self.setup_environment():
|
218 |
+
return False, "β Failed to setup environment"
|
219 |
+
|
220 |
+
# Download model
|
221 |
+
self.update_progress("downloading", 80, f"Downloading model: {model_id}")
|
222 |
+
model_dir = os.path.join(self.temp_dir, "original_model")
|
223 |
+
|
224 |
+
try:
|
225 |
+
snapshot_download(
|
226 |
+
repo_id=model_id,
|
227 |
+
local_dir=model_dir,
|
228 |
+
token=hf_token
|
229 |
+
)
|
230 |
+
except Exception as e:
|
231 |
+
return False, f"β Failed to download model: {str(e)}"
|
232 |
+
|
233 |
+
# Convert to GGUF
|
234 |
+
self.update_progress("converting", 85, "Converting to GGUF format...")
|
235 |
+
gguf_dir = os.path.join(self.temp_dir, "gguf_output")
|
236 |
+
os.makedirs(gguf_dir, exist_ok=True)
|
237 |
+
|
238 |
+
# Convert to f16 first
|
239 |
+
convert_script = os.path.join(self.llama_cpp_dir, "convert_hf_to_gguf.py")
|
240 |
+
f16_output = os.path.join(gguf_dir, "model-f16.gguf")
|
241 |
+
|
242 |
+
convert_result = subprocess.run([
|
243 |
+
sys.executable, convert_script,
|
244 |
+
model_dir,
|
245 |
+
"--outfile", f16_output,
|
246 |
+
"--outtype", "f16"
|
247 |
+
], capture_output=True, text=True)
|
248 |
+
|
249 |
+
if convert_result.returncode != 0:
|
250 |
+
return False, f"β F16 conversion failed: {convert_result.stderr}"
|
251 |
+
|
252 |
+
# Find quantize binary
|
253 |
+
quantize_binary = self._find_quantize_binary()
|
254 |
+
if not quantize_binary:
|
255 |
+
return False, "β Could not find llama-quantize binary"
|
256 |
+
|
257 |
+
# Create quantizations
|
258 |
+
successful_quants = ["f16"]
|
259 |
+
for i, quant in enumerate(quantizations):
|
260 |
+
if quant == "f16":
|
261 |
+
continue
|
262 |
+
|
263 |
+
progress = 85 + (10 * i / len(quantizations))
|
264 |
+
self.update_progress("converting", int(progress), f"Creating {quant} quantization...")
|
265 |
+
|
266 |
+
quant_output = os.path.join(gguf_dir, f"model-{quant}.gguf")
|
267 |
+
|
268 |
+
quant_result = subprocess.run([
|
269 |
+
quantize_binary,
|
270 |
+
f16_output,
|
271 |
+
quant_output,
|
272 |
+
quant.upper()
|
273 |
+
], capture_output=True, text=True)
|
274 |
+
|
275 |
+
if quant_result.returncode == 0:
|
276 |
+
successful_quants.append(quant)
|
277 |
+
else:
|
278 |
+
logger.warning(f"Failed to create {quant} quantization: {quant_result.stderr}")
|
279 |
+
|
280 |
+
# Create model card
|
281 |
+
self._create_model_card(model_id, gguf_dir, successful_quants)
|
282 |
+
|
283 |
+
# Upload to Hugging Face
|
284 |
+
self.update_progress("uploading", 95, f"Uploading to {output_repo}...")
|
285 |
+
|
286 |
+
try:
|
287 |
+
api = HfApi(token=hf_token)
|
288 |
+
create_repo(output_repo, private=private_repo, exist_ok=True, token=hf_token)
|
289 |
+
|
290 |
+
for file_path in Path(gguf_dir).rglob("*"):
|
291 |
+
if file_path.is_file():
|
292 |
+
relative_path = file_path.relative_to(gguf_dir)
|
293 |
+
api.upload_file(
|
294 |
+
path_or_fileobj=str(file_path),
|
295 |
+
path_in_repo=str(relative_path),
|
296 |
+
repo_id=output_repo,
|
297 |
+
repo_type="model",
|
298 |
+
token=hf_token
|
299 |
+
)
|
300 |
+
|
301 |
+
except Exception as e:
|
302 |
+
return False, f"β Upload failed: {str(e)}"
|
303 |
+
|
304 |
+
self.update_progress("complete", 100, "Conversion completed successfully!")
|
305 |
+
|
306 |
+
return True, f"""β
Conversion completed successfully!
|
307 |
+
|
308 |
+
π **Results:**
|
309 |
+
- Successfully created: {', '.join(successful_quants)} quantizations
|
310 |
+
- Uploaded to: https://huggingface.co/{output_repo}
|
311 |
+
- Files created: {len(successful_quants)} GGUF files + README.md
|
312 |
+
|
313 |
+
π **Links:**
|
314 |
+
- View model: https://huggingface.co/{output_repo}
|
315 |
+
- Download files: https://huggingface.co/{output_repo}/tree/main
|
316 |
+
"""
|
317 |
+
|
318 |
+
except Exception as e:
|
319 |
+
self.update_progress("error", 0, f"Conversion failed: {str(e)}")
|
320 |
+
return False, f"β Conversion failed: {str(e)}"
|
321 |
+
|
322 |
+
finally:
|
323 |
+
# Cleanup
|
324 |
+
self._cleanup()
|
325 |
+
gc.collect()
|
326 |
+
|
327 |
+
def _find_quantize_binary(self) -> Optional[str]:
|
328 |
+
"""Find the llama-quantize binary"""
|
329 |
+
possible_locations = [
|
330 |
+
os.path.join(self.llama_cpp_dir, "build", "bin", "llama-quantize"),
|
331 |
+
os.path.join(self.llama_cpp_dir, "build", "llama-quantize"),
|
332 |
+
os.path.join(self.llama_cpp_dir, "build", "llama-quantize.exe"),
|
333 |
+
os.path.join(self.llama_cpp_dir, "build", "bin", "llama-quantize.exe")
|
334 |
+
]
|
335 |
+
|
336 |
+
for location in possible_locations:
|
337 |
+
if os.path.exists(location):
|
338 |
+
return location
|
339 |
+
|
340 |
+
return None
|
341 |
+
|
342 |
+
def _create_model_card(self, original_model_id: str, output_dir: str, quantizations: List[str]):
|
343 |
+
"""Create a model card for the GGUF model"""
|
344 |
+
|
345 |
+
quant_table = []
|
346 |
+
for quant in quantizations:
|
347 |
+
filename = f"model-{quant}.gguf"
|
348 |
+
if quant == "f16":
|
349 |
+
desc = "Original precision (largest file)"
|
350 |
+
elif "q4" in quant:
|
351 |
+
desc = "4-bit quantization (good balance)"
|
352 |
+
elif "q5" in quant:
|
353 |
+
desc = "5-bit quantization (higher quality)"
|
354 |
+
elif "q8" in quant:
|
355 |
+
desc = "8-bit quantization (high quality)"
|
356 |
+
else:
|
357 |
+
desc = "Quantized version"
|
358 |
+
|
359 |
+
quant_table.append(f"| {filename} | {quant.upper()} | {desc} |")
|
360 |
+
|
361 |
+
model_card_content = f"""---
|
362 |
+
language:
|
363 |
+
- en
|
364 |
+
library_name: gguf
|
365 |
+
base_model: {original_model_id}
|
366 |
+
tags:
|
367 |
+
- gguf
|
368 |
+
- quantized
|
369 |
+
- llama.cpp
|
370 |
+
- converted
|
371 |
+
license: apache-2.0
|
372 |
+
---
|
373 |
+
|
374 |
+
# {original_model_id} - GGUF
|
375 |
+
|
376 |
+
This repository contains GGUF quantizations of [{original_model_id}](https://huggingface.co/{original_model_id}).
|
377 |
+
|
378 |
+
**Converted using [HF GGUF Converter Space](https://huggingface.co/spaces/)**
|
379 |
+
|
380 |
+
## About GGUF
|
381 |
+
|
382 |
+
GGUF is a quantization method that allows you to run large language models on consumer hardware by reducing the precision of the model weights.
|
383 |
+
|
384 |
+
## Files
|
385 |
+
|
386 |
+
| Filename | Quant type | Description |
|
387 |
+
| -------- | ---------- | ----------- |
|
388 |
+
{chr(10).join(quant_table)}
|
389 |
+
|
390 |
+
## Usage
|
391 |
+
|
392 |
+
You can use these models with llama.cpp or any other GGUF-compatible inference engine.
|
393 |
+
|
394 |
+
### llama.cpp
|
395 |
+
|
396 |
+
```bash
|
397 |
+
./llama-cli -m model-q4_0.gguf -p "Your prompt here"
|
398 |
+
```
|
399 |
+
|
400 |
+
### Python (using llama-cpp-python)
|
401 |
+
|
402 |
+
```python
|
403 |
+
from llama_cpp import Llama
|
404 |
+
|
405 |
+
llm = Llama(model_path="model-q4_0.gguf")
|
406 |
+
output = llm("Your prompt here", max_tokens=512)
|
407 |
+
print(output['choices'][0]['text'])
|
408 |
+
```
|
409 |
+
|
410 |
+
## Original Model
|
411 |
+
|
412 |
+
This is a quantized version of [{original_model_id}](https://huggingface.co/{original_model_id}). Please refer to the original model card for more information about the model's capabilities, training data, and usage guidelines.
|
413 |
+
|
414 |
+
## Conversion Details
|
415 |
+
|
416 |
+
- Converted using llama.cpp
|
417 |
+
- Original model downloaded from Hugging Face
|
418 |
+
- Multiple quantization levels provided for different use cases
|
419 |
+
- Conversion completed on: {datetime.now().strftime("%Y-%m-%d %H:%M:%S UTC")}
|
420 |
+
|
421 |
+
## License
|
422 |
+
|
423 |
+
This model inherits the license from the original model. Please check the original model's license for usage terms.
|
424 |
+
"""
|
425 |
+
|
426 |
+
model_card_path = os.path.join(output_dir, "README.md")
|
427 |
+
with open(model_card_path, "w", encoding="utf-8") as f:
|
428 |
+
f.write(model_card_content)
|
429 |
+
|
430 |
+
def _cleanup(self):
|
431 |
+
"""Clean up temporary files"""
|
432 |
+
if self.temp_dir and os.path.exists(self.temp_dir):
|
433 |
+
try:
|
434 |
+
shutil.rmtree(self.temp_dir)
|
435 |
+
logger.info("Cleaned up temporary files")
|
436 |
+
except Exception as e:
|
437 |
+
logger.warning(f"Failed to cleanup: {e}")
|
438 |
+
|
439 |
+
# Initialize converter
|
440 |
+
converter = SpaceGGUFConverter()
|
441 |
+
|
442 |
+
def get_current_status():
|
443 |
+
"""Get current conversion status"""
|
444 |
+
global current_status
|
445 |
+
return f"""**Status:** {current_status['status']}
|
446 |
+
**Progress:** {current_status['progress']}%
|
447 |
+
**Message:** {current_status['message']}
|
448 |
+
**Time:** {current_status.get('timestamp', 'N/A')}"""
|
449 |
+
|
450 |
+
def validate_model_interface(model_id: str):
|
451 |
+
"""Interface function for model validation"""
|
452 |
+
if not model_id.strip():
|
453 |
+
return "β Please enter a model ID"
|
454 |
+
|
455 |
+
valid, message = converter.validate_model(model_id.strip())
|
456 |
+
return message
|
457 |
+
|
458 |
+
def check_resources_interface():
|
459 |
+
"""Interface function for resource checking"""
|
460 |
+
resources = converter.check_resources()
|
461 |
+
if "error" in resources:
|
462 |
+
return f"β Error checking resources: {resources['error']}"
|
463 |
+
|
464 |
+
return f"""## π» System Resources
|
465 |
+
|
466 |
+
**Memory:**
|
467 |
+
- Total: {resources['memory_total']}
|
468 |
+
- Available: {resources['memory_available']} ({100-resources['memory_percent']:.1f}% free)
|
469 |
+
- Usage: {resources['memory_percent']:.1f}%
|
470 |
+
|
471 |
+
**Storage:**
|
472 |
+
- Total: {resources['disk_total']}
|
473 |
+
- Free: {resources['disk_free']} ({100-resources['disk_percent']:.1f}% free)
|
474 |
+
- Usage: {resources['disk_percent']:.1f}%
|
475 |
+
|
476 |
+
**Compute:**
|
477 |
+
- CPU Cores: {resources['cpu_count']}
|
478 |
+
- GPU Available: {'β
Yes' if resources['gpu_available'] else 'β No'}
|
479 |
+
- GPU Memory: {resources['gpu_memory']}
|
480 |
+
|
481 |
+
**Status:** {'π’ Good' if resources['memory_percent'] < 80 and resources['disk_percent'] < 80 else 'π‘ Limited' if resources['memory_percent'] < 90 else 'π΄ Critical'}
|
482 |
+
"""
|
483 |
+
|
484 |
+
def convert_model_interface(
|
485 |
+
model_id: str,
|
486 |
+
output_repo: str,
|
487 |
+
hf_token: str,
|
488 |
+
quant_f16: bool,
|
489 |
+
quant_q4_0: bool,
|
490 |
+
quant_q4_1: bool,
|
491 |
+
quant_q5_0: bool,
|
492 |
+
quant_q5_1: bool,
|
493 |
+
quant_q8_0: bool,
|
494 |
+
private_repo: bool
|
495 |
+
):
|
496 |
+
"""Interface function for model conversion"""
|
497 |
+
|
498 |
+
# Validate inputs
|
499 |
+
if not model_id.strip():
|
500 |
+
return "β Please enter a model ID"
|
501 |
+
|
502 |
+
if not output_repo.strip():
|
503 |
+
return "β Please enter an output repository name"
|
504 |
+
|
505 |
+
if not hf_token.strip():
|
506 |
+
return "β Please enter your Hugging Face token"
|
507 |
+
|
508 |
+
# Collect selected quantizations
|
509 |
+
quantizations = []
|
510 |
+
if quant_f16:
|
511 |
+
quantizations.append("f16")
|
512 |
+
if quant_q4_0:
|
513 |
+
quantizations.append("q4_0")
|
514 |
+
if quant_q4_1:
|
515 |
+
quantizations.append("q4_1")
|
516 |
+
if quant_q5_0:
|
517 |
+
quantizations.append("q5_0")
|
518 |
+
if quant_q5_1:
|
519 |
+
quantizations.append("q5_1")
|
520 |
+
if quant_q8_0:
|
521 |
+
quantizations.append("q8_0")
|
522 |
+
|
523 |
+
if not quantizations:
|
524 |
+
return "β Please select at least one quantization type"
|
525 |
+
|
526 |
+
# Start conversion
|
527 |
+
success, message = converter.convert_model(
|
528 |
+
model_id.strip(),
|
529 |
+
output_repo.strip(),
|
530 |
+
quantizations,
|
531 |
+
hf_token.strip(),
|
532 |
+
private_repo
|
533 |
+
)
|
534 |
+
|
535 |
+
return message
|
536 |
+
|
537 |
+
# Create Gradio interface
|
538 |
+
def create_interface():
|
539 |
+
"""Create the Gradio interface"""
|
540 |
+
|
541 |
+
with gr.Blocks(
|
542 |
+
title="π€ GGUF Model Converter",
|
543 |
+
theme=gr.themes.Soft(),
|
544 |
+
css="""
|
545 |
+
.status-box {
|
546 |
+
background-color: #f0f0f0;
|
547 |
+
padding: 10px;
|
548 |
+
border-radius: 5px;
|
549 |
+
margin: 10px 0;
|
550 |
+
}
|
551 |
+
"""
|
552 |
+
) as demo:
|
553 |
+
|
554 |
+
gr.Markdown("""
|
555 |
+
# π€ GGUF Model Converter
|
556 |
+
|
557 |
+
Convert Hugging Face models to GGUF format for use with llama.cpp and other inference engines.
|
558 |
+
|
559 |
+
β οΈ **Important Notes:**
|
560 |
+
- Large models (>7B parameters) may take a long time and require significant memory
|
561 |
+
- Make sure you have sufficient disk space (models can be several GB)
|
562 |
+
- You need a Hugging Face token with write access to upload models
|
563 |
+
""")
|
564 |
+
|
565 |
+
with gr.Tab("π§ Model Converter"):
|
566 |
+
with gr.Row():
|
567 |
+
with gr.Column(scale=2):
|
568 |
+
gr.Markdown("### π Model Configuration")
|
569 |
+
|
570 |
+
model_id_input = gr.Textbox(
|
571 |
+
label="Model ID",
|
572 |
+
placeholder="e.g., microsoft/DialoGPT-small",
|
573 |
+
info="Hugging Face model repository ID"
|
574 |
+
)
|
575 |
+
|
576 |
+
validate_btn = gr.Button("β
Validate Model", variant="secondary")
|
577 |
+
validation_output = gr.Markdown()
|
578 |
+
|
579 |
+
output_repo_input = gr.Textbox(
|
580 |
+
label="Output Repository",
|
581 |
+
placeholder="e.g., your-username/model-name-GGUF",
|
582 |
+
info="Where to upload the converted model"
|
583 |
+
)
|
584 |
+
|
585 |
+
hf_token_input = gr.Textbox(
|
586 |
+
label="Hugging Face Token",
|
587 |
+
type="password",
|
588 |
+
placeholder="hf_xxxxxxxxxxxxxxxx",
|
589 |
+
info="Get your token from https://huggingface.co/settings/tokens"
|
590 |
+
)
|
591 |
+
|
592 |
+
private_repo_checkbox = gr.Checkbox(
|
593 |
+
label="Make repository private",
|
594 |
+
value=False
|
595 |
+
)
|
596 |
+
|
597 |
+
with gr.Column(scale=1):
|
598 |
+
gr.Markdown("### ποΈ Quantization Options")
|
599 |
+
|
600 |
+
quant_f16 = gr.Checkbox(label="F16 (Original precision)", value=True)
|
601 |
+
quant_q4_0 = gr.Checkbox(label="Q4_0 (Small, fast)", value=True)
|
602 |
+
quant_q4_1 = gr.Checkbox(label="Q4_1 (Small, balanced)", value=False)
|
603 |
+
quant_q5_0 = gr.Checkbox(label="Q5_0 (Medium, good quality)", value=False)
|
604 |
+
quant_q5_1 = gr.Checkbox(label="Q5_1 (Medium, better quality)", value=False)
|
605 |
+
quant_q8_0 = gr.Checkbox(label="Q8_0 (Large, high quality)", value=False)
|
606 |
+
|
607 |
+
gr.Markdown("### π Start Conversion")
|
608 |
+
convert_btn = gr.Button("π Convert Model", variant="primary", size="lg")
|
609 |
+
|
610 |
+
conversion_output = gr.Markdown()
|
611 |
+
|
612 |
+
with gr.Tab("π System Status"):
|
613 |
+
gr.Markdown("### π» Resource Monitor")
|
614 |
+
|
615 |
+
refresh_btn = gr.Button("π Refresh Resources", variant="secondary")
|
616 |
+
resources_output = gr.Markdown()
|
617 |
+
|
618 |
+
gr.Markdown("### π Conversion Status")
|
619 |
+
status_btn = gr.Button("π Check Status", variant="secondary")
|
620 |
+
status_output = gr.Markdown(get_current_status())
|
621 |
+
|
622 |
+
with gr.Tab("π Help & Examples"):
|
623 |
+
gr.Markdown("""
|
624 |
+
## π― Quick Start Guide
|
625 |
+
|
626 |
+
1. **Enter Model ID**: Use any Hugging Face model ID (e.g., `microsoft/DialoGPT-small`)
|
627 |
+
2. **Validate Model**: Click "Validate Model" to check if the model is accessible
|
628 |
+
3. **Set Output Repository**: Choose where to upload (e.g., `your-username/model-name-GGUF`)
|
629 |
+
4. **Add HF Token**: Get your token from [Hugging Face Settings](https://huggingface.co/settings/tokens)
|
630 |
+
5. **Select Quantizations**: Choose which formats to create
|
631 |
+
6. **Convert**: Click "Convert Model" and wait for completion
|
632 |
+
|
633 |
+
## π Quantization Guide
|
634 |
+
|
635 |
+
- **F16**: Original precision, largest file size, best quality
|
636 |
+
- **Q4_0**: 4-bit quantization, smallest size, good for most uses
|
637 |
+
- **Q4_1**: 4-bit with better quality than Q4_0
|
638 |
+
- **Q5_0/Q5_1**: 5-bit quantization, balance of size and quality
|
639 |
+
- **Q8_0**: 8-bit quantization, high quality, larger files
|
640 |
+
|
641 |
+
## π‘ Tips for Success
|
642 |
+
|
643 |
+
- Start with small models (< 1B parameters) to test
|
644 |
+
- Use Q4_0 for mobile/edge deployment
|
645 |
+
- Use Q8_0 or F16 for best quality
|
646 |
+
- Monitor system resources in the Status tab
|
647 |
+
- Large models may take 30+ minutes to convert
|
648 |
+
|
649 |
+
## π§ Supported Models
|
650 |
+
|
651 |
+
This converter works with most language models that use standard architectures:
|
652 |
+
- LLaMA, LLaMA 2, Code Llama
|
653 |
+
- Mistral, Mixtral
|
654 |
+
- Phi, Phi-2, Phi-3
|
655 |
+
- Qwen, ChatGLM
|
656 |
+
- And many others!
|
657 |
+
""")
|
658 |
+
|
659 |
+
# Event handlers
|
660 |
+
validate_btn.click(
|
661 |
+
fn=validate_model_interface,
|
662 |
+
inputs=[model_id_input],
|
663 |
+
outputs=[validation_output]
|
664 |
+
)
|
665 |
+
|
666 |
+
convert_btn.click(
|
667 |
+
fn=convert_model_interface,
|
668 |
+
inputs=[
|
669 |
+
model_id_input,
|
670 |
+
output_repo_input,
|
671 |
+
hf_token_input,
|
672 |
+
quant_f16,
|
673 |
+
quant_q4_0,
|
674 |
+
quant_q4_1,
|
675 |
+
quant_q5_0,
|
676 |
+
quant_q5_1,
|
677 |
+
quant_q8_0,
|
678 |
+
private_repo_checkbox
|
679 |
+
],
|
680 |
+
outputs=[conversion_output]
|
681 |
+
)
|
682 |
+
|
683 |
+
refresh_btn.click(
|
684 |
+
fn=check_resources_interface,
|
685 |
+
outputs=[resources_output]
|
686 |
+
)
|
687 |
+
|
688 |
+
status_btn.click(
|
689 |
+
fn=get_current_status,
|
690 |
+
outputs=[status_output]
|
691 |
+
)
|
692 |
+
|
693 |
+
# Auto-refresh status every 5 seconds during conversion
|
694 |
+
demo.load(fn=check_resources_interface, outputs=[resources_output])
|
695 |
+
|
696 |
+
return demo
|
697 |
+
|
698 |
+
# Launch the interface
|
699 |
+
if __name__ == "__main__":
|
700 |
+
demo = create_interface()
|
701 |
+
demo.launch(
|
702 |
+
server_name="0.0.0.0",
|
703 |
+
server_port=7860,
|
704 |
+
share=False,
|
705 |
+
show_error=True
|
706 |
+
)
|