samith-a commited on
Commit
ea721a6
·
1 Parent(s): 5db110f

try model unsloth/Llama-3.2-1B-bnb-4bit

Browse files
Files changed (1) hide show
  1. app.py +2 -2
app.py CHANGED
@@ -29,7 +29,7 @@ class ModelManager:
29
  if HAS_UNSLOTH and self.device != "cpu":
30
  # GPU via Unsloth + LoRA
31
  backbone, tokenizer = FastLanguageModel.from_pretrained(
32
- "unsloth/Llama-3.2-1B-Instruct-bnb-4bit",
33
  load_in_4bit=True,
34
  dtype=torch.float16,
35
  device_map="auto",
@@ -50,7 +50,7 @@ class ModelManager:
50
 
51
  # --- Fallback: CPU-only via HF Transformers + PEFT ---
52
  print("Falling back to CPU-only Transformers + PEFT")
53
- base_name = "unsloth/Llama-3.2-1B-Instruct" # non-4bit to run on CPU
54
  tokenizer = AutoTokenizer.from_pretrained(base_name, use_fast=True)
55
  base = AutoModelForCausalLM.from_pretrained(
56
  base_name,
 
29
  if HAS_UNSLOTH and self.device != "cpu":
30
  # GPU via Unsloth + LoRA
31
  backbone, tokenizer = FastLanguageModel.from_pretrained(
32
+ "unsloth/Llama-3.2-1B-bnb-4bit",
33
  load_in_4bit=True,
34
  dtype=torch.float16,
35
  device_map="auto",
 
50
 
51
  # --- Fallback: CPU-only via HF Transformers + PEFT ---
52
  print("Falling back to CPU-only Transformers + PEFT")
53
+ base_name = "unsloth/Llama-3.2-1B" # non-4bit to run on CPU
54
  tokenizer = AutoTokenizer.from_pretrained(base_name, use_fast=True)
55
  base = AutoModelForCausalLM.from_pretrained(
56
  base_name,