Update app.py

app.py

@@ -86,7 +86,12 @@ class BasicModel:
         #                  )
         
         # Download the model weights to the local machine and build the pipeline
-        quantization_config = BitsAndBytesConfig(load_in_4bit=True)
+        quantization_config = BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_quant_type="nf4",
+            bnb_4bit_compute_dtype=torch.float16, 
+            bnb_4bit_use_double_quant=True,
+        )
         tok = AutoTokenizer.from_pretrained(model_id, token=hf_token)
         mod = AutoModelForCausalLM.from_pretrained(
             model_id,