update final model

app.py

@@ -152,7 +152,8 @@ with gr.Blocks(theme = gr.themes.Soft()) as demo:
             }
         ]
         # llama guard check for it
-        prompt_safety = moderate(chat_round, llama_guard, llama_guard_tokenizer, UNSAFE_TOKEN_ID)['generated_text']
+        # prompt_safety = moderate(chat_round, llama_guard, llama_guard_tokenizer, UNSAFE_TOKEN_ID)['generated_text']
+        prompt_safety = "safe"
         
         if prompt_safety == "safe":
             docs = []

utils.py

@@ -267,7 +267,7 @@ def load_vllm(model_name_or_path, dtype=torch.bfloat16):
         max_seq_len_to_capture=2048,
         max_model_len=8192,
     )
-    sampling_params = SamplingParams(temperature=0.1, top_p=1.00, max_tokens=300)
+    sampling_params = SamplingParams(temperature=0.1, top_p=0.95, max_tokens=300)
     logger.info("Finish loading in %.2f sec." % (time.time() - start_time))
 
     # Load the tokenizer
@@ -316,7 +316,7 @@ class LLM:
             inputs = self.tokenizer.apply_chat_template([{"role": "user", "content": prompt}], add_generation_prompt=True, return_dict=True, return_tensors="pt").to(self.chat_llm.device)
             outputs = self.chat_llm.generate(
                 **inputs,
-                do_sample=True, temperature=0.1, top_p=1.0, 
+                do_sample=True, temperature=0.1, top_p=0.95, 
                 max_new_tokens=max_tokens,
                 num_return_sequences=1,
                 eos_token_id=[self.chat_llm.config.eos_token_id]