Update app.py

app.py

@@ -36,18 +36,17 @@ def build_faiss_vectorstore(chunks):
     print(f"Total number of documents: {num_documents}")
     return vectorstore
 
-# Function to retrieve similar text
 # Function to retrieve similar text
 def retrieve(query, vectorstore, top_k=5):
     docs_and_scores = vectorstore.similarity_search_with_score(query=query, k=top_k)
 
-    # Return (page_content, score) for docs that meet the score threshold
-    return [
-        (doc.page_content, float(score))  # Ensure score is a standard float
-        for doc, score in docs_and_scores
-        if float(score) <= 0.75
-    ]
+    # Filter results based on score threshold
+    filtered_docs_and_scores = [(doc.page_content, float(score)) for doc, score in docs_and_scores if float(score) <= 0.75]
+
+    # Separate docs from the (doc, score) tuples
+    docs_content = [doc for doc, _ in filtered_docs_and_scores]
 
+    return docs_content, filtered_docs_and_scores
     
 class ChatRequest(BaseModel):
     message: str
@@ -99,7 +98,7 @@ async def chat(request: ChatRequest):
             
 
             # Retrieve relevant text
-            docs, scores = retrieve(request.message, vectorstore, top_k=5)
+            docs, filtered_docs_and_scores = retrieve(request.message, vectorstore, top_k=5)
             docs = "\n\n".join(docs)
 
             rag_prompt = f"""Use the following information to answer the user's query. You do not have to use all the information, just the pieces that directly 
@@ -142,7 +141,7 @@ async def chat(request: ChatRequest):
             del request.chat_history[-1]
             request.chat_history.append({"role": "user", "parts": [{"text": request.message}]})
 
-            return {"response": response.text, "dataset_str": concise_text_string, "docs": docs, "history": request.chat_history, "RAG_prompt": rag_prompt, "chunks": chunks}
+            return {"response": response.text, "dataset_str": concise_text_string, "docs": docs, "filtered_docs_and_scores": filtered_docs_and_scores, "history": request.chat_history, "RAG_prompt": rag_prompt, "chunks": chunks}
 
         if request.model_choice == "HF":
             if hf_token: