second try

agent/agent_1.py

@@ -8,21 +8,30 @@ from langchain_google_genai import ChatGoogleGenerativeAI
 from langchain_groq import ChatGroq
 from langchain_huggingface import HuggingFaceEmbeddings
 from langchain_community.tools.tavily_search import TavilySearchResults
-from langchain_community.document_loaders import WikipediaLoader
-from langchain_community.document_loaders import ArxivLoader
+from langchain_community.document_loaders import WikipediaLoader, ArxivLoader
 from langchain_community.vectorstores import FAISS
 from langchain_core.messages import SystemMessage, HumanMessage, AIMessage
 from langchain_core.tools import tool
 from langchain.tools.retriever import create_retriever_tool
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, BlipProcessor, BlipForConditionalGeneration
+from youtube_transcript_api import YouTubeTranscriptApi
+from PIL import Image
+import requests
 import torch
+import pandas as pd
+import numpy as np
+from sklearn.metrics.pairwise import cosine_similarity
 
 load_dotenv()
 
+# Load QA pairs and compute embeddings once
+qa_df = pd.read_csv("/statics/qa_pairs.csv")
+embeddings_model = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
+qa_embeddings = embeddings_model.embed_documents(qa_df["question"].tolist())
+
 class LocalChatModel:
-    # mistralai/Mistral-7B-Instruct-v0.3 or TinyLlama/TinyLlama-1.1B-Chat-v1.0
-    def __init__(self, model_name="mistralai/Mistral-7B-Instruct-v0.3"):
-        print("Loading LLM on CPU...")
+    def __init__(self, model_name="TinyLlama/TinyLlama-1.1B-Chat-v1.0"):
+        print(f"Loading {model_name} on CPU...")
         self.tokenizer = AutoTokenizer.from_pretrained(model_name)
         self.model = AutoModelForCausalLM.from_pretrained(model_name)
         self.model.eval()
@@ -47,7 +56,7 @@ class LocalChatModel:
         with torch.no_grad():
             outputs = self.model.generate(
                 **inputs,
-                max_new_tokens=256,
+                max_new_tokens=512,
                 do_sample=True,
                 temperature=0.7,
                 pad_token_id=self.tokenizer.eos_token_id
@@ -58,25 +67,21 @@ class LocalChatModel:
         return AIMessage(content=response)
 
 @tool
-
 def multiply(a: int, b: int) -> int:
     """Multiply two integers."""
     return a * b
 
 @tool
-
 def add(a: int, b: int) -> int:
     """Add two integers."""
     return a + b
 
 @tool
-
 def subtract(a: int, b: int) -> int:
     """Subtract second integer from first."""
     return a - b
 
 @tool
-
 def divide(a: int, b: int) -> float:
     """Divide first integer by second. Raises error if divisor is zero."""
     if b == 0:
@@ -84,61 +89,67 @@ def divide(a: int, b: int) -> float:
     return a / b
 
 @tool
-
 def modulus(a: int, b: int) -> int:
     """Get the modulus (remainder) of first integer divided by second."""
     return a % b
 
 @tool
-
 def wiki_search(query: str) -> str:
     """Search Wikipedia for a query and return formatted results."""
     search_docs = WikipediaLoader(query=query, load_max_docs=2).load()
-    formatted_search_docs = "\n\n---\n\n".join(
-        [
-            f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content}\n</Document>'
-            for doc in search_docs
-        ])
-    return {"wiki_results": formatted_search_docs}
+    return "\n\n---\n\n".join([doc.page_content for doc in search_docs])
 
 @tool
-
 def web_search(query: str) -> str:
     """Search Tavily for a query and return formatted results."""
     search_docs = TavilySearchResults(max_results=3).invoke(query=query)
-    formatted_search_docs = "\n\n---\n\n".join(
-        [
-            f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content}\n</Document>'
-            for doc in search_docs
-        ])
-    return {"web_results": formatted_search_docs}
+    return "\n\n---\n\n".join([doc.page_content for doc in search_docs])
 
 @tool
-
 def arvix_search(query: str) -> str:
     """Search Arxiv for a query and return formatted results."""
     search_docs = ArxivLoader(query=query, load_max_docs=3).load()
-    formatted_search_docs = "\n\n---\n\n".join(
-        [
-            f'<Document source="{doc.metadata["source"]}" page="{doc.metadata.get("page", "")}"/>\n{doc.page_content[:1000]}\n</Document>'
-            for doc in search_docs
-        ])
-    return {"arvix_results": formatted_search_docs}
-
-#dir = os.getcwd()
-#print(dir.rsplit('/')[:-1])
-with open("statics/system_prompt.txt", "r", encoding="utf-8") as f:
+    return "\n\n---\n\n".join([doc.page_content[:1000] for doc in search_docs])
+
+@tool
+def youtube_summary(video_url: str) -> str:
+    """Fetch and summarize a YouTube video using transcript (if available)."""
+    import re
+    match = re.search(r"(?<=v=|youtu.be/)[^&#]+", video_url)
+    if not match:
+        return "Invalid YouTube URL."
+    video_id = match.group()
+    try:
+        transcript = YouTubeTranscriptApi.get_transcript(video_id)
+        return " ".join([seg["text"] for seg in transcript])[:3000]
+    except Exception as e:
+        return f"Transcript not available or error: {e}"
+
+@tool
+def image_caption(image_url: str) -> str:
+    """Generate a description of an image from a public URL."""
+    processor = BlipProcessor.from_pretrained("Salesforce/blip-image-captioning-base")
+    model = BlipForConditionalGeneration.from_pretrained("Salesforce/blip-image-captioning-base")
+    image = Image.open(requests.get(image_url, stream=True).raw).convert("RGB")
+    inputs = processor(image, return_tensors="pt")
+    out = model.generate(**inputs)
+    return processor.decode(out[0], skip_special_tokens=True)
+
+@tool
+def qa_reference(query: str) -> str:
+    """Search example QA dataset for similar questions and return the closest answer."""
+    query_embedding = embeddings_model.embed_query(query)
+    sims = cosine_similarity([query_embedding], qa_embeddings)[0]
+    top_idx = int(np.argmax(sims))
+    return f"Similar question: {qa_df.question[top_idx]}\nAnswer: {qa_df.answer[top_idx]}"
+
+with open("system_prompt.txt", "r", encoding="utf-8") as f:
     system_prompt = f.read()
 
 sys_msg = SystemMessage(content=system_prompt)
 
 embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/all-mpnet-base-v2")
 vector_store = FAISS.from_texts(["Sample text 1", "Sample text 2"], embedding=embeddings)
-create_retriever_tool = create_retriever_tool(
-    retriever=vector_store.as_retriever(),
-    name="Question Search",
-    description="A tool to retrieve similar questions from a vector store."
-)
 
 tools = [
     multiply,
@@ -149,6 +160,9 @@ tools = [
     wiki_search,
     web_search,
     arvix_search,
+    youtube_summary,
+    image_caption,
+    qa_reference,
 ]
 
 def build_graph(provider: str = "huggingface"):
@@ -157,36 +171,24 @@ def build_graph(provider: str = "huggingface"):
     elif provider == "groq":
         llm = ChatGroq(model="qwen-qwq-32b", temperature=0)
     elif provider == "huggingface":
-        llm = LocalChatModel(model_name="mistralai/Mistral-7B-Instruct-v0.3")
+        llm = LocalChatModel()
     else:
         raise ValueError("Invalid provider. Choose 'google', 'groq' or 'huggingface'.")
 
     def assistant(state: MessagesState):
         return {"messages": [llm.invoke(state["messages"]) ]}
 
-    def retriever(state: MessagesState):
-        similar_question = vector_store.similarity_search(state["messages"][0].content)
-        example_msg = HumanMessage(
-            content=f"Here I provide a similar question and answer for reference: \n\n{similar_question[0].page_content}",
-        )
-        return {"messages": [sys_msg] + state["messages"] + [example_msg]}
-
     builder = StateGraph(MessagesState)
-    builder.add_node("retriever", retriever)
     builder.add_node("assistant", assistant)
     builder.add_node("tools", ToolNode(tools))
-    builder.add_edge(START, "retriever")
-    builder.add_edge("retriever", "assistant")
-    builder.add_conditional_edges(
-        "assistant",
-        tools_condition,
-    )
+    builder.add_edge(START, "assistant")
+    builder.add_conditional_edges("assistant", tools_condition)
     builder.add_edge("tools", "assistant")
 
     return builder.compile()
 
 if __name__ == "__main__":
-    question = "When was a picture of St. Thomas Aquinas first added to the Wikipedia page on the Principle of double effect?"
+    question = "Describe this image: https://example.com/sample.jpg"
     graph = build_graph(provider="huggingface")
     messages = [HumanMessage(content=question)]
     messages = graph.invoke({"messages": messages})

requirements.txt

@@ -17,4 +17,5 @@ wikipedia
 pgvector
 python-dotenv
 faiss-cpu
-sentencepiece
+sentencepiece
+youtube-transcript-api