feat: adding project files.

pyproject.toml

@@ -0,0 +1,36 @@
+[project]
+name = "agent-hackathon"
+version = "0.1.0"
+description = "Agent hackathon"
+readme = "README.md"
+authors = [
+    { name = "shamik", email = "39588365+Shamik-07@users.noreply.github.com" }
+]
+requires-python = ">=3.12"
+dependencies = [
+    "arxiv>=2.2.0",
+    "flagembedding>=1.3.5",
+    "httpx>=0.28.1",
+    "huggingface-hub[hf-xet]>=0.32.4",
+    "llama-hub>=0.0.79.post1",
+    "llama-index-embeddings-huggingface>=0.5.4",
+    "llama-index-embeddings-huggingface-api>=0.3.1",
+    "llama-index-llms-huggingface>=0.5.0",
+    "llama-index-llms-huggingface-api>=0.5.0",
+    "llama-index-vector-stores-milvus>=0.8.4",
+    "openai>=1.84.0",
+    "pyprojroot>=0.3.0",
+    "python-dotenv>=1.1.0",
+    "smolagents>=1.17.0",
+]
+
+
+[dependency-groups]
+dev = [
+    "google-generativeai>=0.8.5",
+    "ipykernel>=6.29.5",
+    "ipywidgets>=8.1.7",
+    "marimo>=0.13.15",
+    "nbformat>=5.10.4",
+    "ruff>=0.11.13",
+]

src/agent_hackathon/__init__.py

@@ -0,0 +1,2 @@
+def hello() -> str:
+    return "Hello from agent-hackathon!"

src/agent_hackathon/consts.py

@@ -0,0 +1,3 @@
+from pyprojroot import find_root, has_file
+
+PROJECT_ROOT_DIR = find_root(criterion=has_file(file="README.md"))

src/agent_hackathon/create_vector_db.py

@@ -0,0 +1,149 @@
+import json
+from copy import deepcopy
+
+from dotenv import find_dotenv, load_dotenv
+from llama_index.core import StorageContext, VectorStoreIndex
+from llama_index.core.node_parser import SentenceSplitter
+from llama_index.core.schema import Document
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+from llama_index.vector_stores.milvus import MilvusVectorStore
+from llama_index.vector_stores.milvus.utils import BGEM3SparseEmbeddingFunction
+
+from src.agent_hackathon.consts import PROJECT_ROOT_DIR
+from src.agent_hackathon.logger import get_logger
+
+logger = get_logger(log_name="create_vector_db", log_dir=PROJECT_ROOT_DIR / "logs")
+
+
+class VectorDBCreator:
+    """Handles creation of a Milvus vector database from arXiv data."""
+
+    def __init__(
+        self,
+        data_path: str,
+        db_uri: str,
+        embedding_model: str = "Qwen/Qwen3-Embedding-0.6B",
+        chunk_size: int = 20_000,
+        chunk_overlap: int = 0,
+        vector_dim: int = 1024,
+        insert_batch_size: int = 8192,
+    ) -> None:
+        """
+        Initialize the VectorDBCreator.
+
+        Args:
+            data_path: Path to the JSON data file.
+            db_uri: URI for the Milvus database.
+            embedding_model: Name of the embedding model.
+            chunk_size: Size of text chunks for splitting.
+            chunk_overlap: Overlap between text chunks.
+            vector_dim: Dimension of the embedding vectors.
+            insert_batch_size: Batch size for insertion.
+        """
+        self.data_path = data_path
+        self.db_uri = db_uri
+        self.embedding_model = embedding_model
+        self.chunk_size = chunk_size
+        self.chunk_overlap = chunk_overlap
+        self.vector_dim = vector_dim
+        self.insert_batch_size = insert_batch_size
+        self.embed_model = HuggingFaceEmbedding(
+            model_name=self.embedding_model, device="cpu"
+        )
+        self.sent_splitter = SentenceSplitter(
+            chunk_size=self.chunk_size, chunk_overlap=self.chunk_overlap
+        )
+        logger.info("VectorDBCreator initialized.")
+
+    def load_data(self) -> list[dict]:
+        """
+        Load and return data from the JSON file.
+
+        Returns:
+            List of dictionaries containing arXiv data.
+        """
+        logger.info(f"Loading data from {self.data_path}")
+        with open(file=self.data_path) as f:
+            data = json.load(fp=f)
+        logger.info("Data loaded successfully.")
+        return deepcopy(x=data)
+
+    def prepare_documents(self, data: list[dict]) -> list[Document]:
+        """
+        Convert raw data into a list of Document objects.
+
+        Args:
+            data: List of dictionaries with arXiv data.
+
+        Returns:
+            List of Document objects.
+        """
+        logger.info("Preparing documents from data.")
+        docs = [Document(text=d.pop("abstract"), metadata=d) for d in data]
+        logger.info(f"Prepared {len(docs)} documents.")
+        return docs
+
+    def create_vector_store(self) -> MilvusVectorStore:
+        """
+        Create and return a MilvusVectorStore instance.
+
+        Returns:
+            Configured MilvusVectorStore.
+        """
+        logger.info(f"Creating MilvusVectorStore at {self.db_uri}")
+        store = MilvusVectorStore(
+            uri=self.db_uri,
+            dim=self.vector_dim,
+            enable_sparse=True,
+            sparse_embedding_function=BGEM3SparseEmbeddingFunction(),
+        )
+        logger.info("MilvusVectorStore created.")
+        return store
+
+    def build_index(
+        self, docs_list: list[Document], vector_store: MilvusVectorStore
+    ) -> VectorStoreIndex:
+        """
+        Build and return a VectorStoreIndex from documents.
+
+        Args:
+            docs_list: List of Document objects.
+            vector_store: MilvusVectorStore instance.
+
+        Returns:
+            VectorStoreIndex object.
+        """
+        logger.info("Building VectorStoreIndex.")
+        storage_context = StorageContext.from_defaults(vector_store=vector_store)
+        index = VectorStoreIndex.from_documents(
+            documents=docs_list,
+            storage_context=storage_context,
+            embed_model=self.embed_model,
+            transformations=[self.sent_splitter],
+            show_progress=True,
+            insert_batch_size=self.insert_batch_size,
+        )
+        logger.info("VectorStoreIndex built.")
+        return index
+
+    def run(self) -> None:
+        """
+        Execute the full pipeline: load data, prepare documents, create vector store, and build index.
+        """
+        logger.info("Running full vector DB creation pipeline.")
+        data = self.load_data()
+        docs_list = self.prepare_documents(data=data)
+        vector_store = self.create_vector_store()
+        self.build_index(docs_list=docs_list, vector_store=vector_store)
+        logger.info("Pipeline finished.")
+
+
+if __name__ == "__main__":
+    logger.info("Script started.")
+    # Optionally load environment variables if needed
+    _ = load_dotenv(dotenv_path=find_dotenv(raise_error_if_not_found=True))
+    creator = VectorDBCreator(
+        data_path=f"{PROJECT_ROOT_DIR}/data/cs_data_arxiv.json", db_uri="arxiv_docs.db"
+    )
+    creator.run()
+    logger.info("Script finished.")

src/agent_hackathon/generate_arxiv_responses.py

@@ -0,0 +1,105 @@
+import json
+from pathlib import Path
+from typing import Any
+
+from huggingface_hub import InferenceClient
+
+from src.agent_hackathon.consts import PROJECT_ROOT_DIR
+from src.agent_hackathon.create_vector_db import VectorDBCreator
+from src.agent_hackathon.logger import get_logger
+from src.agent_hackathon.query_vector_db import RetrieverEngineBuilder
+
+logger = get_logger(log_name="arxiv_responses", log_dir=PROJECT_ROOT_DIR / "logs")
+
+
+class ArxivResponseGenerator:
+    """
+    Handles retrieval and formatting of arXiv papers using a vector database and LLM.
+    """
+
+    def __init__(self, vector_store_path: Path) -> None:
+        """Initializes the ArxivResponseGenerator."""
+        self.vector_store_path = vector_store_path
+        self.client = self._initialise_client()
+        logger.info("ArxivResponseGenerator initialized.")
+
+    def _initialise_retriever(self) -> Any:
+        """
+        Initializes and returns a retriever engine.
+
+        Returns:
+            Any: Retriever engine object.
+        """
+        logger.info("Initializing retriever engine.")
+        vector_db_creator = VectorDBCreator(
+            data_path=..., db_uri=self.vector_store_path.as_posix()
+        )
+        vector_store = vector_db_creator.create_vector_store()
+        retriever_class = RetrieverEngineBuilder(
+            vector_store=vector_store,
+        )
+        retriever = retriever_class.build_retriever_engine()
+        logger.info("Retriever engine initialized.")
+        return retriever, retriever_class
+
+    def _initialise_client(self) -> InferenceClient:
+        """
+        Initializes and returns an InferenceClient.
+
+        Returns:
+            InferenceClient: HuggingFace InferenceClient instance.
+        """
+        logger.info("Initializing InferenceClient.")
+        client = InferenceClient(
+            provider="auto",
+            bill_to="VitalNest",
+        )
+        logger.info("InferenceClient initialized.")
+        return client
+
+    def retrieve_arxiv_papers(self, query: str) -> str:
+        """
+        Retrieves and formats arXiv papers for a given query.
+
+        Args:
+            query (str): The search query.
+
+        Returns:
+            str: Formatted response from the LLM.
+        """
+        logger.info(f"Retrieving arXiv papers for query: {query}")
+        retriever, retriever_class = self._initialise_retriever()
+        retrieved_content = json.dumps(
+            obj=[(i.get_content(), i.metadata) for i in retriever.retrieve(query)]
+        )
+        logger.info("Retrieved content from vector DB.")
+        completion = self.client.chat.completions.create(
+            model="meta-llama/Llama-4-Scout-17B-16E-Instruct",
+            temperature=0.1,
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": f"Format the following output neatly:{retrieved_content}. Return only the output.",
+                        },
+                    ],
+                }
+            ],
+        )
+        logger.info("Received completion from LLM.")
+        retriever_class.vector_store.client.close()
+        logger.info("Closed vector store client.")
+        return completion.choices[0].message.content
+
+
+if __name__ == "__main__":
+    logger.info("Script started.")
+    generator = ArxivResponseGenerator(
+        vector_store_path=PROJECT_ROOT_DIR / "db/arxiv_docs.db"
+    )
+    query = "deep learning for NLP"  # Example query, replace as needed
+    result = generator.retrieve_arxiv_papers(query=query)
+    print(result)
+    logger.info("Script finished.")

src/agent_hackathon/logger.py

@@ -0,0 +1,45 @@
+import logging
+from datetime import datetime
+from pathlib import Path
+
+from rich.logging import RichHandler
+
+
+def get_logger(log_name: str, log_dir: Path) -> logging.Logger:
+    """
+    Returns a logger with RichHandler and file handler.
+
+    Args:
+        log_name (str): Name prefix for the log file.
+        log_dir (Path): Directory to store log files.
+
+    Returns:
+        logging.Logger: Configured logger instance.
+    """
+    log_dir.mkdir(parents=True, exist_ok=True)
+    date_str = datetime.now().strftime(format="%m_%d_%Y")
+    log_file = log_dir / f"{log_name}_{date_str}.log"
+
+    logger = logging.getLogger(name=log_name)
+    logger.setLevel(level=logging.INFO)
+    logger.handlers.clear()
+
+    # Rich console handler
+    rich_handler = RichHandler(
+        rich_tracebacks=True, show_time=True, show_level=True, show_path=True
+    )
+    rich_handler.setLevel(level=logging.INFO)
+
+    # File handler
+    file_handler = logging.FileHandler(filename=log_file, encoding="utf-8")
+    file_handler.setLevel(level=logging.INFO)
+    formatter = logging.Formatter(
+        fmt="%(asctime)s | %(levelname)s | %(name)s | %(message)s"
+    )
+    file_handler.setFormatter(formatter)
+
+    logger.addHandler(rich_handler)
+    logger.addHandler(file_handler)
+    logger.propagate = False
+
+    return logger

src/agent_hackathon/query_vector_db.py

@@ -0,0 +1,87 @@
+import os
+from typing import Any
+
+from dotenv import find_dotenv, load_dotenv
+from huggingface_hub import login
+from llama_index.core import VectorStoreIndex
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+from llama_index.llms.huggingface_api import HuggingFaceInferenceAPI
+from llama_index.vector_stores.milvus import MilvusVectorStore
+
+from src.agent_hackathon.consts import PROJECT_ROOT_DIR
+from src.agent_hackathon.logger import get_logger
+
+logger = get_logger(log_name="query_vector_db", log_dir=PROJECT_ROOT_DIR / "logs")
+
+
+class RetrieverEngineBuilder:
+    """
+    Handles the creation of a query engine for a vector database using HuggingFace and LlamaIndex.
+    """
+
+    def __init__(
+        self,
+        hf_token_env: str = "HF_TOKEN",
+        embedding_model: str = "Qwen/Qwen3-Embedding-0.6B",
+        llm_model: str = "meta-llama/Llama-4-Scout-17B-16E-Instruct",
+        vector_store: MilvusVectorStore = None,
+        device: str = "cpu",
+    ) -> None:
+        """
+        Initialize the QueryEngineBuilder.
+
+        Args:
+            hf_token_env: Environment variable name for HuggingFace token.
+            embedding_model: Name of the embedding model.
+            llm_model: Name of the LLM model.
+            vector_store: An instance of MilvusVectorStore.
+            device: Device to run the embedding model on.
+        """
+        self.hf_token_env = hf_token_env
+        self.embedding_model = embedding_model
+        self.llm_model = llm_model
+        self.vector_store = vector_store
+        self.device = device
+
+        logger.info("Initializing RetrieverEngineBuilder.")
+        self._login_huggingface()
+        self._load_env()
+
+        self.embed_model = HuggingFaceEmbedding(
+            model_name=self.embedding_model, device=self.device
+        )
+        self.llm = HuggingFaceInferenceAPI(
+            model=self.llm_model,
+            provider="auto",
+        )
+        logger.info("RetrieverEngineBuilder initialized.")
+
+    def _login_huggingface(self) -> None:
+        """Login to HuggingFace using the token from environment variable."""
+        logger.info("Logging in to HuggingFace.")
+        login(token=os.getenv(key=self.hf_token_env))
+        logger.info("Logged in to HuggingFace.")
+
+    def _load_env(self) -> None:
+        """Load environment variables from .env file."""
+        logger.info("Loading environment variables.")
+        _ = load_dotenv(dotenv_path=find_dotenv(raise_error_if_not_found=True))
+        logger.info("Environment variables loaded.")
+
+    def build_retriever_engine(self) -> Any:
+        """
+        Build and return the retriever engine.
+
+        Returns:
+            Retriever engine object.
+        """
+        logger.info("Building retriever engine.")
+        index = VectorStoreIndex.from_vector_store(
+            vector_store=self.vector_store, embed_model=self.embed_model
+        )
+        retriever = index.as_retriever(
+            vector_store_query_mode="hybrid",
+            similarity_top_k=5,
+        )
+        logger.info("Retriever engine built.")
+        return retriever