{ "cells": [ { "cell_type": "markdown", "id": "bf597549", "metadata": {}, "source": [ "PRIMERO PREPARAMOS TODAS NUESTRAS FUNCIONES PARA PODER SER CONVOCADAS LUEGO." ] }, { "cell_type": "markdown", "id": "7968949c", "metadata": {}, "source": [ "Instalamos " ] }, { "cell_type": "code", "execution_count": 1, "id": "9a192af6", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Collecting qdrant-client\n", " Obtaining dependency information for qdrant-client from https://files.pythonhosted.org/packages/e4/52/f49b0aa96253010f57cf80315edecec4f469e7a39c1ed92bf727fa290e57/qdrant_client-1.14.2-py3-none-any.whl.metadata\n", " Downloading qdrant_client-1.14.2-py3-none-any.whl.metadata (10 kB)\n", "Requirement already satisfied: transformers in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (4.51.3)\n", "Requirement already satisfied: torch in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (2.7.0)\n", "Requirement already satisfied: langchain in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (0.3.24)\n", "Collecting pymupdf\n", " Obtaining dependency information for pymupdf from https://files.pythonhosted.org/packages/71/c2/a9059607f80dcaf2392f991748cfc53456820392c0220cff02572653512a/pymupdf-1.25.5-cp39-abi3-win_amd64.whl.metadata\n", " Downloading pymupdf-1.25.5-cp39-abi3-win_amd64.whl.metadata (3.4 kB)\n", "Collecting grpcio>=1.41.0 (from qdrant-client)\n", " Obtaining dependency information for grpcio>=1.41.0 from https://files.pythonhosted.org/packages/ee/3f/cf92e7e62ccb8dbdf977499547dfc27133124d6467d3a7d23775bcecb0f9/grpcio-1.71.0-cp311-cp311-win_amd64.whl.metadata\n", " Using cached grpcio-1.71.0-cp311-cp311-win_amd64.whl.metadata (4.0 kB)\n", "Requirement already satisfied: httpx[http2]>=0.20.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from qdrant-client) (0.28.1)\n", "Requirement already satisfied: numpy>=1.21 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from qdrant-client) (2.2.5)\n", "Collecting portalocker<3.0.0,>=2.7.0 (from qdrant-client)\n", " Obtaining dependency information for portalocker<3.0.0,>=2.7.0 from https://files.pythonhosted.org/packages/9b/fb/a70a4214956182e0d7a9099ab17d50bfcba1056188e9b14f35b9e2b62a0d/portalocker-2.10.1-py3-none-any.whl.metadata\n", " Using cached portalocker-2.10.1-py3-none-any.whl.metadata (8.5 kB)\n", "Collecting protobuf>=3.20.0 (from qdrant-client)\n", " Obtaining dependency information for protobuf>=3.20.0 from https://files.pythonhosted.org/packages/97/e9/7b9f1b259d509aef2b833c29a1f3c39185e2bf21c9c1be1cd11c22cb2149/protobuf-6.30.2-cp310-abi3-win_amd64.whl.metadata\n", " Downloading protobuf-6.30.2-cp310-abi3-win_amd64.whl.metadata (593 bytes)\n", "Requirement already satisfied: pydantic!=2.0.*,!=2.1.*,!=2.2.0,>=1.10.8 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from qdrant-client) (2.11.4)\n", "Requirement already satisfied: urllib3<3,>=1.26.14 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from qdrant-client) (2.4.0)\n", "Requirement already satisfied: filelock in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from transformers) (3.18.0)\n", "Requirement already satisfied: huggingface-hub<1.0,>=0.30.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from transformers) (0.30.2)\n", "Requirement already satisfied: packaging>=20.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from transformers) (24.2)\n", "Requirement already satisfied: pyyaml>=5.1 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from transformers) (6.0.2)\n", "Requirement already satisfied: regex!=2019.12.17 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from transformers) (2024.11.6)\n", "Requirement already satisfied: requests in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from transformers) (2.32.3)\n", "Requirement already satisfied: tokenizers<0.22,>=0.21 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from transformers) (0.21.1)\n", "Requirement already satisfied: safetensors>=0.4.3 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from transformers) (0.5.3)\n", "Requirement already satisfied: tqdm>=4.27 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from transformers) (4.67.1)\n", "Requirement already satisfied: typing-extensions>=4.10.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from torch) (4.13.2)\n", "Requirement already satisfied: sympy>=1.13.3 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from torch) (1.14.0)\n", "Requirement already satisfied: networkx in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from torch) (3.4.2)\n", "Requirement already satisfied: jinja2 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from torch) (3.1.6)\n", "Requirement already satisfied: fsspec in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from torch) (2025.3.2)\n", "Requirement already satisfied: langchain-core<1.0.0,>=0.3.55 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from langchain) (0.3.56)\n", "Requirement already satisfied: langchain-text-splitters<1.0.0,>=0.3.8 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from langchain) (0.3.8)\n", "Requirement already satisfied: langsmith<0.4,>=0.1.17 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from langchain) (0.3.39)\n", "Requirement already satisfied: SQLAlchemy<3,>=1.4 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from langchain) (2.0.40)\n", "Requirement already satisfied: anyio in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from httpx[http2]>=0.20.0->qdrant-client) (4.9.0)\n", "Requirement already satisfied: certifi in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from httpx[http2]>=0.20.0->qdrant-client) (2025.4.26)\n", "Requirement already satisfied: httpcore==1.* in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from httpx[http2]>=0.20.0->qdrant-client) (1.0.9)\n", "Requirement already satisfied: idna in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from httpx[http2]>=0.20.0->qdrant-client) (3.10)\n", "Collecting h2<5,>=3 (from httpx[http2]>=0.20.0->qdrant-client)\n", " Obtaining dependency information for h2<5,>=3 from https://files.pythonhosted.org/packages/d0/9e/984486f2d0a0bd2b024bf4bc1c62688fcafa9e61991f041fb0e2def4a982/h2-4.2.0-py3-none-any.whl.metadata\n", " Using cached h2-4.2.0-py3-none-any.whl.metadata (5.1 kB)\n", "Requirement already satisfied: h11>=0.16 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from httpcore==1.*->httpx[http2]>=0.20.0->qdrant-client) (0.16.0)\n", "Requirement already satisfied: tenacity!=8.4.0,<10.0.0,>=8.1.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from langchain-core<1.0.0,>=0.3.55->langchain) (9.1.2)\n", "Requirement already satisfied: jsonpatch<2.0,>=1.33 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from langchain-core<1.0.0,>=0.3.55->langchain) (1.33)\n", "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from langsmith<0.4,>=0.1.17->langchain) (3.10.18)\n", "Requirement already satisfied: requests-toolbelt<2.0.0,>=1.0.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from langsmith<0.4,>=0.1.17->langchain) (1.0.0)\n", "Requirement already satisfied: zstandard<0.24.0,>=0.23.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from langsmith<0.4,>=0.1.17->langchain) (0.23.0)\n", "Requirement already satisfied: pywin32>=226 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from portalocker<3.0.0,>=2.7.0->qdrant-client) (310)\n", "Requirement already satisfied: annotated-types>=0.6.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.0,>=1.10.8->qdrant-client) (0.7.0)\n", "Requirement already satisfied: pydantic-core==2.33.2 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.0,>=1.10.8->qdrant-client) (2.33.2)\n", "Requirement already satisfied: typing-inspection>=0.4.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from pydantic!=2.0.*,!=2.1.*,!=2.2.0,>=1.10.8->qdrant-client) (0.4.0)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from requests->transformers) (3.4.1)\n", "Requirement already satisfied: greenlet>=1 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from SQLAlchemy<3,>=1.4->langchain) (3.2.1)\n", "Requirement already satisfied: mpmath<1.4,>=1.1.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from sympy>=1.13.3->torch) (1.3.0)\n", "Requirement already satisfied: colorama in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from tqdm>=4.27->transformers) (0.4.6)\n", "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from jinja2->torch) (3.0.2)\n", "Collecting hyperframe<7,>=6.1 (from h2<5,>=3->httpx[http2]>=0.20.0->qdrant-client)\n", " Obtaining dependency information for hyperframe<7,>=6.1 from https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl.metadata\n", " Using cached hyperframe-6.1.0-py3-none-any.whl.metadata (4.3 kB)\n", "Collecting hpack<5,>=4.1 (from h2<5,>=3->httpx[http2]>=0.20.0->qdrant-client)\n", " Obtaining dependency information for hpack<5,>=4.1 from https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl.metadata\n", " Using cached hpack-4.1.0-py3-none-any.whl.metadata (4.6 kB)\n", "Requirement already satisfied: jsonpointer>=1.9 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from jsonpatch<2.0,>=1.33->langchain-core<1.0.0,>=0.3.55->langchain) (3.0.0)\n", "Requirement already satisfied: sniffio>=1.1 in c:\\users\\adm\\documents\\rag_milvus\\.venv\\lib\\site-packages (from anyio->httpx[http2]>=0.20.0->qdrant-client) (1.3.1)\n", "Downloading qdrant_client-1.14.2-py3-none-any.whl (327 kB)\n", " ---------------------------------------- 0.0/327.7 kB ? eta -:--:--\n", " -------------- ------------------------- 122.9/327.7 kB 3.6 MB/s eta 0:00:01\n", " ---------------------------------------- 327.7/327.7 kB 5.1 MB/s eta 0:00:00\n", "Downloading pymupdf-1.25.5-cp39-abi3-win_amd64.whl (16.6 MB)\n", " ---------------------------------------- 0.0/16.6 MB ? eta -:--:--\n", " - -------------------------------------- 0.5/16.6 MB 14.4 MB/s eta 0:00:02\n", " -- ------------------------------------- 1.0/16.6 MB 12.8 MB/s eta 0:00:02\n", " ---- ----------------------------------- 1.8/16.6 MB 14.0 MB/s eta 0:00:02\n", " ------ --------------------------------- 2.5/16.6 MB 14.6 MB/s eta 0:00:01\n", " ------- -------------------------------- 3.3/16.6 MB 15.0 MB/s eta 0:00:01\n", " --------- ------------------------------ 4.0/16.6 MB 14.9 MB/s eta 0:00:01\n", " ----------- ---------------------------- 4.7/16.6 MB 15.1 MB/s eta 0:00:01\n", " ------------- -------------------------- 5.4/16.6 MB 15.1 MB/s eta 0:00:01\n", " -------------- ------------------------- 6.2/16.6 MB 15.1 MB/s eta 0:00:01\n", " ---------------- ----------------------- 6.9/16.6 MB 15.2 MB/s eta 0:00:01\n", " ------------------ --------------------- 7.6/16.6 MB 15.2 MB/s eta 0:00:01\n", " -------------------- ------------------- 8.3/16.6 MB 15.6 MB/s eta 0:00:01\n", " --------------------- ------------------ 9.1/16.6 MB 15.7 MB/s eta 0:00:01\n", " ----------------------- ---------------- 9.8/16.6 MB 15.6 MB/s eta 0:00:01\n", " ------------------------- -------------- 10.4/16.6 MB 15.6 MB/s eta 0:00:01\n", " -------------------------- ------------- 11.0/16.6 MB 15.6 MB/s eta 0:00:01\n", " ---------------------------- ----------- 11.8/16.6 MB 15.6 MB/s eta 0:00:01\n", " ------------------------------ --------- 12.5/16.6 MB 15.6 MB/s eta 0:00:01\n", " ------------------------------- -------- 13.2/16.6 MB 15.6 MB/s eta 0:00:01\n", " --------------------------------- ------ 13.9/16.6 MB 15.6 MB/s eta 0:00:01\n", " ----------------------------------- ---- 14.6/16.6 MB 15.2 MB/s eta 0:00:01\n", " ------------------------------------ --- 15.3/16.6 MB 15.2 MB/s eta 0:00:01\n", " -------------------------------------- - 16.1/16.6 MB 16.0 MB/s eta 0:00:01\n", " --------------------------------------- 16.6/16.6 MB 15.6 MB/s eta 0:00:01\n", " ---------------------------------------- 16.6/16.6 MB 15.2 MB/s eta 0:00:00\n", "Using cached grpcio-1.71.0-cp311-cp311-win_amd64.whl (4.3 MB)\n", "Using cached portalocker-2.10.1-py3-none-any.whl (18 kB)\n", "Downloading protobuf-6.30.2-cp310-abi3-win_amd64.whl (431 kB)\n", " ---------------------------------------- 0.0/431.0 kB ? eta -:--:--\n", " --------------------------------------- 431.0/431.0 kB 26.3 MB/s eta 0:00:00\n", "Using cached h2-4.2.0-py3-none-any.whl (60 kB)\n", "Using cached hpack-4.1.0-py3-none-any.whl (34 kB)\n", "Using cached hyperframe-6.1.0-py3-none-any.whl (13 kB)\n", "Installing collected packages: pymupdf, protobuf, portalocker, hyperframe, hpack, grpcio, h2, qdrant-client\n", "Successfully installed grpcio-1.71.0 h2-4.2.0 hpack-4.1.0 hyperframe-6.1.0 portalocker-2.10.1 protobuf-6.30.2 pymupdf-1.25.5 qdrant-client-1.14.2\n", "Note: you may need to restart the kernel to use updated packages.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\n", "[notice] A new release of pip is available: 23.2.1 -> 25.1\n", "[notice] To update, run: python.exe -m pip install --upgrade pip\n" ] } ], "source": [ "%pip install qdrant-client transformers torch langchain pymupdf" ] }, { "cell_type": "markdown", "id": "a4833977", "metadata": {}, "source": [ "Importamos librerias" ] }, { "cell_type": "code", "execution_count": 19, "id": "1684b4de", "metadata": {}, "outputs": [], "source": [ "from sentence_transformers import SentenceTransformer\n", "from langchain.text_splitter import RecursiveCharacterTextSplitter\n", "from langchain.schema import Document\n", "from qdrant_client import QdrantClient\n", "from qdrant_client.models import PointStruct, Distance, VectorParams\n", "import fitz # PyMuPDF" ] }, { "cell_type": "markdown", "id": "e612e674", "metadata": {}, "source": [ "Definimos funciones\n", "1) Cargar los pdf por sus bloques de paginas" ] }, { "cell_type": "code", "execution_count": 20, "id": "5a594ed8", "metadata": {}, "outputs": [], "source": [ "def pdfachunk(path, chunk_size_pages=20):\n", " doc = fitz.open(path)\n", " chunks = []\n", " for i in range(0, len(doc), chunk_size_pages):\n", " text = \"\"\n", " for page_num in range(i, min(i + chunk_size_pages, len(doc))):\n", " text += doc[page_num].get_text()\n", " chunks.append(text)\n", " doc.close()\n", " return chunks" ] }, { "cell_type": "markdown", "id": "59d048b7", "metadata": {}, "source": [ "2) Dividir texto en chunks más pequeños con solapamiento" ] }, { "cell_type": "code", "execution_count": 21, "id": "bffac6eb", "metadata": {}, "outputs": [], "source": [ "def split_chunks(raw_chunks, chunk_size=1024, chunk_overlap=100):\n", " docs = [Document(page_content=chunk) for chunk in raw_chunks]\n", " splitter = RecursiveCharacterTextSplitter(\n", " chunk_size=chunk_size,\n", " chunk_overlap=chunk_overlap,\n", " separators=[\"\\n\\n\", \"\\n\", \".\", \" \"]\n", " )\n", " return splitter.split_documents(docs)" ] }, { "cell_type": "markdown", "id": "8664bf6f", "metadata": {}, "source": [ "3) Generar embeddings en batch" ] }, { "cell_type": "code", "execution_count": 22, "id": "35a4df0b", "metadata": {}, "outputs": [], "source": [ "def generaremben(model, texts):\n", " texts = [t for t in texts if t.strip()] # filtra vacíos\n", " if not texts:\n", " raise ValueError(\"No hay textos válidos para generar embeddings.\")\n", " return model.encode(texts, batch_size=16, show_progress_bar=True)\n" ] }, { "cell_type": "markdown", "id": "c28a5724", "metadata": {}, "source": [ "4) Insertar los docemtos en QDRANT localemente" ] }, { "cell_type": "code", "execution_count": 23, "id": "3c61ddca", "metadata": {}, "outputs": [], "source": [ "def insertarenqdra(embeddings, texts, collection_name=\"pdf_chunks\"):\n", " client = QdrantClient(path=\"./qdrant_data\") # persistente\n", "\n", " dim = len(embeddings[0])\n", " client.recreate_collection(\n", " collection_name=collection_name,\n", " vectors_config=VectorParams(size=dim, distance=Distance.COSINE)\n", " )\n", "\n", " points = [\n", " PointStruct(id=i, vector=embeddings[i].tolist(), payload={\"text\": texts[i]})\n", " for i in range(len(embeddings))\n", " ]\n", "\n", " client.upsert(collection_name=collection_name, points=points)\n", " print(f\"✅ Insertados {len(points)} vectores en Qdrant.\")" ] }, { "cell_type": "markdown", "id": "566e06c8", "metadata": {}, "source": [ "5) Funcion modelo para no cargarlo siempre" ] }, { "cell_type": "code", "execution_count": 6, "id": "eec86477", "metadata": {}, "outputs": [], "source": [ "def load_nv_model():\n", " return AutoModel.from_pretrained(\"nvidia/NV-Embed-v2\", trust_remote_code=True)" ] }, { "cell_type": "markdown", "id": "08510d36", "metadata": {}, "source": [ "Probamos" ] }, { "cell_type": "code", "execution_count": 24, "id": "2735d1a1", "metadata": {}, "outputs": [], "source": [ "pdf_path=\"./DOCS/Decreto-Supremo-N_-018-2019-JUS.pdf\" " ] }, { "cell_type": "code", "execution_count": 25, "id": "6ede8122", "metadata": {}, "outputs": [], "source": [ "pdf_chunks = pdfachunk(pdf_path)" ] }, { "cell_type": "code", "execution_count": 26, "id": "8f33af13", "metadata": {}, "outputs": [], "source": [ "split_docs = split_chunks(pdf_chunks)" ] }, { "cell_type": "code", "execution_count": 27, "id": "b0fb3761", "metadata": {}, "outputs": [], "source": [ "texts = [doc.page_content for doc in split_docs]" ] }, { "cell_type": "markdown", "id": "85f2ee3f", "metadata": {}, "source": [ "Definimos nuestro modelo de embbending" ] }, { "cell_type": "markdown", "id": "5eb18c36", "metadata": {}, "source": [ "NECESITAMOS DATASETS Y EINOPS" ] }, { "cell_type": "code", "execution_count": null, "id": "74262eaa", "metadata": {}, "outputs": [], "source": [ "%pip install datasets einops" ] }, { "cell_type": "code", "execution_count": 28, "id": "93bbbcde", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Xet Storage is enabled for this repo, but the 'hf_xet' package is not installed. Falling back to regular HTTP download. For better performance, install the package with: `pip install huggingface_hub[hf_xet]` or `pip install hf_xet`\n" ] } ], "source": [ "model = SentenceTransformer(\"all-MiniLM-L6-v2\")" ] }, { "cell_type": "code", "execution_count": 13, "id": "f9f4d5bd", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "\n" ] } ], "source": [ "print(type(model))" ] }, { "cell_type": "code", "execution_count": 29, "id": "e594d6af", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Batches: 100%|██████████| 12/12 [00:03<00:00, 3.29it/s]\n" ] } ], "source": [ "embeddings = generaremben(model, texts)" ] }, { "cell_type": "code", "execution_count": 30, "id": "beba3991", "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "C:\\Users\\adm\\AppData\\Local\\Temp\\ipykernel_26272\\752761030.py:5: DeprecationWarning: `recreate_collection` method is deprecated and will be removed in the future. Use `collection_exists` to check collection existence and `create_collection` instead.\n", " client.recreate_collection(\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "✅ Insertados 181 vectores en Qdrant.\n" ] } ], "source": [ "insertarenqdra(embeddings, texts, collection_name=\"jus_decreto_018\")" ] }, { "cell_type": "markdown", "id": "2c241dc6", "metadata": {}, "source": [ "Funcion para consultar con qdrant" ] }, { "cell_type": "code", "execution_count": 31, "id": "86beaf73", "metadata": {}, "outputs": [], "source": [ "from qdrant_client import QdrantClient\n", "import numpy as np" ] }, { "cell_type": "code", "execution_count": null, "id": "b81de586", "metadata": {}, "outputs": [], "source": [ "def query_qdrant(query, model, collection_name, top_k=5):\n", " # Generar embedding de la consulta\n", " query_embedding = model.encode([query])[0]\n", " \n", " # Conexión al cliente Qdrant\n", " client = QdrantClient(path=\"./qdrant_data\")\n", "\n", " # Realizar búsqueda en la colección\n", " results = client.search(\n", " collection_name=collection_name,\n", " query_vector=query_embedding.tolist(),\n", " limit=top_k, # Limitar a los primeros K resultados más similares\n", " with_payload=True # Incluir el texto en los resultados\n", " )\n", "\n", " return results" ] }, { "cell_type": "code", "execution_count": 36, "id": "9d448736", "metadata": {}, "outputs": [ { "ename": "RuntimeError", "evalue": "Storage folder ./qdrant_data is already accessed by another instance of Qdrant client. If you require concurrent access, use Qdrant server instead.", "output_type": "error", "traceback": [ "\u001b[31m---------------------------------------------------------------------------\u001b[39m", "\u001b[31merror\u001b[39m Traceback (most recent call last)", "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\adm\\Documents\\rag_MILVUS\\.venv\\Lib\\site-packages\\portalocker\\portalocker.py:49\u001b[39m, in \u001b[36mlock\u001b[39m\u001b[34m(file_, flags)\u001b[39m\n\u001b[32m 48\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m---> \u001b[39m\u001b[32m49\u001b[39m \u001b[43mwin32file\u001b[49m\u001b[43m.\u001b[49m\u001b[43mLockFileEx\u001b[49m\u001b[43m(\u001b[49m\u001b[43mos_fh\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[32;43m0\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m-\u001b[49m\u001b[32;43m0x10000\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m__overlapped\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 50\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m pywintypes.error \u001b[38;5;28;01mas\u001b[39;00m exc_value:\n\u001b[32m 51\u001b[39m \u001b[38;5;66;03m# error: (33, 'LockFileEx', 'The process cannot access the file\u001b[39;00m\n\u001b[32m 52\u001b[39m \u001b[38;5;66;03m# because another process has locked a portion of the file.')\u001b[39;00m\n", "\u001b[31merror\u001b[39m: (33, 'LockFileEx', 'El proceso no tiene acceso al archivo porque otro proceso tiene bloqueada una parte del archivo.')", "\nThe above exception was the direct cause of the following exception:\n", "\u001b[31mAlreadyLocked\u001b[39m Traceback (most recent call last)", "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\adm\\Documents\\rag_MILVUS\\.venv\\Lib\\site-packages\\qdrant_client\\local\\qdrant_local.py:133\u001b[39m, in \u001b[36mQdrantLocal._load\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 132\u001b[39m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m133\u001b[39m \u001b[43mportalocker\u001b[49m\u001b[43m.\u001b[49m\u001b[43mlock\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 134\u001b[39m \u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_flock_file\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 135\u001b[39m \u001b[43m \u001b[49m\u001b[43mportalocker\u001b[49m\u001b[43m.\u001b[49m\u001b[43mLockFlags\u001b[49m\u001b[43m.\u001b[49m\u001b[43mEXCLUSIVE\u001b[49m\u001b[43m \u001b[49m\u001b[43m|\u001b[49m\u001b[43m \u001b[49m\u001b[43mportalocker\u001b[49m\u001b[43m.\u001b[49m\u001b[43mLockFlags\u001b[49m\u001b[43m.\u001b[49m\u001b[43mNON_BLOCKING\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 136\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 137\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m portalocker.exceptions.LockException:\n", "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\adm\\Documents\\rag_MILVUS\\.venv\\Lib\\site-packages\\portalocker\\portalocker.py:54\u001b[39m, in \u001b[36mlock\u001b[39m\u001b[34m(file_, flags)\u001b[39m\n\u001b[32m 53\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m exc_value.winerror == winerror.ERROR_LOCK_VIOLATION:\n\u001b[32m---> \u001b[39m\u001b[32m54\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m exceptions.AlreadyLocked(\n\u001b[32m 55\u001b[39m exceptions.LockException.LOCK_FAILED,\n\u001b[32m 56\u001b[39m exc_value.strerror,\n\u001b[32m 57\u001b[39m fh=file_,\n\u001b[32m 58\u001b[39m ) \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mexc_value\u001b[39;00m\n\u001b[32m 59\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 60\u001b[39m \u001b[38;5;66;03m# Q: Are there exceptions/codes we should be dealing with\u001b[39;00m\n\u001b[32m 61\u001b[39m \u001b[38;5;66;03m# here?\u001b[39;00m\n", "\u001b[31mAlreadyLocked\u001b[39m: (1, 'El proceso no tiene acceso al archivo porque otro proceso tiene bloqueada una parte del archivo.')", "\nDuring handling of the above exception, another exception occurred:\n", "\u001b[31mRuntimeError\u001b[39m Traceback (most recent call last)", "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[36]\u001b[39m\u001b[32m, line 2\u001b[39m\n\u001b[32m 1\u001b[39m query = \u001b[33m\"\u001b[39m\u001b[33m¿Cuál es el propósito de la Ley 018-2019?\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m results = \u001b[43mquery_qdrant\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquery\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mmodel\u001b[49m\u001b[43m,\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43mjus_decreto_018\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[34]\u001b[39m\u001b[32m, line 6\u001b[39m, in \u001b[36mquery_qdrant\u001b[39m\u001b[34m(query, model, collection_name, top_k)\u001b[39m\n\u001b[32m 3\u001b[39m query_embedding = model.encode([query])[\u001b[32m0\u001b[39m]\n\u001b[32m 5\u001b[39m \u001b[38;5;66;03m# Conexión al cliente Qdrant\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m6\u001b[39m client = \u001b[43mQdrantClient\u001b[49m\u001b[43m(\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m=\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m./qdrant_data\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[32m 8\u001b[39m \u001b[38;5;66;03m# Realizar búsqueda en la colección\u001b[39;00m\n\u001b[32m 9\u001b[39m results = client.search(\n\u001b[32m 10\u001b[39m collection_name=collection_name,\n\u001b[32m 11\u001b[39m query_vector=query_embedding.tolist(),\n\u001b[32m 12\u001b[39m limit=top_k, \u001b[38;5;66;03m# Limitar a los primeros K resultados más similares\u001b[39;00m\n\u001b[32m 13\u001b[39m with_payload=\u001b[38;5;28;01mTrue\u001b[39;00m \u001b[38;5;66;03m# Incluir el texto en los resultados\u001b[39;00m\n\u001b[32m 14\u001b[39m )\n", "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\adm\\Documents\\rag_MILVUS\\.venv\\Lib\\site-packages\\qdrant_client\\qdrant_client.py:133\u001b[39m, in \u001b[36mQdrantClient.__init__\u001b[39m\u001b[34m(self, location, url, port, grpc_port, prefer_grpc, https, api_key, prefix, timeout, host, path, force_disable_check_same_thread, grpc_options, auth_token_provider, cloud_inference, local_inference_batch_size, check_compatibility, **kwargs)\u001b[39m\n\u001b[32m 128\u001b[39m \u001b[38;5;28mself\u001b[39m._client = QdrantLocal(\n\u001b[32m 129\u001b[39m location=location,\n\u001b[32m 130\u001b[39m force_disable_check_same_thread=force_disable_check_same_thread,\n\u001b[32m 131\u001b[39m )\n\u001b[32m 132\u001b[39m \u001b[38;5;28;01melif\u001b[39;00m path \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m133\u001b[39m \u001b[38;5;28mself\u001b[39m._client = \u001b[43mQdrantLocal\u001b[49m\u001b[43m(\u001b[49m\n\u001b[32m 134\u001b[39m \u001b[43m \u001b[49m\u001b[43mlocation\u001b[49m\u001b[43m=\u001b[49m\u001b[43mpath\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 135\u001b[39m \u001b[43m \u001b[49m\u001b[43mforce_disable_check_same_thread\u001b[49m\u001b[43m=\u001b[49m\u001b[43mforce_disable_check_same_thread\u001b[49m\u001b[43m,\u001b[49m\n\u001b[32m 136\u001b[39m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 137\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 138\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m location \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m url \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n", "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\adm\\Documents\\rag_MILVUS\\.venv\\Lib\\site-packages\\qdrant_client\\local\\qdrant_local.py:66\u001b[39m, in \u001b[36mQdrantLocal.__init__\u001b[39m\u001b[34m(self, location, force_disable_check_same_thread)\u001b[39m\n\u001b[32m 64\u001b[39m \u001b[38;5;28mself\u001b[39m.aliases: \u001b[38;5;28mdict\u001b[39m[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;28mstr\u001b[39m] = {}\n\u001b[32m 65\u001b[39m \u001b[38;5;28mself\u001b[39m._flock_file: Optional[TextIOWrapper] = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m66\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_load\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 67\u001b[39m \u001b[38;5;28mself\u001b[39m._closed: \u001b[38;5;28mbool\u001b[39m = \u001b[38;5;28;01mFalse\u001b[39;00m\n", "\u001b[36mFile \u001b[39m\u001b[32mc:\\Users\\adm\\Documents\\rag_MILVUS\\.venv\\Lib\\site-packages\\qdrant_client\\local\\qdrant_local.py:138\u001b[39m, in \u001b[36mQdrantLocal._load\u001b[39m\u001b[34m(self)\u001b[39m\n\u001b[32m 133\u001b[39m portalocker.lock(\n\u001b[32m 134\u001b[39m \u001b[38;5;28mself\u001b[39m._flock_file,\n\u001b[32m 135\u001b[39m portalocker.LockFlags.EXCLUSIVE | portalocker.LockFlags.NON_BLOCKING,\n\u001b[32m 136\u001b[39m )\n\u001b[32m 137\u001b[39m \u001b[38;5;28;01mexcept\u001b[39;00m portalocker.exceptions.LockException:\n\u001b[32m--> \u001b[39m\u001b[32m138\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[32m 139\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mStorage folder \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mself\u001b[39m.location\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m is already accessed by another instance of Qdrant client.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 140\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33m If you require concurrent access, use Qdrant server instead.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 141\u001b[39m )\n", "\u001b[31mRuntimeError\u001b[39m: Storage folder ./qdrant_data is already accessed by another instance of Qdrant client. If you require concurrent access, use Qdrant server instead." ] } ], "source": [ "query = \"¿Cuál es el propósito de la Ley 018-2019?\"\n", "results = query_qdrant(query, model,\"jus_decreto_018\")" ] }, { "cell_type": "code", "execution_count": null, "id": "61d76427", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": ".venv", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.5" } }, "nbformat": 4, "nbformat_minor": 5 }