First app commit

.env.example

@@ -0,0 +1,3 @@
+# HuggingFace API Token (required for LLM access)
+# Get your token from: https://huggingface.co/settings/tokens
+HF_TOKEN=your_huggingface_token_here

.gitattributes

@@ -1,35 +1,4 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
 *.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+faiss_index/** filter=lfs diff=lfs merge=lfs -text
+*.index filter=lfs diff=lfs merge=lfs -text
+*.faiss filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,46 @@
+# Python
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+.Python
+env/
+venv/
+ENV/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Environment
+.env
+.env.local
+
+# IDE
+.vscode/
+.idea/
+*.swp
+*.swo
+*~
+
+# OS
+.DS_Store
+Thumbs.db
+
+# Gradio
+flagged/
+
+# Model cache
+.cache/
+

README.md

@@ -8,7 +8,5 @@ sdk_version: 5.49.1
 app_file: app.py
 pinned: false
 license: mit
-short_description: Demo space for NoterLLM, the RAG system for Turkish notaries
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+short_description: AI-powered Turkish Notary Law Assistant using RAG
+---

app.py

@@ -0,0 +1,191 @@
+import gradio as gr
+from llm_rag_setup import query_rag
+
+custom_css = """
+.container {
+    max-width: 1200px;
+    margin: auto;
+}
+.source-box {
+    background-color: #f0f0f0;
+    padding: 10px;
+    border-radius: 5px;
+    margin: 5px 0;
+    border-left: 3px solid #2196F3;
+}
+.footer {
+    text-align: center;
+    margin-top: 20px;
+    color: #666;
+}
+"""
+
+
+def format_sources(source_documents):
+    if not source_documents:
+        return "Kaynak bulunamadı."
+
+    sources_html = ""
+    for i, doc in enumerate(source_documents[:3], 1):
+        metadata = doc.metadata
+        source_type = metadata.get("source_type", "genelge")
+
+        if source_type == "kanun":
+            madde_no = metadata.get("madde_no", "N/A")
+            madde_baslik = metadata.get("madde_baslik", "")
+            title = f"📜 Noterlik Kanunu - Madde {madde_no}"
+            if madde_baslik:
+                title += f" ({madde_baslik})"
+            kisim = metadata.get("kisim", "")
+            content = f"{kisim}\n\n{doc.page_content[:200]}..."
+        else:
+            genelge_no = metadata.get("genelge_no", "N/A")
+            madde_no = metadata.get("madde_no", "N/A")
+            title = f"📋 Genelge {genelge_no} - Madde {madde_no}"
+            genelge_baslik = metadata.get("genelge_baslik", "N/A")
+            content = f"{genelge_baslik}\n\n{doc.page_content[:200]}..."
+
+        sources_html += f"""
+<div class="source-box">
+    <strong>{i}. {title}</strong><br>
+    <small>{content}</small>
+</div>
+"""
+
+    return sources_html
+
+
+def chat_with_rag(message, history):
+    if not message.strip():
+        return "", history
+
+    try:
+        history.append((message, "⚙️ Sistemi başlatıyor — lütfen bekleyin..."))
+        result = query_rag(message)
+        if result is None:
+            answer = "❌ Sistem başlatılamadı veya veri eksik. Lütfen sunucu günlüklerini kontrol edin."
+        else:
+            answer = result.get("result", "(Cevap alınamadı)")
+
+        sources_html = ""
+        if result and "source_documents" in result and result["source_documents"]:
+            sources_html = (
+                "<br><br><strong>📚 Kaynaklar:</strong><br>"
+                + format_sources(result["source_documents"])
+            )
+
+        full_response = answer + sources_html
+
+
+        if history:
+            history[-1] = (message, full_response)
+        else:
+            history.append((message, full_response))
+
+        return "", history
+
+    except Exception as e:
+        error_message = f"❌ Hata oluştu: {str(e)}"
+        history.append((message, error_message))
+        return "", history
+
+
+def clear_chat():
+    return [], []
+
+
+examples = [
+    "Araç satış işlemlerinde hangi belgeler gereklidir?",
+    "Noterlik işlemlerinde harç ve karar pulu nasıl hesaplanır?",
+    "Vekaletname düzenlenirken dikkat edilmesi gereken hususlar nelerdir?",
+    "Gayrimenkul satış vaadi sözleşmesi nedir?",
+]
+
+
+with gr.Blocks(css=custom_css, theme=gr.themes.Soft()) as demo:
+    gr.Markdown(
+        """
+        # ⚖️ NoterLLM - Türk Noter Hukuku Asistanı
+        
+        Noterlik Kanunu ve Türkiye Noterler Birliği genelgelerine dayalı AI destekli soru-cevap sistemi.
+        
+        **Kaynaklar:**
+        - 📜 Noterlik Kanunu (1512) - 213 madde
+        - 📋 TNB Genelgeleri - 125+ genelge
+        
+        *Bu sistem genelgelere dayalı bilgi sağlar ancak resmi hukuki danışmanlık yerine geçmez.*
+        """
+    )
+
+    with gr.Row():
+        with gr.Column(scale=4):
+            chatbot = gr.Chatbot(
+                label="Sohbet Geçmişi",
+                height=500,
+                show_label=True,
+                avatar_images=("👤", "⚖️"),
+                bubble_full_width=False,
+            )
+
+            with gr.Row():
+                msg = gr.Textbox(
+                    label="Sorunuz",
+                    placeholder="Noterlik hukuku ile ilgili sorunuzu yazın...",
+                    show_label=False,
+                    scale=9,
+                    container=False,
+                )
+                submit_btn = gr.Button("Gönder", variant="primary", scale=1)
+
+            clear_btn = gr.Button("🗑️ Sohbeti Temizle", size="sm")
+
+        with gr.Column(scale=1):
+            gr.Markdown("### 💡 Örnek Sorular")
+            gr.Examples(
+                examples=examples,
+                inputs=msg,
+                label="Aşağıdaki sorulardan birini seçebilirsiniz:",
+            )
+
+            gr.Markdown(
+                """
+                ### ℹ️ Bilgi
+                
+                **Model:** Mistral-7B-Instruct-v0.2  
+                **Embedding:** multilingual-e5-base  
+                **Retrieval:** FAISS + BM25 (Hybrid)
+                
+                **Özellikler:**
+                - 🔍 Semantic & Keyword Search
+                - 📚 Kaynak Referansları
+                - 🇹🇷 Türkçe Optimizasyonu
+                """
+            )
+
+    submit_btn.click(
+        fn=chat_with_rag,
+        inputs=[msg, chatbot],
+        outputs=[msg, chatbot],
+    )
+
+    msg.submit(
+        fn=chat_with_rag,
+        inputs=[msg, chatbot],
+        outputs=[msg, chatbot],
+    )
+
+    clear_btn.click(
+        fn=clear_chat,
+        inputs=None,
+        outputs=[msg, chatbot],
+    )
+
+    gr.Markdown(
+        """
+        <div class="footer">
+            <p>Powered by HuggingFace 🤗 | Built with Gradio</p>
+        </div>
+        """
+    )
+
+demo.launch()

bm25_retriever.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee3d727748117c15f270eee4c7dc5b7be2df510e3be48fa172ac2b0db6aa391e
+size 16329768

faiss_index/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:075350f184b5ee60e6aab7a1a020a330554134bf2e5d9823e7d16b7b3b9baf9c
+size 13283373

faiss_index/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0fed25d8c490d29c0c3795b175c93d8bb7ba157b16af7c147969fbb5ecb0a329
+size 9677628

llm_rag_setup.py

@@ -0,0 +1,210 @@
+from langchain.schema import Document
+import json
+import os
+import pickle
+from langchain_community.vectorstores import FAISS
+from langchain_huggingface import HuggingFaceEmbeddings
+from langchain_community.retrievers import BM25Retriever
+from langchain.retrievers import EnsembleRetriever
+from langchain_huggingface import HuggingFaceEndpoint
+from langchain.chains import RetrievalQA
+from langchain.prompts import PromptTemplate
+from typing import Optional
+
+_qa_chain: Optional[RetrievalQA] = None
+_initialized = False
+
+
+def init_rag():
+    global _qa_chain, _initialized
+
+    if _initialized:
+        return
+
+    HF_TOKEN = os.getenv("HF_TOKEN")
+    if not HF_TOKEN:
+        print(
+            "⚠️  HF_TOKEN not found in environment variables. Set it in Spaces secrets or .env file"
+        )
+
+    documents = []
+
+    try:
+        with open("tnb_genelgeler_rag.json", "r", encoding="utf-8") as f:
+            genelge_data = json.load(f)
+        print(f"✅ Loaded {len(genelge_data)} chunks from tnb_genelgeler_rag.json")
+
+        for item in genelge_data:
+            if "source_type" not in item.get("metadata", {}):
+                item.setdefault("metadata", {})["source_type"] = "genelge"
+            documents.append(
+                Document(
+                    page_content=item.get("content", ""),
+                    metadata=item.get("metadata", {}),
+                )
+            )
+
+    except FileNotFoundError:
+        print("⚠️  tnb_genelgeler_rag.json not found. Please upload data files.")
+
+    try:
+        with open("noterlik_kanunu_rag.json", "r", encoding="utf-8") as f:
+            kanun_data = json.load(f)
+        print(f"✅ Loaded {len(kanun_data)} chunks from noterlik_kanunu_rag.json")
+
+        for item in kanun_data:
+            documents.append(
+                Document(
+                    page_content=item.get("content", ""),
+                    metadata=item.get("metadata", {}),
+                )
+            )
+
+    except FileNotFoundError:
+        print("⚠️  noterlik_kanunu_rag.json not found. Please upload data files.")
+
+    if not documents:
+        print("❌ No documents loaded. Please prepare data files first.")
+        _initialized = False
+        return
+
+    print(f"📚 Total documents loaded: {len(documents)}")
+
+    faiss_index_path = "faiss_index"
+
+    print("🔄 Initializing embedding model (multilingual-e5-base)...")
+    embedding_model = HuggingFaceEmbeddings(
+        model_name="intfloat/multilingual-e5-base", encode_kwargs={"batch_size": 32}
+    )
+    print("✅ Embedding model initialized")
+
+    # Load or create FAISS index
+    if os.path.exists(faiss_index_path):
+        print(f"✅ Found existing FAISS index at {faiss_index_path} — loading...")
+        try:
+            vector_db = FAISS.load_local(
+                faiss_index_path, embedding_model, allow_dangerous_deserialization=True
+            )
+            print("✅ FAISS index loaded successfully!")
+        except Exception as e:
+            print(f"❌ Failed to load FAISS index: {e}")
+            print(f"🔄 Creating new FAISS index...")
+            vector_db = FAISS.from_documents(documents, embedding_model)
+            vector_db.save_local(faiss_index_path)
+            print(f"✅ FAISS index created and saved to {faiss_index_path}")
+    else:
+        print(f"🔄 Creating new FAISS index (this may take a few minutes)...")
+        vector_db = FAISS.from_documents(documents, embedding_model)
+        vector_db.save_local(faiss_index_path)
+        print(f"✅ FAISS index created and saved to {faiss_index_path}")
+
+    bm25_path = "bm25_retriever.pkl"
+    if os.path.exists(bm25_path):
+        print(f"✅ Loading existing BM25 index from {bm25_path}...")
+        with open(bm25_path, "rb") as f:
+            bm25_retriever = pickle.load(f)
+        print(f"✅ BM25 index loaded successfully!")
+    else:
+        print(f"🔄 Creating new BM25 index...")
+        bm25_retriever = BM25Retriever.from_documents(documents)
+        bm25_retriever.k = 5
+        with open(bm25_path, "wb") as f:
+            pickle.dump(bm25_retriever, f)
+        print(f"✅ BM25 index created and saved to {bm25_path}")
+
+    vector_retriever = vector_db.as_retriever(search_kwargs={"k": 5})
+
+    ensemble_retriever = EnsembleRetriever(
+        retrievers=[bm25_retriever, vector_retriever], weights=[0.5, 0.5]
+    )
+
+    print("🔄 Initializing HuggingFace LLM (Mistral-7B-Instruct)...")
+    try:
+        llm = HuggingFaceEndpoint(
+            repo_id="mistralai/Mistral-7B-Instruct-v0.2",
+            huggingfacehub_api_token=HF_TOKEN,
+            temperature=0.3,
+            max_new_tokens=1024,
+            top_p=0.95,
+            repetition_penalty=1.1,
+        )
+        print("✅ HuggingFace LLM initialized (Mistral-7B-Instruct-v0.2)")
+    except Exception as e:
+        print(f"❌ Failed to initialize LLM: {e}")
+        print(f"   HF_TOKEN is {'set' if HF_TOKEN else 'NOT set'}")
+        _initialized = False
+        return
+
+    turkish_legal_prompt = """Sen Türk Noter Hukuku konusunda uzman bir yapay zeka asistanısın. Görevin, Noterlik Kanunu ve Türkiye Noterler Birliği genelgelerinden yararlanarak kullanıcının sorusunu doğru ve eksiksiz yanıtlamaktır.
+
+    BAĞLAM BİLGİLERİ (Kanun ve Genelgelerden):
+    {context}
+
+    KULLANICI SORUSU: {question}
+
+    YANITLAMA STRATEJİSİ:
+    1. **KAYNAK ÖNCELİĞİ**: 
+       - Noterlik Kanunu → Temel yasal çerçeve ve genel kurallar
+       - TNB Genelgeleri → Kanunun uygulanmasına ilişkin özel düzenlemeler ve açıklamalar
+       - Her iki kaynağı da kontrol et ve ilgili olanları kullan
+
+    2. **HİBRİT YANITLAMA**: 
+       - Kanun maddeleri varsa bunları temel al
+       - Genelgelerdeki uygulama detayları varsa ekle
+       - Kaynak belirtmeyi unutma!
+
+    3. **KAYNAK BELİRTME**:
+       - Kanundan alınan bilgi → "Noterlik Kanunu Madde X'e göre..."
+       - Genelgelerden alınan bilgi → "Genelge X, Madde Y'ye göre..."
+       - Genel bilgi → "Genel olarak..." veya "Türk Hukuku'nda..."
+
+    4. **KALİTE KURALLARI**:
+       - Yanıtını net, anlaşılır ve yapılandırılmış şekilde sun
+       - Hukuki terminolojiyi doğru kullan
+       - Kesin olmadığın konularda varsayımda bulunma
+       - Hem kanunu hem genelgeleri kaynak olarak kullanabilirsin
+
+    YANITINIZ:"""
+
+    prompt_template = PromptTemplate(
+        template=turkish_legal_prompt, input_variables=["context", "question"]
+    )
+
+    _qa_chain = RetrievalQA.from_chain_type(
+        llm=llm,
+        retriever=ensemble_retriever,
+        chain_type="stuff",
+        chain_type_kwargs={"prompt": prompt_template, "document_separator": "\n---\n"},
+        return_source_documents=True,
+        verbose=False,
+    )
+
+    if _qa_chain is None:
+        print("❌ QA Chain creation failed silently")
+        return
+
+    print("✅ RAG system initialized successfully!\n")
+    _initialized = True
+
+
+def query_rag(question: str):
+    global _qa_chain, _initialized
+
+    if not _initialized:
+        init_rag()
+
+    if not _initialized or _qa_chain is None:
+        print("❌ RAG system is not properly initialized. Chain or data missing.")
+        return None
+
+    try:
+        print(f"DEBUG: _qa_chain type: {type(_qa_chain)}")
+        print(f"DEBUG: _qa_chain.invoke type: {type(_qa_chain.invoke)}")
+        print(f"DEBUG: Calling invoke with question: {question[:50]}...")
+        result = _qa_chain.invoke({"query": question})
+        return result
+    except Exception as e:
+        print(f"❌ Error querying RAG: {e}")
+        import traceback
+        traceback.print_exc()
+        return None

requirements.txt

@@ -0,0 +1,18 @@
+# Core Framework
+gradio==5.49.1
+
+# LangChain
+langchain==0.3.0
+langchain-community==0.3.0
+langchain-huggingface==0.1.0
+
+# HuggingFace
+huggingface-hub>=0.30.0,<1.0
+
+# Vector Store & Embeddings
+faiss-cpu==1.7.4
+sentence-transformers==2.2.2
+
+# BM25 Retriever
+rank-bm25==0.2.2
+