Upload 3 files

chatbot.py

@@ -0,0 +1,131 @@
+import streamlit as st
+from langchain.document_loaders import PyPDFLoader, Docx2txtLoader, TextLoader
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.embeddings.openai import OpenAIEmbeddings
+from langchain.vectorstores import FAISS
+from langchain.chains import ConversationalRetrievalChain
+from langchain.llms import OpenAI
+import os
+import tempfile
+from doc_qa import  embeddings,llm
+from doc_qa_1 import embeddings,doc_qa
+
+def start_message(doc_name):
+    st.success("✅ ドキュメントのアップロードが完了しました！")
+    st.markdown(f"### 📄 アップロードされました: `{doc_name}`")
+    st.markdown("これで文書に関する質問ができます。 💬")
+    st.markdown("例えば、次のような質問ができます。:")
+    st.markdown("- この文書は何について書かれていますか？")
+    st.markdown("- 重要なポイントを要約してください。")
+    st.markdown("- 著者は誰ですか？")
+    st.markdown("はじめるには、下に質問を入力してください。!")
+
+# Function to load individual file
+def load_file(file, suffix):
+    with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as temp_file:
+        temp_file.write(file.read())
+        temp_file_path = temp_file.name
+
+    if suffix == ".pdf":
+        loader = PyPDFLoader(temp_file_path)
+    elif suffix == ".docx":
+        loader = Docx2txtLoader(temp_file_path)
+    elif suffix == ".txt":
+        loader = TextLoader(temp_file_path)
+    else:
+        return []
+
+    return loader.load()
+st.set_page_config(
+    page_title="QA Assistant",
+    page_icon="https://yourdomain.com/logo.png",
+    layout="centered"
+)
+# Title
+st.title("📄 ドキュメント質問応答支援ツール")
+
+# Step 1: Upload document
+if "file_uploaded" not in st.session_state:
+    st.session_state.file_uploaded = False
+    st.markdown("""
+    👋 こちらへようこそ！私は文書の内容を理解するためのインテリジェントアシスタントです。
+
+あなたは以下のことができます：
+
+PDF、DOCX、TXTファイルをアップロード
+
+文書の内容について質問
+
+要約、重要ポイント、または具体的な詳細の取得
+
+🛠️ 質問の例：
+この文書は何について書かれていますか？
+
+主要なポイントを要約してください。
+
+著者は誰ですか？
+
+重要な日付や締め切りは何ですか？
+
+結論や推奨事項は何ですか？
+
+📂 まず、1つ以上の文書をアップロードしてください。
+💬 その後、下に質問を入力しましょう！
+    """)
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+
+
+flag = 0
+# Upload multiple files
+with st.sidebar:
+    uploaded_files = st.file_uploader("PDF、DOCX、またはTXTファイルをアップロードしてください。", type=["pdf", "docx", "txt"], accept_multiple_files=True)
+    # Load and process documents
+    file_names=[]
+    if uploaded_files:
+        all_docs = []
+        for file in uploaded_files:
+            suffix = os.path.splitext(file.name)[1]
+            docs = load_file(file, suffix)
+            all_docs.extend(docs)
+            file_names.append(file.name)
+
+        # Split and embed documents
+        text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=100)
+        split_docs = text_splitter.split_documents(all_docs)
+        #embeddings = OpenAIEmbeddings()
+        vectorstore = FAISS.from_documents(split_docs, embeddings)
+
+        # Setup ConversationalRetrievalChain
+        qa_chain = ConversationalRetrievalChain.from_llm(
+            llm=llm,
+            retriever=vectorstore.as_retriever(),
+            return_source_documents=False
+        )
+        start_message('\n'.join(file_names))
+        flag = 1
+
+# Initialize session state
+if "chat_history" not in st.session_state:
+    st.session_state.chat_history = []
+
+for msg in st.session_state.messages:
+    st.chat_message(msg["role"]).write(msg["content"])
+
+if flag==1:
+    if user_query := st.chat_input():
+        st.session_state.messages.append({"role": "user", "content": user_query})
+        with st.chat_message("user"):
+            st.markdown(f"**Q:** {user_query}")
+        result=doc_qa(user_query,vectorstore)
+        st.session_state.messages.append({"role": "assistant", "content": result["answer"]})
+        with st.chat_message("assistant"):
+            st.markdown(f"**A:** {result["answer"]}")
+        st.session_state.chat_history.append((user_query, result["answer"]))
+
+        # # Display conversation history
+        # if st.session_state.chat_history:
+        #     st.markdown("### 🗨️ Chat History")
+        #     for i, (q, a) in enumerate(st.session_state.chat_history, 1):
+        #         st.markdown(f"**Q{i}:** {q}")
+        #         st.markdown(f"**A{i}:** {a}")

doc_qa.py

@@ -0,0 +1,124 @@
+from typing import Optional, List
+
+from langchain.document_loaders import TextLoader  #for textfiles
+from langchain.text_splitter import CharacterTextSplitter #text splitter
+from langchain.embeddings import HuggingFaceEmbeddings #for using HugginFace models
+
+from langchain.document_loaders import UnstructuredPDFLoader  #load pdf
+from langchain.indexes import VectorstoreIndexCreator #vectorize db index with chromadb
+from langchain.chains import RetrievalQA
+from langchain.document_loaders import UnstructuredURLLoader  #load urls into docoument-loader
+from langchain.chains.question_answering import load_qa_chain
+from langchain import HuggingFaceHub
+import os
+from langchain.document_loaders import TextLoader, PyMuPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.llms import HuggingFacePipeline
+from langchain.vectorstores import FAISS
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain import PromptTemplate
+from langchain.chains import LLMChain
+from langchain.base_language import BaseLanguageModel
+from docx import Document
+from langchain.document_loaders import DirectoryLoader
+multi_directory_path=r'tmp/'
+
+from transformers import pipeline
+
+embeddings = HuggingFaceEmbeddings(model_name='sentence-transformers/LaBSE')
+
+
+from langchain_community.document_loaders import TextLoader, PyPDFLoader, Docx2txtLoader
+
+after_rag_template = """Answer the question based only on the following context:
+   {context}
+   Question: {question}
+   """
+#pipe = pipeline("text2text-generation", model="google/flan-t5-large" ,max_new_tokens=100)
+#pipe = pipeline("text2text-generation", model="google/mt5-large" ,max_new_tokens=200)
+from transformers import pipeline, AutoTokenizer, AutoModelForSeq2SeqLM
+#tokenizer = AutoTokenizer.from_pretrained("rinna/bilingual-gpt-neox-4b", use_fast=False)
+#model = AutoModelForSeq2SeqLM.from_pretrained("google/mt5-base")
+# Load model directly
+from transformers import AutoTokenizer, AutoModelForCausalLM
+
+#tokenizer = AutoTokenizer.from_pretrained("rinna/bilingual-gpt-neox-4b")
+#model = AutoModelForCausalLM.from_pretrained("rinna/bilingual-gpt-neox-4b")
+#pipe = pipeline("text2text-generation", model="rinna/bilingual-gpt-neox-4b" ,max_new_tokens=200)
+#pipe = pipeline("text2text-generation", model=model, tokenizer=tokenizer, max_new_tokens=200)
+
+pipe = pipeline("question-answering", model="deepset/xlm-roberta-base-squad2")
+
+llm = HuggingFacePipeline(pipeline=pipe)
+
+def run_custom_qa(question, retrieved_docs):
+    context = " ".join([doc.page_content for doc in retrieved_docs])
+    output = pipe(question=question, context=context)
+    return output["answer"]
+
+def docs_vector_index():
+    from langchain.document_loaders import DirectoryLoader
+    # Define a directory path
+    directory_path = r"C:\Users\savni\PycharmProjects\DocsSearchEngine\tmp"
+
+    # Create the DirectoryLoader, specifying loaders for each file type
+    loader = DirectoryLoader(
+        directory_path,
+        glob="**/*",  # This pattern loads all files; modify as needed
+
+    )
+    docs = loader.load()
+
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=1024, chunk_overlap=100, separators=[" ", ",", "\n", "."]
+    )
+    print(docs)
+    docs_chunks = text_splitter.split_documents(docs)
+
+    print(f"docs_chunks length: {len(docs_chunks)}")
+    print('********************docs_chunks',docs_chunks)
+    if len(docs_chunks)>0:
+        db = FAISS.from_documents(docs_chunks, embeddings)
+        return db
+    else:
+        return ''
+
+
+#chain = load_qa_chain(llm, chain_type="stuff")
+
+from langchain.prompts import PromptTemplate
+
+template = """You are an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. Below is some information. 
+{context}
+
+Based on the above information only, answer the below question. 
+
+{question} Be concise."""
+
+prompt = PromptTemplate.from_template(template)
+print(prompt.input_variables)
+
+
+#query_llm = LLMChain(llm=llm, prompt=prompt)
+
+# def doc_qa1(query, db):
+#     similar_doc = db.similarity_search(query, k=2)
+#     doc_c=[]
+#     for c in similar_doc:
+#         doc_c.append(c.page_content)
+#     context=''.join(doc_c)
+#     #response = query_llm.run({"context": context, "question": query})
+#     response = query_llm.run(context=context, question=query)
+#     print('response',response)
+#     return response
+
+def doc_qa(query, db):
+    print("*************************custom qa doc_qa",query)
+    retriever = db.as_retriever()
+    relevant_docs = retriever.get_relevant_documents(query)
+    response=run_custom_qa(query, relevant_docs)
+    print('response', response)
+    return response
+
+
+

doc_qa_1.py

@@ -0,0 +1,62 @@
+from langchain.vectorstores import FAISS
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.text_splitter import CharacterTextSplitter
+from langchain.docstore.document import Document
+from transformers import pipeline
+from langchain.chains.question_answering import load_qa_chain
+import os
+
+# Step 1: Load QA pipeline (don't wrap in HuggingFacePipeline)
+embeddings = HuggingFaceEmbeddings(model_name="intfloat/multilingual-e5-small")
+qa_pipeline = pipeline("question-answering", model="deepset/xlm-roberta-base-squad2")
+multi_directory_path=r'tmp/'
+
+def docs_vector_index():
+    from langchain.document_loaders import DirectoryLoader
+    # Define a directory path
+    directory_path = r"C:\Users\savni\PycharmProjects\DocsSearchEngine\tmp"
+
+    # Create the DirectoryLoader, specifying loaders for each file type
+    loader = DirectoryLoader(
+        directory_path,
+        glob="**/*",  # This pattern loads all files; modify as needed
+
+    )
+    docs = loader.load()
+
+    text_splitter = RecursiveCharacterTextSplitter(
+        chunk_size=1024, chunk_overlap=100, separators=[" ", ",", "\n", "."]
+    )
+    print(docs)
+    docs_chunks = text_splitter.split_documents(docs)
+
+    print(f"docs_chunks length: {len(docs_chunks)}")
+    print('********************docs_chunks',docs_chunks)
+    if len(docs_chunks)>0:
+        db = FAISS.from_documents(docs_chunks, embeddings)
+        return db
+    else:
+        return ''
+
+
+
+def run_custom_qa(question, retrieved_docs):
+    context = " ".join([doc.page_content for doc in retrieved_docs])
+    output = qa_pipeline(question=question, context=context)
+    return output #output["answer"]
+
+# # Step 6: Ask question
+# question = "東京大学はいつ設立されましたか？"
+# relevant_docs = retriever.get_relevant_documents(question)
+# answer = run_custom_qa(question, relevant_docs)
+#
+# print("Answer:", answer)
+
+def doc_qa(query, db):
+    print("*************************custom qa doc_qa",query)
+    retriever = db.as_retriever()
+    relevant_docs = retriever.get_relevant_documents(query)
+    response=run_custom_qa(query, relevant_docs)
+    print('response', response)
+    return response