init

.env

@@ -0,0 +1 @@
+OPENAI_API_KEY = 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

.gitignore

@@ -0,0 +1 @@
+*.env

Dockerfile

@@ -0,0 +1,20 @@
+FROM python:3.9
+
+WORKDIR /code
+
+COPY ./requirements.txt /code/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt
+
+COPY . .
+
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
+WORKDIR $HOME/app
+
+COPY --chown=user . $HOME/app
+
+CMD ["uvicorn", "backend:app", "--host", "0.0.0.0", "--port", "7860"]

app.py

@@ -0,0 +1,32 @@
+import streamlit as st
+import requests
+
+backend_url = "http://0.0.0.0:7860"
+st.header('fiXit Assignment', divider='orange')
+st.header('Document-based QA Chatbot')
+
+if 'uploadFlag' not in st.session_state:
+    upload = st.file_uploader("Upload a DOCX file", type="docx")
+    if upload:
+        files = {'file': upload.getvalue()}
+        response = requests.post(f"{backend_url}/uploaddoc", files=files)
+        if response.status_code == 200:
+            st.success("Document uploaded sucessfully")
+            st.session_state.uploadFlag = True
+
+if 'uploadFlag' in st.session_state:
+    if 'chat_history' not in st.session_state:
+        st.session_state.chat_history = []
+
+    question = st.text_input("Enter your question/query: ")
+    if question:
+        response = requests.post(f"{backend_url}/question", json={"question": question})
+        if response.status_code == 200:
+            answer = response.json().get("answer")
+            st.session_state.chat_history.append(("user", question))
+            st.session_state.chat_history.append(("assistant", answer))
+        else:
+            st.error("Failed to get an answer from the backend.")
+    for role, message in st.session_state.chat_history:
+        with st.chat_message(role):
+            st.markdown(message)

backend.py

@@ -0,0 +1,40 @@
+from fastapi import FastAPI, UploadFile, File
+from pydantic import BaseModel
+from utils import *
+from io import BytesIO
+
+document_data = {}
+
+app = FastAPI()
+class Q(BaseModel):
+    question: str
+
+from utils import *
+
+
+@app.post("/uploaddoc")
+async def upload_document(file: UploadFile = File()):
+    content = await file.read()
+    file = BytesIO(content)
+    
+    text, tables = textprocessing(file)
+
+    text_embds = embed_query(text)
+    table_texts = ["-".join(["|".join(row) for row in table]) for table in tables]
+    table_embds = embed_query(table_texts)
+
+    vectordb = setvecdb(text_embds, table_embds, text, table_texts)
+    document_data["vectordb"] = vectordb
+    return {"message": "Document uploaded and processed successfully"}
+
+@app.post("/question")
+async def processquestion(question: Q):
+    query_text = question.question
+    vectordb = document_data.get("vectordb")
+    if vectordb is None:
+        return {"error": "No document uploaded"}
+    
+    search_results = retriever(vectordb, query_text)
+    context = "\n".join(search_results['content'].sum())
+    answer = generate_answer(query_text, context)
+    return {"answer": answer}

requirements.txt

@@ -0,0 +1,7 @@
+openai
+scikit-learn
+streamlit
+pandas
+pydantic
+python-docx
+numpy

utils.py

@@ -0,0 +1,82 @@
+import docx
+from openai import OpenAI
+import pandas as pd
+from sklearn.metrics.pairwise import cosine_similarity
+import numpy as np
+from dotenv import load_dotenv
+import os
+
+
+load_dotenv()
+client = OpenAI(
+    api_key= os.getenv("OPENAI_API_KEY")
+)
+
+def textprocessing(file_path):
+    doc = docx.Document(file_path)
+    text = []
+    tables = []
+
+    for paragraph in doc.paragraphs:
+        if paragraph.text.strip():  
+            text.append(paragraph.text)
+
+    for table in doc.tables:
+        table_data = []
+        for row in table.rows:
+            row_data = []
+            for cell in row.cells:
+                row_data.append(cell.text)
+            table_data.append(row_data)
+        tables.append(table_data)
+
+    return text, tables
+
+def embed_query(docs):
+    embds = []
+    for text in docs:
+        response = client.embeddings.create(input=text, model="text-embedding-ada-002")
+        embds.append(response.data[0].embedding)
+    return embds
+
+def setvecdb(text_embds, table_embds, text, tables):
+    data = []
+    
+    for idx, embedding in enumerate(text_embds):
+        data.append({"type": "text", "content": text[idx], "embedding": embedding})
+
+    for idx, (embedding, table_text) in enumerate(zip(table_embds, tables)):
+        data.append({"type": "table", "content": table_text, "embedding": embedding})
+    
+    vectordb = pd.DataFrame(data)
+    return vectordb
+
+def retriever(vectordb, query_text):
+    qembed = client.embeddings.create(input=query_text, model="text-embedding-ada-002")
+    query_embedding = np.array(qembed.data[0].embedding).reshape(1, -1)
+    
+    embeddings = np.stack(vectordb['embedding'].values)
+    similarities = cosine_similarity(query_embedding, embeddings).flatten()
+    
+    vectordb['similarity'] = similarities
+    results = vectordb.nlargest(4, 'similarity')
+    
+    return results
+
+prompt = """You are a smart chatbot that answers simple and complex questions of a user on the basis of contents from a document uploaded by the user. The document can have tabular data and textual data, with tables seperated by bar(|) and hyphens(-) to represent its rows and columns.
+Don't simply rely on direct logic for questions and use complex reasoning often for answering them. Below given is the context and the question. If unaware of the answer, simply respond that you cannot answer it based on the data, do not hallucinate. Make sure to answer questions related to tabular data properly as well. 
+Explain your purpose if asked, make sure you are helpful"""
+
+def generate_answer(question,ctxt):
+    response =  client.chat.completions.create(
+        model = "gpt-3.5-turbo-0125",
+        messages = [
+            {
+                "role" : "system",
+                "content" : prompt
+            },
+            {"role" : "user","content" : question},
+            {"role" : "assistant","content" : ctxt}
+        ]
+    )
+    return response.choices[0].message.content