Duplicate from asifhugs/InfiniteGPT

Co-authored-by: Asif Ahmad <asifhugs@users.noreply.huggingface.co>

.gitattributes

@@ -0,0 +1,34 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,13 @@
+---
+title: InfiniteGPT
+emoji: 📚
+colorFrom: indigo
+colorTo: yellow
+sdk: streamlit
+sdk_version: 1.19.0
+app_file: app.py
+pinned: false
+duplicated_from: asifhugs/InfiniteGPT
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,62 @@
+import openai
+from concurrent.futures import ThreadPoolExecutor
+import streamlit as st
+import os
+
+# Add your own OpenAI API key
+# openai.api_key = "sk-XXXXXXXXXXXXXXXXXXXXX"
+
+openai.api_key = os.environ["OPENAI_API_KEY"]
+
+def load_text(file_path):
+    with open(file_path, 'r') as file:
+        return file.read()
+
+def save_to_file(responses, output_file):
+    with open(output_file, 'w') as file:
+        for response in responses:
+            file.write(response + '\n')
+
+def call_openai_api(chunk):
+    response = openai.ChatCompletion.create(
+        model="gpt-3.5-turbo",
+        messages=[
+            {"role": "system", "content": "PASS IN ANY ARBITRARY SYSTEM VALUE TO GIVE THE AI AN IDENITY"},
+            {"role": "user", "content": f"OPTIONAL PREPROMPTING FOLLOWING BY YOUR DATA TO PASS IN: {chunk}."},
+        ],
+        max_tokens=1750,
+        n=1,
+        stop=None,
+        temperature=0.5,
+    )
+    return response.choices[0]['message']['content'].strip()
+
+def split_into_chunks(text, tokens=1500):
+    words = text.split()
+    chunks = [' '.join(words[i:i + tokens]) for i in range(0, len(words), tokens)]
+    return chunks
+
+def process_chunks(input_text):
+    chunks = split_into_chunks(input_text)
+
+    # Processes chunks in parallel
+    with ThreadPoolExecutor() as executor:
+        responses = list(executor.map(call_openai_api, chunks))
+
+    return "\n".join(responses)
+
+def main():
+    st.title("InfiniteGPT")
+    st.write("Input unlimited size text and process it using the OpenAI API.")
+    st.write("For more details and credit check this [GitHub repository](https://github.com/emmethalm/infiniteGPT).")
+
+    input_text = st.text_area("Enter or paste your text here", height=300)
+    process_button = st.button("Process Text")
+
+    if process_button and input_text:
+        with st.spinner("Processing text..."):
+            output_text = process_chunks(input_text)
+            st.write(output_text)
+
+if __name__ == "__main__":
+    main()

requirements.txt

@@ -0,0 +1,2 @@
+openai
+streamlit