streamlit init

app.py

@@ -27,7 +27,7 @@ def run_survey(surveyor, research_keywords, max_search, num_papers):
             st.write(line)
 
 
-def survey_space():
+def survey_space(surveyor):
     
     st.title('Automated Survey generation from research keywords - Auto-Research V0.1')
 
@@ -41,12 +41,10 @@ def survey_space():
 
     if submit:
         st.write("hello")
-        if surveyor_obj is None:
-            surveyor_obj = Surveyor()
-        run_survey(surveyor_obj, research_keywords, max_search, num_papers)
+        run_survey(surveyor, research_keywords, max_search, num_papers)
 
 
 if __name__ == '__main__':
-    global surveyor_obj
-    surveyor_obj = None
-    survey_space()
+    global surveyor
+    surveyor_obj = Surveyor()
+    survey_space(surveyor_obj)

arxiv_public_data/config.py

@@ -9,7 +9,7 @@ logging.basicConfig(
 baselog = logging.getLogger('arxivdata')
 logger = baselog.getChild('config')
 
-DEFAULT_PATH = os.path.join(os.path.abspath('/'), 'arxiv-data')
+DEFAULT_PATH = os.path.join(os.path.abspath('../'), 'arxiv-data')
 JSONFILE = './config.json'
 KEY = 'ARXIV_DATA'
 

pyproject.toml

@@ -0,0 +1,6 @@
+[build-system]
+requires = [
+    "setuptools>=42",
+    "wheel"
+]
+build-backend = "setuptools.build_meta"

src/Surveyor.py

@@ -16,7 +16,7 @@ except:
     from transformers import AutoTokenizer, AutoModelForSeq2SeqLM, AutoConfig, AutoModel, LEDTokenizer, \
         LEDForConditionalGeneration
 
-from src.defaults import DEFAULTS
+from src.defaults import DEFAULTS_CPU_COMPAT, DEFAULTS_HIGH_GPU
 
 
 class Surveyor:
@@ -70,18 +70,20 @@ class Surveyor:
             - num_papers: int maximium number of papers to download and analyse - defaults to 25
 
         '''
-        self.torch_device = 'cuda' if torch.cuda.is_available() else 'cpu'
+        self.torch_device = 'cpu'
         print("\nTorch_device: " + self.torch_device)
-        if 'cuda' in self.torch_device:
-            print("\nloading spacy for gpu")
+        if torch.cuda.is_available():
+            print("\nloading defaults for gpu")
+            self.torch_device = 'cuda'
             spacy.require_gpu()
 
+        self.high_gpu = high_gpu
+        DEFAULTS = DEFAULTS_CPU_COMPAT
+        if self.high_gpu:
+            DEFAULTS = DEFAULTS_HIGH_GPU
+
         if not kw_model_name:
             kw_model_name = DEFAULTS["kw_model_name"]
-        if not high_gpu:
-            self.high_gpu = DEFAULTS["high_gpu"]
-        else:
-            self.high_gpu = high_gpu
         self.num_papers = DEFAULTS['num_papers']
         self.max_search = DEFAULTS['max_search']
         if not models_dir:
@@ -110,8 +112,8 @@ class Surveyor:
             if not no_save_models:
                 self.clean_dirs([models_dir])
 
-            self.title_tokenizer = AutoTokenizer.from_pretrained(title_model_name)
-            self.title_model = AutoModelForSeq2SeqLM.from_pretrained(title_model_name).to(self.torch_device)
+            self.title_tokenizer = AutoTokenizer.from_pretrained(title_model_name, trust_remote_code=True)
+            self.title_model = AutoModelForSeq2SeqLM.from_pretrained(title_model_name, trust_remote_code=True).to(self.torch_device)
             self.title_model.eval()
             if not no_save_models:
                 self.title_model.save_pretrained(models_dir + "/title_model")
@@ -142,7 +144,7 @@ class Surveyor:
                 self.embedder.save(models_dir + "/embedder")
         else:
             print("\nInitializing from previously saved models at" + models_dir)
-            self.title_tokenizer = AutoTokenizer.from_pretrained(title_model_name)
+            self.title_tokenizer = AutoTokenizer.from_pretrained(title_model_name).to(self.torch_device)
             self.title_model = AutoModelForSeq2SeqLM.from_pretrained(models_dir + "/title_model").to(self.torch_device)
             self.title_model.eval()
 
@@ -615,7 +617,11 @@ class Surveyor:
         paper_body = ""
         for k, v in research_sections.items():
             paper_body += v
-        return self.abstractive_summary(paper_body)
+
+        try:
+            return self.abstractive_summary(paper_body)
+        except:
+            return self.abstractive_summary(self.extractive_summary(paper_body))
 
     def build_corpus_sectionwise(self, papers):
         known = ['abstract', 'introduction', 'conclusion']

src/defaults.py

@@ -1,5 +1,5 @@
 # defaults for arxiv
-DEFAULTS = {
+DEFAULTS_HIGH_GPU = {
     "max_search": 100,
     "num_papers": 20,
     "high_gpu": False,
@@ -16,5 +16,23 @@ DEFAULTS = {
     "nlp_name": "en_core_sci_scibert",
     "similarity_nlp_name": "en_core_sci_lg",
     "kw_model_name": "distilbert-base-nli-mean-tokens",
+}
 
+DEFAULTS_CPU_COMPAT = {
+    "max_search": 100,
+    "num_papers": 20,
+    "high_gpu": False,
+    "pdf_dir": "arxiv_data/tarpdfs/",
+    "txt_dir": "arxiv_data/fulltext/",
+    "img_dir": "arxiv_data/images/",
+    "tab_dir": "arxiv_data/tables/",
+    "dump_dir": "arxiv_dumps/",
+    "models_dir": "saved_models/",
+    "title_model_name": "ccdv/lsg-bart-base-4096-arxiv",
+    "ex_summ_model_name": "allenai/scibert_scivocab_uncased",
+    "ledmodel_name": "bhuvaneswari/t5-small-text_summarization",
+    "embedder_name": "paraphrase-MiniLM-L6-v2",
+    "nlp_name": "en_core_sci_scibert",
+    "similarity_nlp_name": "en_core_sci_lg",
+    "kw_model_name": "distilbert-base-nli-mean-tokens",
 }