Delete papers.py

papers.py

@@ -1,116 +0,0 @@
-import os
-import requests
-import tempfile
-from datetime import datetime, timezone
-import base64
-from tqdm.auto import tqdm
-import pymupdf
-
-DAILY_PAPERS_API_URL = "https://huggingface.co/api/daily_papers"
-
-class PaperManager:
-    def __init__(self, papers_per_page=30):
-        self.papers = []
-        self.raw_papers = []  # To store fetched data
-
-    def calculate_rising_score(self, paper):
-        """
-        Calculate the rising score of a paper.
-        This emphasizes recent upvotes and the rate of upvote accumulation.
-        """
-        upvotes = paper.get('paper', {}).get('upvotes', 0)
-        published_at_str = paper.get('publishedAt', datetime.now(timezone.utc).isoformat())
-        try:
-            published_time = datetime.fromisoformat(published_at_str.replace('Z', '+00:00'))
-        except ValueError:
-            published_time = datetime.now(timezone.utc)
-
-        time_diff = datetime.now(timezone.utc) - published_time
-        time_diff_hours = time_diff.total_seconds() / 3600  # Convert time difference to hours
-
-        # Rising score favors papers that are gaining upvotes quickly
-        # Adjusted to have a linear decay over time
-        score = upvotes / (time_diff_hours + 1)
-        return score
-
-    def fetch_papers(self):
-        try:
-            response = requests.get(f"{DAILY_PAPERS_API_URL}?limit=100")
-            response.raise_for_status()
-            data = response.json()
-
-            if not data:
-                print("No data received from API.")
-                return False
-
-            self.raw_papers = data  # Store raw data
-
-            return True
-
-        except requests.RequestException as e:
-            print(f"Error fetching papers: {e}")
-            return False
-        except Exception as e:
-            print(f"Unexpected error: {e}")
-            return False
-
-    def filter_top_papers(self, threshold_general=2.0, threshold_agent=0.7):
-        self.papers = []
-        for paper in self.raw_papers:
-            paper_score = self.calculate_rising_score(paper)
-            # if paper_score >= threshold_general or ('agent' in paper['title'].lower() and paper_score >= threshold_agent):
-            self.papers.append(paper)
-
-        self.papers = sorted(
-            self.papers,
-            key=lambda x: self.calculate_rising_score(x) * (3 if 'agent' in x['title'].lower() else 1),
-            reverse=True
-        )[:2]
-        return self.papers
-
-    # def get_paper_content(self, paper_id):
-    #     pdf_url = f"https://arxiv.org/pdf/{paper_id}.pdf"
-    #     print("Processing paper:", pdf_url)
-    #     client = httpx.Client(follow_redirects=True)
-    #     response = client.get(pdf_url)
-
-    #     # First verification - check if we got a valid PDF response
-    #     if response.status_code != 200:
-    #         raise Exception(f"Failed to fetch PDF: {response.status_code}")
-        
-    #     if not response.headers.get('content-type', '').startswith('application/pdf'):
-    #         raise Exception(f"Unexpected content type: {response.headers.get('content-type')}")
-        
-    #     # Second verification - check the first few bytes of the content
-    #     if not response.content.startswith(b'%PDF'):
-    #         raise Exception("Content doesn't appear to be a valid PDF")
-    
-    #     pdf_data = base64.standard_b64encode(response.content).decode("utf-8")
-    #     return {"pdf": pdf_data, "url": pdf_url}
-
-    def get_paper_text(self, paper_id):
-        url = f"https://arxiv.org/pdf/{paper_id}.pdf"
-        response = requests.get(url)
-        
-        if response.status_code != 200:
-            raise Exception(f"Failed to download PDF: {response.status_code}")
-        
-        with open("temp.pdf", "wb") as f:
-            f.write(response.content)
-
-        with pymupdf.open("temp.pdf") as doc:
-            text = ""
-            for page in doc:
-                text += page.get_text()        
-        return text
-
-
-    def get_top_content(self):
-        self.fetch_papers()
-        self.filter_top_papers()
-        contents =  {}
-        print(f"Processing {len(self.papers)} papers:")
-        for paper in tqdm(self.papers):
-            paper_id = paper["paper"]['id']
-            contents[paper["paper"]['title']] = self.get_paper_text(paper_id)
-        return contents