Spaces:
Sleeping
Sleeping
File size: 1,468 Bytes
0d10b91 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 |
import fitz # PyMuPDF
import requests
from io import BytesIO
from concurrent.futures import ThreadPoolExecutor
import os
def extract_page_text(page):
text = page.get_text()
return text if text.strip() else None
def parse_pdf_from_url_multithreaded(url, max_workers=None):
# Automatically detect and use all available CPU cores if max_workers not set
if max_workers is None:
max_workers = os.cpu_count() or 8
res = requests.get(url)
doc = fitz.open(stream=BytesIO(res.content), filetype="pdf")
pages = [page for page in doc]
chunks = [None] * len(pages)
# Process pages in parallel, preserving page order
with ThreadPoolExecutor(max_workers=max_workers) as executor:
results = list(executor.map(extract_page_text, pages))
# Keep only non-empty page results, preserving order
doc.close()
return [r for r in results if r]
def parse_pdf_from_file_multithreaded(file_path, max_workers=None):
if max_workers is None:
max_workers = os.cpu_count() or 8
try:
doc = fitz.open(file_path)
pages = [page for page in doc]
chunks = [None] * len(pages)
with ThreadPoolExecutor(max_workers=max_workers) as executor:
results = list(executor.map(extract_page_text, pages))
doc.close()
return [r for r in results if r]
except Exception as e:
raise Exception(f"Error parsing PDF file {file_path}: {str(e)}")
|