diff --git a/src/config.py b/src/config.py index 24d4484..548209b 100644 --- a/src/config.py +++ b/src/config.py @@ -47,6 +47,8 @@ class Settings(BaseSettings): request_timeout: int = Field(30) request_delay: float = Field(1.0) # seconds between requests min_content_length: int = Field(200) # chars + # Libros/dumps enteros (100k+ palabras) inflan RAM y DB sin aportar al RAG + max_content_length: int = Field(300_000) # chars # Fuentes opcionales — desactivadas por defecto: la IP del homelab está # bloqueada por Reddit (403) y YouTube (transcripts vacíos), eran peso muerto. diff --git a/src/scraper/exhaustive.py b/src/scraper/exhaustive.py index 8c124e2..c2b261b 100644 --- a/src/scraper/exhaustive.py +++ b/src/scraper/exhaustive.py @@ -628,6 +628,11 @@ class ExhaustiveScraper: error="Content too short or empty") return + if len(content) > settings.max_content_length: + logger.info("Content truncated", source_id=source_id, + original_length=len(content), url=url[:60]) + content = content[:settings.max_content_length] + word_count = len(content.split()) await self.db.save_source_content(source_id, content) @@ -819,30 +824,49 @@ class ExhaustiveScraper: entries=len(entries), added=added) return added + # pdfplumber es síncrono y CPU-intensivo: parsear inline congela el event + # loop, y con PDFs grandes el pico de RAM puede matar el pod (OOM con + # límite de 1-2Gi). Ejecutar SIEMPRE vía run_in_executor. + + @staticmethod + def _parse_pdf_sync(path: str) -> str: + import pdfplumber + + with pdfplumber.open(path) as pdf: + pages = [] + for page in pdf.pages[:50]: # max 50 pages + pages.append(page.extract_text() or "") + page.flush_cache() # pdfplumber cachea objetos de página: liberar + return "\n\n".join(pages) + async def _extract_pdf(self, url: str) -> tuple[Optional[str], Optional[str]]: """Download and extract PDF text""" - import pdfplumber import tempfile import os + max_pdf_bytes = 15 * 1024 * 1024 # varios PDFs concurrentes en RAM: cap agresivo + http = await self._get_http() try: async with http.get(url) as resp: if resp.status != 200: return None, None content_length = int(resp.headers.get("content-length", 0)) - if content_length > 50 * 1024 * 1024: # skip PDFs > 50MB + if content_length > max_pdf_bytes: return None, None pdf_bytes = await resp.read() + # Sin Content-Length el check anterior no protege + if len(pdf_bytes) > max_pdf_bytes: + return None, None with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: f.write(pdf_bytes) tmp_path = f.name + del pdf_bytes try: - with pdfplumber.open(tmp_path) as pdf: - pages = [p.extract_text() or "" for p in pdf.pages[:50]] # max 50 pages - text = "\n\n".join(pages) + loop = asyncio.get_running_loop() + text = await loop.run_in_executor(None, self._parse_pdf_sync, tmp_path) return text, url.split("/")[-1] finally: os.unlink(tmp_path)