From 8dfd0116b2aa0c2a54cc32e3d02d5934f54e5c05 Mon Sep 17 00:00:00 2001 From: ChemaVX Date: Sat, 4 Jul 2026 10:49:22 +0000 Subject: [PATCH] =?UTF-8?q?fix(scraper):=20DDGS=20s=C3=ADncrono=20ya=20no?= =?UTF-8?q?=20bloquea=20el=20event=20loop?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Las llamadas DDGS() en _seed_search y _seed_youtube eran síncronas y se ejecutaban directamente en código async: durante cada búsqueda DDG el bot entero (comandos de Telegram incluidos) quedaba congelado. Extraídas a helpers sync (_ddg_text_sync/_ddg_videos_sync) ejecutados vía run_in_executor. Sin cambios en queries ni procesamiento de resultados. Verificado con mock de DDG lento (3s): peor parón del loop 52ms. Co-Authored-By: Claude Fable 5 --- src/scraper/exhaustive.py | 49 +++++++++++++++++++++++++-------------- 1 file changed, 31 insertions(+), 18 deletions(-) diff --git a/src/scraper/exhaustive.py b/src/scraper/exhaustive.py index 28228bf..a8c23ff 100644 --- a/src/scraper/exhaustive.py +++ b/src/scraper/exhaustive.py @@ -266,6 +266,21 @@ class ExhaustiveScraper: logger.warning("SearXNG failed", query=query, error=str(e)) return [] + # DDGS es síncrono (requests bloqueante por dentro): llamarlo directamente + # desde código async congela el event loop entero — bot de Telegram incluido — + # mientras dura la búsqueda. Estos helpers deben ejecutarse SIEMPRE vía + # run_in_executor. + + @staticmethod + def _ddg_text_sync(query: str) -> list[dict]: + with DDGS() as ddgs: + return list(ddgs.text(query, max_results=settings.max_pages_per_search)) + + @staticmethod + def _ddg_videos_sync(query: str) -> list[dict]: + with DDGS() as ddgs: + return list(ddgs.videos(query, max_results=10)) + async def _seed_search(self): """SearXNG primary + DDG fallback per query""" queries = await self._generate_ddg_queries() @@ -276,12 +291,10 @@ class ExhaustiveScraper: if not results: logger.info("SearXNG vacío, usando DDG", query=query) try: - with DDGS() as ddgs: - ddg_results = list(ddgs.text( - query, - max_results=settings.max_pages_per_search - )) - results = ddg_results + loop = asyncio.get_running_loop() + results = await loop.run_in_executor( + None, self._ddg_text_sync, query + ) logger.info("DDG fallback ok", query=query, results=len(results)) except Exception as e: logger.warning("DDG fallback failed", query=query, error=str(e)) @@ -395,18 +408,18 @@ class ExhaustiveScraper: async def _seed_youtube(self): """Search YouTube via DDG for video transcripts""" try: - with DDGS() as ddgs: - results = list(ddgs.videos( - f"{self.topic} documentary explanation", - max_results=10 - )) - for r in results: - url = r.get("content", "") - if "youtube.com" in url or "youtu.be" in url: - await self.db.add_source( - self.session_id, url, "youtube", depth=0, - title=r.get("title") - ) + loop = asyncio.get_running_loop() + results = await loop.run_in_executor( + None, self._ddg_videos_sync, + f"{self.topic} documentary explanation" + ) + for r in results: + url = r.get("content", "") + if "youtube.com" in url or "youtu.be" in url: + await self.db.add_source( + self.session_id, url, "youtube", depth=0, + title=r.get("title") + ) except Exception as e: logger.warning("YouTube seed failed", error=str(e))