fix(scraper): DDGS síncrono ya no bloquea el event loop
Build & Deploy ResearchOwl / build-and-push (push) Successful in 7s

Las llamadas DDGS() en _seed_search y _seed_youtube eran síncronas y se
ejecutaban directamente en código async: durante cada búsqueda DDG el bot
entero (comandos de Telegram incluidos) quedaba congelado. Extraídas a
helpers sync (_ddg_text_sync/_ddg_videos_sync) ejecutados vía
run_in_executor. Sin cambios en queries ni procesamiento de resultados.

Verificado con mock de DDG lento (3s): peor parón del loop 52ms.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
ChemaVX
2026-07-04 10:49:22 +00:00
co-authored by Claude Fable 5
parent c701628144
commit 8dfd0116b2
+24 -11
View File
@@ -266,6 +266,21 @@ class ExhaustiveScraper:
logger.warning("SearXNG failed", query=query, error=str(e)) logger.warning("SearXNG failed", query=query, error=str(e))
return [] return []
# DDGS es síncrono (requests bloqueante por dentro): llamarlo directamente
# desde código async congela el event loop entero — bot de Telegram incluido —
# mientras dura la búsqueda. Estos helpers deben ejecutarse SIEMPRE vía
# run_in_executor.
@staticmethod
def _ddg_text_sync(query: str) -> list[dict]:
with DDGS() as ddgs:
return list(ddgs.text(query, max_results=settings.max_pages_per_search))
@staticmethod
def _ddg_videos_sync(query: str) -> list[dict]:
with DDGS() as ddgs:
return list(ddgs.videos(query, max_results=10))
async def _seed_search(self): async def _seed_search(self):
"""SearXNG primary + DDG fallback per query""" """SearXNG primary + DDG fallback per query"""
queries = await self._generate_ddg_queries() queries = await self._generate_ddg_queries()
@@ -276,12 +291,10 @@ class ExhaustiveScraper:
if not results: if not results:
logger.info("SearXNG vacío, usando DDG", query=query) logger.info("SearXNG vacío, usando DDG", query=query)
try: try:
with DDGS() as ddgs: loop = asyncio.get_running_loop()
ddg_results = list(ddgs.text( results = await loop.run_in_executor(
query, None, self._ddg_text_sync, query
max_results=settings.max_pages_per_search )
))
results = ddg_results
logger.info("DDG fallback ok", query=query, results=len(results)) logger.info("DDG fallback ok", query=query, results=len(results))
except Exception as e: except Exception as e:
logger.warning("DDG fallback failed", query=query, error=str(e)) logger.warning("DDG fallback failed", query=query, error=str(e))
@@ -395,11 +408,11 @@ class ExhaustiveScraper:
async def _seed_youtube(self): async def _seed_youtube(self):
"""Search YouTube via DDG for video transcripts""" """Search YouTube via DDG for video transcripts"""
try: try:
with DDGS() as ddgs: loop = asyncio.get_running_loop()
results = list(ddgs.videos( results = await loop.run_in_executor(
f"{self.topic} documentary explanation", None, self._ddg_videos_sync,
max_results=10 f"{self.topic} documentary explanation"
)) )
for r in results: for r in results:
url = r.get("content", "") url = r.get("content", "")
if "youtube.com" in url or "youtu.be" in url: if "youtube.com" in url or "youtu.be" in url: