fix(scraper): DDGS síncrono ya no bloquea el event loop
Build & Deploy ResearchOwl / build-and-push (push) Successful in 7s

Las llamadas DDGS() en _seed_search y _seed_youtube eran síncronas y se
ejecutaban directamente en código async: durante cada búsqueda DDG el bot
entero (comandos de Telegram incluidos) quedaba congelado. Extraídas a
helpers sync (_ddg_text_sync/_ddg_videos_sync) ejecutados vía
run_in_executor. Sin cambios en queries ni procesamiento de resultados.

Verificado con mock de DDG lento (3s): peor parón del loop 52ms.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
ChemaVX
2026-07-04 10:49:22 +00:00
co-authored by Claude Fable 5
parent c701628144
commit 8dfd0116b2
+31 -18
View File
@@ -266,6 +266,21 @@ class ExhaustiveScraper:
logger.warning("SearXNG failed", query=query, error=str(e))
return []
# DDGS es síncrono (requests bloqueante por dentro): llamarlo directamente
# desde código async congela el event loop entero — bot de Telegram incluido —
# mientras dura la búsqueda. Estos helpers deben ejecutarse SIEMPRE vía
# run_in_executor.
@staticmethod
def _ddg_text_sync(query: str) -> list[dict]:
with DDGS() as ddgs:
return list(ddgs.text(query, max_results=settings.max_pages_per_search))
@staticmethod
def _ddg_videos_sync(query: str) -> list[dict]:
with DDGS() as ddgs:
return list(ddgs.videos(query, max_results=10))
async def _seed_search(self):
"""SearXNG primary + DDG fallback per query"""
queries = await self._generate_ddg_queries()
@@ -276,12 +291,10 @@ class ExhaustiveScraper:
if not results:
logger.info("SearXNG vacío, usando DDG", query=query)
try:
with DDGS() as ddgs:
ddg_results = list(ddgs.text(
query,
max_results=settings.max_pages_per_search
))
results = ddg_results
loop = asyncio.get_running_loop()
results = await loop.run_in_executor(
None, self._ddg_text_sync, query
)
logger.info("DDG fallback ok", query=query, results=len(results))
except Exception as e:
logger.warning("DDG fallback failed", query=query, error=str(e))
@@ -395,18 +408,18 @@ class ExhaustiveScraper:
async def _seed_youtube(self):
"""Search YouTube via DDG for video transcripts"""
try:
with DDGS() as ddgs:
results = list(ddgs.videos(
f"{self.topic} documentary explanation",
max_results=10
))
for r in results:
url = r.get("content", "")
if "youtube.com" in url or "youtu.be" in url:
await self.db.add_source(
self.session_id, url, "youtube", depth=0,
title=r.get("title")
)
loop = asyncio.get_running_loop()
results = await loop.run_in_executor(
None, self._ddg_videos_sync,
f"{self.topic} documentary explanation"
)
for r in results:
url = r.get("content", "")
if "youtube.com" in url or "youtu.be" in url:
await self.db.add_source(
self.session_id, url, "youtube", depth=0,
title=r.get("title")
)
except Exception as e:
logger.warning("YouTube seed failed", error=str(e))