fix(scraper): DDGS síncrono ya no bloquea el event loop
Build & Deploy ResearchOwl / build-and-push (push) Successful in 7s
Build & Deploy ResearchOwl / build-and-push (push) Successful in 7s
Las llamadas DDGS() en _seed_search y _seed_youtube eran síncronas y se ejecutaban directamente en código async: durante cada búsqueda DDG el bot entero (comandos de Telegram incluidos) quedaba congelado. Extraídas a helpers sync (_ddg_text_sync/_ddg_videos_sync) ejecutados vía run_in_executor. Sin cambios en queries ni procesamiento de resultados. Verificado con mock de DDG lento (3s): peor parón del loop 52ms. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
c701628144
commit
8dfd0116b2
+24
-11
@@ -266,6 +266,21 @@ class ExhaustiveScraper:
|
||||
logger.warning("SearXNG failed", query=query, error=str(e))
|
||||
return []
|
||||
|
||||
# DDGS es síncrono (requests bloqueante por dentro): llamarlo directamente
|
||||
# desde código async congela el event loop entero — bot de Telegram incluido —
|
||||
# mientras dura la búsqueda. Estos helpers deben ejecutarse SIEMPRE vía
|
||||
# run_in_executor.
|
||||
|
||||
@staticmethod
|
||||
def _ddg_text_sync(query: str) -> list[dict]:
|
||||
with DDGS() as ddgs:
|
||||
return list(ddgs.text(query, max_results=settings.max_pages_per_search))
|
||||
|
||||
@staticmethod
|
||||
def _ddg_videos_sync(query: str) -> list[dict]:
|
||||
with DDGS() as ddgs:
|
||||
return list(ddgs.videos(query, max_results=10))
|
||||
|
||||
async def _seed_search(self):
|
||||
"""SearXNG primary + DDG fallback per query"""
|
||||
queries = await self._generate_ddg_queries()
|
||||
@@ -276,12 +291,10 @@ class ExhaustiveScraper:
|
||||
if not results:
|
||||
logger.info("SearXNG vacío, usando DDG", query=query)
|
||||
try:
|
||||
with DDGS() as ddgs:
|
||||
ddg_results = list(ddgs.text(
|
||||
query,
|
||||
max_results=settings.max_pages_per_search
|
||||
))
|
||||
results = ddg_results
|
||||
loop = asyncio.get_running_loop()
|
||||
results = await loop.run_in_executor(
|
||||
None, self._ddg_text_sync, query
|
||||
)
|
||||
logger.info("DDG fallback ok", query=query, results=len(results))
|
||||
except Exception as e:
|
||||
logger.warning("DDG fallback failed", query=query, error=str(e))
|
||||
@@ -395,11 +408,11 @@ class ExhaustiveScraper:
|
||||
async def _seed_youtube(self):
|
||||
"""Search YouTube via DDG for video transcripts"""
|
||||
try:
|
||||
with DDGS() as ddgs:
|
||||
results = list(ddgs.videos(
|
||||
f"{self.topic} documentary explanation",
|
||||
max_results=10
|
||||
))
|
||||
loop = asyncio.get_running_loop()
|
||||
results = await loop.run_in_executor(
|
||||
None, self._ddg_videos_sync,
|
||||
f"{self.topic} documentary explanation"
|
||||
)
|
||||
for r in results:
|
||||
url = r.get("content", "")
|
||||
if "youtube.com" in url or "youtu.be" in url:
|
||||
|
||||
Reference in New Issue
Block a user