fix(scraper): DDGS síncrono ya no bloquea el event loop
Build & Deploy ResearchOwl / build-and-push (push) Successful in 7s
Build & Deploy ResearchOwl / build-and-push (push) Successful in 7s
Las llamadas DDGS() en _seed_search y _seed_youtube eran síncronas y se ejecutaban directamente en código async: durante cada búsqueda DDG el bot entero (comandos de Telegram incluidos) quedaba congelado. Extraídas a helpers sync (_ddg_text_sync/_ddg_videos_sync) ejecutados vía run_in_executor. Sin cambios en queries ni procesamiento de resultados. Verificado con mock de DDG lento (3s): peor parón del loop 52ms. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
c701628144
commit
8dfd0116b2
+24
-11
@@ -266,6 +266,21 @@ class ExhaustiveScraper:
|
|||||||
logger.warning("SearXNG failed", query=query, error=str(e))
|
logger.warning("SearXNG failed", query=query, error=str(e))
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
# DDGS es síncrono (requests bloqueante por dentro): llamarlo directamente
|
||||||
|
# desde código async congela el event loop entero — bot de Telegram incluido —
|
||||||
|
# mientras dura la búsqueda. Estos helpers deben ejecutarse SIEMPRE vía
|
||||||
|
# run_in_executor.
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _ddg_text_sync(query: str) -> list[dict]:
|
||||||
|
with DDGS() as ddgs:
|
||||||
|
return list(ddgs.text(query, max_results=settings.max_pages_per_search))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _ddg_videos_sync(query: str) -> list[dict]:
|
||||||
|
with DDGS() as ddgs:
|
||||||
|
return list(ddgs.videos(query, max_results=10))
|
||||||
|
|
||||||
async def _seed_search(self):
|
async def _seed_search(self):
|
||||||
"""SearXNG primary + DDG fallback per query"""
|
"""SearXNG primary + DDG fallback per query"""
|
||||||
queries = await self._generate_ddg_queries()
|
queries = await self._generate_ddg_queries()
|
||||||
@@ -276,12 +291,10 @@ class ExhaustiveScraper:
|
|||||||
if not results:
|
if not results:
|
||||||
logger.info("SearXNG vacío, usando DDG", query=query)
|
logger.info("SearXNG vacío, usando DDG", query=query)
|
||||||
try:
|
try:
|
||||||
with DDGS() as ddgs:
|
loop = asyncio.get_running_loop()
|
||||||
ddg_results = list(ddgs.text(
|
results = await loop.run_in_executor(
|
||||||
query,
|
None, self._ddg_text_sync, query
|
||||||
max_results=settings.max_pages_per_search
|
)
|
||||||
))
|
|
||||||
results = ddg_results
|
|
||||||
logger.info("DDG fallback ok", query=query, results=len(results))
|
logger.info("DDG fallback ok", query=query, results=len(results))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("DDG fallback failed", query=query, error=str(e))
|
logger.warning("DDG fallback failed", query=query, error=str(e))
|
||||||
@@ -395,11 +408,11 @@ class ExhaustiveScraper:
|
|||||||
async def _seed_youtube(self):
|
async def _seed_youtube(self):
|
||||||
"""Search YouTube via DDG for video transcripts"""
|
"""Search YouTube via DDG for video transcripts"""
|
||||||
try:
|
try:
|
||||||
with DDGS() as ddgs:
|
loop = asyncio.get_running_loop()
|
||||||
results = list(ddgs.videos(
|
results = await loop.run_in_executor(
|
||||||
f"{self.topic} documentary explanation",
|
None, self._ddg_videos_sync,
|
||||||
max_results=10
|
f"{self.topic} documentary explanation"
|
||||||
))
|
)
|
||||||
for r in results:
|
for r in results:
|
||||||
url = r.get("content", "")
|
url = r.get("content", "")
|
||||||
if "youtube.com" in url or "youtu.be" in url:
|
if "youtube.com" in url or "youtu.be" in url:
|
||||||
|
|||||||
Reference in New Issue
Block a user