fix: correcciones de scraping/DB y mejoras de robustez
Sección crítica: - is_blacklisted: match por dominio/subdominio exacto (antes "x.com" como substring bloqueaba netflix.com, phoenix.com, etc.) - normalize_url: conserva el query string (rompía YouTube watch?v= y URLs con ?id=); solo borra el fragment - get_db: PRAGMA busy_timeout=5000 para evitar "database is locked" en /compare y watches solapados - OllamaClient.embed: usa OLLAMA_EMBED_MODEL en vez del modelo de chat - log_api_call: coste por modelo (opus/sonnet/haiku) en vez de Haiku fijo Mejoras: - src/llm.py: cliente Anthropic compartido y cacheado (antes se instanciaba uno por cada llamada/chunk) - SEARXNG_URL configurable via env - get_running_loop() en vez de get_event_loop() (deprecado) - soup.title.get_text() robusto ante <title> con tags anidados - limpieza: import muerto, total_words duplicado, w_id no usado Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
fd9aaa193b
commit
bf275b7f82
@@ -89,15 +89,22 @@ def detect_source_type(url: str) -> str:
|
||||
|
||||
def is_blacklisted(url: str) -> bool:
|
||||
try:
|
||||
domain = urlparse(url).netloc.lower().replace("www.", "")
|
||||
return any(bl in domain for bl in BLACKLIST_DOMAINS)
|
||||
domain = urlparse(url).netloc.lower().split(":")[0]
|
||||
if domain.startswith("www."):
|
||||
domain = domain[4:]
|
||||
# Exact domain or subdomain match — NOT substring (evitaba bloquear
|
||||
# netflix.com / phoenix.com por contener "x.com", etc.)
|
||||
return any(domain == bl or domain.endswith("." + bl)
|
||||
for bl in BLACKLIST_DOMAINS)
|
||||
except Exception:
|
||||
return True
|
||||
|
||||
|
||||
def normalize_url(url: str) -> str:
|
||||
# Strip only the fragment. NO borrar el query string: rompía URLs de
|
||||
# YouTube (watch?v=...) y artículos que enrutan por query (?id=, ?p=).
|
||||
parsed = urlparse(url)
|
||||
clean = parsed._replace(fragment="", query="")
|
||||
clean = parsed._replace(fragment="")
|
||||
return clean.geturl().rstrip("/")
|
||||
|
||||
|
||||
@@ -180,9 +187,9 @@ class ExhaustiveScraper:
|
||||
return fallback
|
||||
|
||||
try:
|
||||
import anthropic
|
||||
from src.llm import get_anthropic_client
|
||||
logger.info("Generating DDG queries with Claude", topic=self.topic)
|
||||
client = anthropic.AsyncAnthropic(api_key=settings.anthropic_api_key)
|
||||
client = get_anthropic_client()
|
||||
prompt = (
|
||||
f'Generate exactly 8 DuckDuckGo search queries to research: "{self.topic}"\n\n'
|
||||
f'Rules:\n'
|
||||
@@ -215,7 +222,7 @@ class ExhaustiveScraper:
|
||||
async def _search_searxng(self, query: str) -> list[dict]:
|
||||
"""Busca en SearXNG y retorna lista de {href, title}. Retorna [] si no disponible."""
|
||||
import aiohttp
|
||||
searxng_url = "http://searxng-svc.researchowl.svc.cluster.local:8080/search"
|
||||
searxng_url = settings.searxng_url
|
||||
params = {
|
||||
"q": query,
|
||||
"format": "json",
|
||||
@@ -533,7 +540,8 @@ class ExhaustiveScraper:
|
||||
|
||||
# Extract title and new URLs with BS4
|
||||
soup = BeautifulSoup(html, "lxml")
|
||||
title = soup.title.string.strip() if soup.title else url
|
||||
# .string es None si el <title> tiene tags anidados; get_text es robusto
|
||||
title = soup.title.get_text(strip=True) if soup.title else url
|
||||
|
||||
new_urls = []
|
||||
if depth < settings.max_depth:
|
||||
@@ -593,7 +601,7 @@ class ExhaustiveScraper:
|
||||
return None, None
|
||||
|
||||
video_id = match.group(1)
|
||||
loop = asyncio.get_event_loop()
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def _fetch():
|
||||
return YouTubeTranscriptApi.get_transcript(
|
||||
|
||||
Reference in New Issue
Block a user