fix: correcciones de scraping/DB y mejoras de robustez
Sección crítica: - is_blacklisted: match por dominio/subdominio exacto (antes "x.com" como substring bloqueaba netflix.com, phoenix.com, etc.) - normalize_url: conserva el query string (rompía YouTube watch?v= y URLs con ?id=); solo borra el fragment - get_db: PRAGMA busy_timeout=5000 para evitar "database is locked" en /compare y watches solapados - OllamaClient.embed: usa OLLAMA_EMBED_MODEL en vez del modelo de chat - log_api_call: coste por modelo (opus/sonnet/haiku) en vez de Haiku fijo Mejoras: - src/llm.py: cliente Anthropic compartido y cacheado (antes se instanciaba uno por cada llamada/chunk) - SEARXNG_URL configurable via env - get_running_loop() en vez de get_event_loop() (deprecado) - soup.title.get_text() robusto ante <title> con tags anidados - limpieza: import muerto, total_words duplicado, w_id no usado Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
fd9aaa193b
commit
bf275b7f82
@@ -23,6 +23,7 @@ class OllamaClient:
|
||||
def __init__(self):
|
||||
self.base_url = settings.ollama_url.rstrip("/")
|
||||
self.model = settings.ollama_model
|
||||
self.embed_model = settings.ollama_embed_model
|
||||
|
||||
async def generate(self, prompt: str, system: str = None,
|
||||
timeout: int = 120, temperature: float = 0.7) -> str:
|
||||
@@ -47,7 +48,7 @@ class OllamaClient:
|
||||
|
||||
async def embed(self, text: str) -> Optional[list[float]]:
|
||||
"""Get embedding vector for a text"""
|
||||
payload = {"model": self.model, "prompt": text}
|
||||
payload = {"model": self.embed_model, "prompt": text}
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=60) as client:
|
||||
resp = await client.post(f"{self.base_url}/api/embeddings", json=payload)
|
||||
@@ -124,7 +125,6 @@ class ContentProcessor:
|
||||
async def process_session(self, session_id: int, topic: str,
|
||||
progress_callback=None) -> dict:
|
||||
"""Process all scraped sources for a session"""
|
||||
from src.db.database import ResearchDB
|
||||
sources = await self.db.get_all_sources(session_id)
|
||||
scraped = [s for s in sources if s["status"] == "scraped"]
|
||||
|
||||
@@ -132,7 +132,6 @@ class ContentProcessor:
|
||||
scraped = await self._dedup_sources(session_id, scraped)
|
||||
logger.info("After dedup", unique=len(scraped))
|
||||
total_chunks = 0
|
||||
total_words = 0
|
||||
|
||||
semaphore = asyncio.Semaphore(3) # process 3 sources at once
|
||||
|
||||
@@ -276,7 +275,7 @@ class ContentProcessor:
|
||||
|
||||
async def _score_with_claude(self, chunk: str, topic: str,
|
||||
session_id: int | None = None) -> float:
|
||||
import anthropic
|
||||
from src.llm import get_anthropic_client
|
||||
prompt = (
|
||||
f'Rate 0-10 how relevant this text is to the topic "{topic}". '
|
||||
f'Be generous — if the text is tangentially related, score 4+. '
|
||||
@@ -284,7 +283,7 @@ class ContentProcessor:
|
||||
f'Reply with only a number.\n\nText:\n{chunk[:500]}'
|
||||
)
|
||||
try:
|
||||
client = anthropic.AsyncAnthropic(api_key=settings.anthropic_api_key)
|
||||
client = get_anthropic_client()
|
||||
msg = await client.messages.create(
|
||||
model=settings.claude_model,
|
||||
max_tokens=10,
|
||||
|
||||
Reference in New Issue
Block a user