ChemaVXandClaude Fable 5
ae56227c03
Un batch de 20 fuentes concurrentes con un documento de 98k palabras y varios PDFs grandes mató el pod (OOMKilled, límite 1Gi) el 2026-07-10 en plena investigación. - _extract_pdf: cap bajado de 50MB a 15MB, verificado también sobre el body real (Content-Length puede faltar); pdfplumber movido a run_in_executor (es síncrono y congelaba el event loop, misma clase de bug que DDGS) con flush_cache() por página. - _mark_scraped: contenido truncado a settings.max_content_length (300k chars) antes de guardarlo en source_contents — libros enteros inflan RAM y DB sin aportar al RAG. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
128 lines
5.3 KiB
Python
128 lines
5.3 KiB
Python
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
from pydantic import Field
|
|
from typing import Optional
|
|
|
|
# Accept-Encoding para TODA petición aiohttp del proyecto. Nunca anunciar "br":
|
|
# el descompresor incremental de aiohttp 3.14 está roto, y con un backend brotli
|
|
# instalado aiohttp lo anunciaría POR DEFECTO en cualquier sesión sin
|
|
# Accept-Encoding explícito (incidente 2026-07-04, ver KNOWN-ISSUES.md).
|
|
# Toda sesión/petición nueva debe usar esta constante, no un literal.
|
|
SAFE_ACCEPT_ENCODING = "gzip, deflate"
|
|
|
|
|
|
class Settings(BaseSettings):
|
|
# Telegram
|
|
telegram_bot_token: str = Field(...)
|
|
telegram_allowed_users: str = Field("") # comma-separated user IDs
|
|
|
|
# Ollama
|
|
ollama_url: str = Field("http://ollama.chemavx.xyz")
|
|
ollama_model: str = Field("qwen2.5:3b")
|
|
ollama_embed_model: str = Field("qwen2.5:3b")
|
|
|
|
# Claude fallback (optional)
|
|
anthropic_api_key: Optional[str] = Field(None)
|
|
claude_model: str = Field("claude-haiku-4-5")
|
|
|
|
# Database
|
|
db_path: str = Field("/data/researchowl.db")
|
|
|
|
# Scraping
|
|
max_depth: int = Field(3) # recursion depth
|
|
max_sources: int = Field(150) # hard cap
|
|
max_pages_per_search: int = Field(5)
|
|
searxng_url: str = Field(
|
|
"http://searxng-svc.researchowl.svc.cluster.local:8080/search",
|
|
)
|
|
# Motores que el scraper pide a SearXNG (todos vienen configurados por los
|
|
# defaults de la instancia, use_default_settings: true). Verificados en vivo
|
|
# 2026-07-03: la lista ampliada duplica resultados y dominios únicos (~2x)
|
|
# por ~+2s/query, y da redundancia cuando brave/DDG caen en rate limit o
|
|
# CAPTCHA. Excluidos: qwant (access denied), yahoo (roto), wikidata (0
|
|
# resultados en queries de texto), wayback machine (duplica contenido vivo).
|
|
searxng_engines: str = Field(
|
|
"duckduckgo,google,bing,brave,startpage,mojeek,presearch,"
|
|
"google news,bing news,internet archive scholar"
|
|
)
|
|
request_timeout: int = Field(30)
|
|
request_delay: float = Field(1.0) # seconds between requests
|
|
min_content_length: int = Field(200) # chars
|
|
# Libros/dumps enteros (100k+ palabras) inflan RAM y DB sin aportar al RAG
|
|
max_content_length: int = Field(300_000) # chars
|
|
|
|
# Fuentes opcionales — desactivadas por defecto: la IP del homelab está
|
|
# bloqueada por Reddit (403) y YouTube (transcripts vacíos), eran peso muerto.
|
|
enable_youtube: bool = Field(False)
|
|
enable_reddit: bool = Field(False)
|
|
# Bing News RSS como semilla de actualidad — off hasta activarlo a propósito
|
|
enable_news_seed: bool = Field(False)
|
|
|
|
# Processing
|
|
chunk_size: int = Field(800) # tokens per chunk
|
|
chunk_overlap: int = Field(100)
|
|
quality_threshold: float = Field(0.3) # 0-1, chunks below discarded
|
|
|
|
# Ghost CMS
|
|
ghost_url: Optional[str] = Field(None)
|
|
ghost_api_key: Optional[str] = Field(None)
|
|
ghost_url_en: str = Field("")
|
|
ghost_api_key_en: str = Field("")
|
|
|
|
# SEO autofill — "off" | "on" | "dryrun" (default off).
|
|
# off = today's exact behavior (bare draft, no second LLM call).
|
|
# on = adds best-effort meta/OG/Twitter/tags/internal-links to the DRAFT
|
|
# (status stays "draft" — NEVER auto-publishes; see src/seo/autofill.py).
|
|
# dryrun = runs the full pipeline + reports the proposed SEO to Telegram, but
|
|
# writes a BARE draft (no seo fields), for inspection before trusting
|
|
# the write path. A `/generate blog en dry` arg forces dryrun per-call.
|
|
seo_autofill: str = Field("off")
|
|
|
|
# News monitor (RSS de noticias UAP/OVNI) — inerte hasta NEWS_ENABLED=true.
|
|
NEWS_ENABLED: bool = Field(False)
|
|
NEWS_POLL_INTERVAL_HOURS: int = Field(6)
|
|
# Chat al que el monitor empuja novedades; si None -> 1er TELEGRAM_ALLOWED_USERS
|
|
# (ver propiedad news_chat_id).
|
|
NEWS_CHAT_ID: Optional[int] = Field(None)
|
|
NEWS_KEYWORDS: str = Field(
|
|
"ovni,ovnis,uap,ufo,desclasificado,desclasificacion,declassified,"
|
|
"avistamiento,sighting,extraterrestre,non-human,nhi,grusch,"
|
|
"aaro,pursue,fenomeno aereo,whistleblower",
|
|
)
|
|
NEWS_MAX_ITEMS: int = Field(15)
|
|
|
|
# Alerts
|
|
cost_alert_threshold: float = Field(0.15)
|
|
|
|
# App
|
|
log_level: str = Field("INFO")
|
|
timezone: str = Field("Europe/Madrid")
|
|
|
|
@property
|
|
def allowed_user_ids(self) -> list[int]:
|
|
if not self.telegram_allowed_users:
|
|
return []
|
|
return [int(uid.strip()) for uid in self.telegram_allowed_users.split(",") if uid.strip()]
|
|
|
|
@property
|
|
def news_chat_id(self) -> Optional[int]:
|
|
"""Chat destino del monitor de noticias: NEWS_CHAT_ID explícito o, si no
|
|
está definido, el primer TELEGRAM_ALLOWED_USERS."""
|
|
if self.NEWS_CHAT_ID is not None:
|
|
return self.NEWS_CHAT_ID
|
|
ids = self.allowed_user_ids
|
|
return ids[0] if ids else None
|
|
|
|
@property
|
|
def seo_autofill_enabled(self) -> bool:
|
|
"""True when autofill should run (either live 'on' or 'dryrun')."""
|
|
return (self.seo_autofill or "").strip().lower() in ("on", "true", "1", "yes", "dryrun")
|
|
|
|
@property
|
|
def seo_autofill_dryrun(self) -> bool:
|
|
return (self.seo_autofill or "").strip().lower() == "dryrun"
|
|
|
|
model_config = SettingsConfigDict(env_file=".env")
|
|
|
|
|
|
settings = Settings()
|