chore(deps): update dependency fastapi to v0.138.1

KNOWN-ISSUES.md

@@ -1,62 +0,0 @@
-# ResearchOwl — Known Issues & Operational Gotchas
-
-## Brotli / aiohttp incompatibility
-
-Do NOT install any brotli backend (`Brotli`, `brotlicffi`) while on aiohttp 3.14.x.
-aiohttp's incremental brotli decompressor is broken and fails intermittently
-depending on stream chunking ("Can not decode content-encoding: br" on VALID
-streams — the same bytes decompress fine offline; httpx is unaffected).
-
-Extra trap: merely *installing* a backend makes aiohttp advertise `br` in the
-default `Accept-Encoding` of every session that doesn't set one explicitly.
-On 2026-07-04 a scraper fix installed brotlicffi and inadvertently enabled br
-project-wide: Ghost/Cloudflare responded in brotli, the decode blew up AFTER
-Ghost had already accepted the POST, and the publish fallback re-published →
-duplicate drafts + silently lost SEO notices.
-
-Fix applied (commits `397546a` + `76c927f` + `7d07375`, guard relocated
-2026-07-05):
-- no brotli backend in requirements.txt (comment there explains why),
-- explicit `Accept-Encoding: gzip, deflate` on all Ghost/autofill aiohttp
-  sessions and on the scraper HEADERS (`57f341f`),
-- duplicate guard INSIDE `GhostPublisher.publish_draft` (covers every caller,
-  including `/publish`): if the POST is accepted (2xx) but reading the response
-  fails, it recovers the just-created draft via `find_draft_by_title(title,
-  since=attempt_start)` instead of raising. The guard only fires after an
-  accepted POST — a pre-POST failure (Ollama down, menu fetch, links) never
-  triggers it, so a stale same-title draft from a previous run can no longer
-  swallow freshly generated content.
-
-Note: with no backend installed there is NO brotli fallback at all — a server
-that responds `Content-Encoding: br` without it being advertised (misbehaving
-CDN) fails undecodable and that source is lost. Accepted trade-off.
-
-The header value lives in one place: `SAFE_ACCEPT_ENCODING` in `src/config.py`.
-Every aiohttp session/request must use it explicitly — never rely on aiohttp's
-default Accept-Encoding, which silently grows `br` if a backend appears.
-
-Re-test with disclosure.org before ever re-enabling br (e.g. after an aiohttp
-upgrade).
-
-## SQLite WAL mode + read-only mounts
-
-The database runs in WAL mode, so even read-only access needs the `-shm` file
-writable, or must open with `sqlite3 "file:...?immutable=1"`. Backups (daily
-CronJob `researchowl-db-backup`, 03:00 Europe/Madrid, PVC `researchowl-backups`,
-7-day retention) inherit WAL mode: to inspect one from a read-only mount use
-`?immutable=1`; to restore, copy it to a writable location first.
-
-## DDG (duckduckgo_search) blocks the event loop
-
-`DDGS()` is synchronous (blocking requests inside). Never call it directly from
-async code — always wrap in `loop.run_in_executor()` (see `_ddg_text_sync` /
-`_ddg_videos_sync` in `src/scraper/exhaustive.py`). Direct calls froze the
-entire Telegram bot during searches until fixed on 2026-07-04 (`8dfd011`).
-
-## Google News RSS is a dead end from this infrastructure
-
-`news.google.com/rss` entry links point to `/rss/articles/CBMi…` redirects that
-hit a consent wall from EU IPs, and the inner `AU_yqL` id is only resolvable via
-Google's private batchexecute API. Do not retry. The news seed uses Bing News
-RSS instead (`ENABLE_NEWS_SEED`, real publisher URL in the `?url=` param of
-apiclick.aspx — unwrapped by `_unwrap_news_link`).

requirements.txt

@@ -1,25 +1,22 @@
 # Core
-python-telegram-bot==22.8
+fastapi==0.138.1
-httpx==0.28.1
+uvicorn==0.30.0
-aiohttp==3.14.1
+python-telegram-bot==21.5
-# NO instalar backend brotli (Brotli/brotlicffi): el decode incremental br de
+httpx==0.27.0
-# aiohttp 3.14 está roto (falla con streams válidos según el troceo), y con un
+aiohttp==3.10.0
-# backend presente aiohttp anuncia br POR DEFECTO en toda sesión sin
-# Accept-Encoding explícito — el 2026-07-04 eso rompió publish_draft/autofill
-# contra Ghost/Cloudflare (drafts duplicados). Sin backend, nada anuncia br.
 
 # Scraping
-beautifulsoup4==4.15.0
+beautifulsoup4==4.12.3
-lxml==5.4.0
+lxml==5.2.2
 trafilatura==1.12.2
 youtube-transcript-api==0.6.3
-pdfplumber==0.11.10
+pdfplumber==0.11.9
 feedparser==6.0.12
-duckduckgo-search==6.4.2
+duckduckgo-search==6.2.6
 
 # Storage & Embeddings
 sqlite-vec==0.1.9
-aiosqlite==0.22.1
+aiosqlite==0.20.0
 
 # Processing
 tiktoken==0.7.0
@@ -30,12 +27,12 @@ scikit-learn==1.5.1
 anthropic>=0.40.0
 
 # PDF export
-markdown==3.10.2
+markdown==3.7
 reportlab==4.2.5
 
 # Utilities
-pydantic==2.13.4
+pydantic==2.8.0
-pydantic-settings==2.14.2
+pydantic-settings==2.4.0
-tenacity==9.1.4
+tenacity==9.0.0
 structlog==24.4.0
-python-dotenv==1.2.2
+python-dotenv==1.0.1

src/bot/bot.py

@@ -5,12 +5,12 @@ Main user interface — all commands handled here
 import asyncio
 import os
 import time
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta
 from typing import Optional
 from zoneinfo import ZoneInfo
 
 import structlog
-from telegram import Update, Message, LinkPreviewOptions
+from telegram import Update, Message
 from telegram.ext import (
     Application, CommandHandler, MessageHandler,
     filters, ContextTypes
@@ -22,7 +22,6 @@ from src.db.database import get_db, close_db, ResearchDB, ResearchStatus, Output
 from src.scraper.exhaustive import ExhaustiveScraper
 from src.processor.processor import OllamaClient, ContentProcessor
 from src.generator.generator import OutputGenerator
-from src.news.monitor import poll_feeds, item_from_row, format_digest
 
 logger = structlog.get_logger()
 
@@ -397,7 +396,7 @@ async def cmd_generate(update: Update, ctx: ContextTypes.DEFAULT_TYPE):
                 await update.message.reply_text(
                     generator.last_publish_notice,
                     parse_mode=ParseMode.MARKDOWN,
-                    link_preview_options=LinkPreviewOptions(is_disabled=True),
+                    disable_web_page_preview=True,
                 )
             except Exception as e:
                 logger.warning("Failed to send SEO summary message", error=str(e))
@@ -504,36 +503,6 @@ async def cmd_outputs(update: Update, ctx: ContextTypes.DEFAULT_TYPE):
         await db_conn.close()
 
 
-async def cmd_news(update: Update, ctx: ContextTypes.DEFAULT_TYPE):
-    """Monitor RSS manual (sin scheduler — F2 lo automatiza). Refresca news_seen
-    (siembra en cold-start) y muestra las novedades matcheadas de las últimas 24h."""
-    if not is_authorized(update.effective_user.id):
-        return
-
-    db_conn = await get_db()
-    db = ResearchDB(db_conn)
-    try:
-        await update.message.reply_text("📡 Buscando novedades UAP/OVNI…")
-        try:
-            await poll_feeds(db, settings)  # refresca tabla (siembra en cold-start)
-        except Exception:
-            logger.exception("news poll failed")  # best-effort: nunca rompe el comando
-
-        recent = await db.get_recent_news(24)
-        if not recent:
-            await update.message.reply_text("Sin novedades UAP/OVNI en las últimas 24h.")
-            return
-
-        items = [item_from_row(r) for r in recent]
-        for chunk in format_digest(items):
-            # Previews activos a propósito (el digest gana con la tarjeta del link).
-            await update.message.reply_text(
-                chunk, link_preview_options=LinkPreviewOptions(is_disabled=False)
-            )
-    finally:
-        await db_conn.close()
-
-
 async def cmd_costs(update: Update, ctx: ContextTypes.DEFAULT_TYPE):
     if not is_authorized(update.effective_user.id):
         return
@@ -882,9 +851,6 @@ async def _safe_send(bot, chat_id, text: str):
 
 
 async def _scheduler_loop(app: Application):
-    # Estado en memoria (no en DB): en un reinicio re-polla una vez, inofensivo
-    # porque solo se empujan items notified=0 y se marcan tras enviarlos.
-    _last_news_poll: Optional[datetime] = None
     while True:
         db_conn = None
         try:
@@ -944,40 +910,6 @@ async def _scheduler_loop(app: Application):
                     app.bot, chat_id,
                     f"🔄 Investigación automática iniciada: `{topic}`"
                 )
-
-            # --- Monitor de noticias (F2) — inerte si NEWS_ENABLED=False.
-            # Best-effort: un fallo del news-poll NUNCA tumba el scheduler de
-            # watched_topics (va en su propio try/except).
-            if settings.NEWS_ENABLED:
-                now = datetime.now(timezone.utc)
-                interval = timedelta(hours=settings.NEWS_POLL_INTERVAL_HOURS)
-                due_news = (_last_news_poll is None
-                            or (now - _last_news_poll) >= interval)
-                if due_news:
-                    _last_news_poll = now  # marca antes de pollear: evita reintentos en bucle si falla
-                    news_chat_id = settings.news_chat_id
-                    if not news_chat_id:
-                        logger.warning("NEWS_ENABLED pero sin chat destino; salto poll")
-                    else:
-                        try:
-                            await poll_feeds(db, settings)        # inserta novedades notified=0
-                            pending = await db.get_unnotified()   # TODOS los pendientes
-                            if pending:
-                                shown = pending[:settings.NEWS_MAX_ITEMS]
-                                extra = len(pending) - len(shown)
-                                items = [item_from_row(r) for r in shown]
-                                chunks = format_digest(items, header="🛸 Novedades UAP/OVNI")
-                                if extra > 0 and chunks:
-                                    chunks[-1] += f"\n…y {extra} más."
-                                for c in chunks:
-                                    await app.bot.send_message(
-                                        chat_id=news_chat_id, text=c,
-                                        link_preview_options=LinkPreviewOptions(is_disabled=False),
-                                    )
-                                # Marca todos los pendientes (incluidos los "…y N más")
-                                await db.mark_news_notified([r["id"] for r in pending])
-                        except Exception as e:
-                            logger.warning("News poll/notify falló", error=str(e))
         except Exception as e:
             logger.warning("Scheduler loop error", error=str(e))
         finally:
@@ -1360,7 +1292,6 @@ def create_bot() -> Application:
     app.add_handler(CommandHandler("generate", cmd_generate))
     app.add_handler(CommandHandler("sources", cmd_sources))
     app.add_handler(CommandHandler("outputs", cmd_outputs))
-    app.add_handler(CommandHandler("news", cmd_news))
     app.add_handler(CommandHandler("export", cmd_export))
     app.add_handler(CommandHandler("costs", cmd_costs))
     app.add_handler(CommandHandler("watch", cmd_watch))

src/config.py

@@ -1,70 +1,52 @@
-from pydantic_settings import BaseSettings, SettingsConfigDict
+from pydantic_settings import BaseSettings
 from pydantic import Field
 from typing import Optional
 
-# Accept-Encoding para TODA petición aiohttp del proyecto. Nunca anunciar "br":
-# el descompresor incremental de aiohttp 3.14 está roto, y con un backend brotli
-# instalado aiohttp lo anunciaría POR DEFECTO en cualquier sesión sin
-# Accept-Encoding explícito (incidente 2026-07-04, ver KNOWN-ISSUES.md).
-# Toda sesión/petición nueva debe usar esta constante, no un literal.
-SAFE_ACCEPT_ENCODING = "gzip, deflate"
-
 
 class Settings(BaseSettings):
     # Telegram
-    telegram_bot_token: str = Field(...)
+    telegram_bot_token: str = Field(..., env="TELEGRAM_BOT_TOKEN")
-    telegram_allowed_users: str = Field("")  # comma-separated user IDs
+    telegram_allowed_users: str = Field("", env="TELEGRAM_ALLOWED_USERS")  # comma-separated user IDs
 
     # Ollama
-    ollama_url: str = Field("http://ollama.chemavx.xyz")
+    ollama_url: str = Field("http://ollama.chemavx.xyz", env="OLLAMA_URL")
-    ollama_model: str = Field("qwen2.5:3b")
+    ollama_model: str = Field("qwen2.5:3b", env="OLLAMA_MODEL")
-    ollama_embed_model: str = Field("qwen2.5:3b")
+    ollama_embed_model: str = Field("qwen2.5:3b", env="OLLAMA_EMBED_MODEL")
 
     # Claude fallback (optional)
-    anthropic_api_key: Optional[str] = Field(None)
+    anthropic_api_key: Optional[str] = Field(None, env="ANTHROPIC_API_KEY")
-    claude_model: str = Field("claude-haiku-4-5")
+    claude_model: str = Field("claude-haiku-4-5", env="CLAUDE_MODEL")
 
     # Database
-    db_path: str = Field("/data/researchowl.db")
+    db_path: str = Field("/data/researchowl.db", env="DB_PATH")
 
     # Scraping
-    max_depth: int = Field(3)                      # recursion depth
+    max_depth: int = Field(3, env="MAX_DEPTH")                      # recursion depth
-    max_sources: int = Field(150)                # hard cap
+    max_sources: int = Field(150, env="MAX_SOURCES")                # hard cap
-    max_pages_per_search: int = Field(5)
+    max_pages_per_search: int = Field(5, env="MAX_PAGES_PER_SEARCH")
     searxng_url: str = Field(
         "http://searxng-svc.researchowl.svc.cluster.local:8080/search",
+        env="SEARXNG_URL"
     )
-    # Motores que el scraper pide a SearXNG (todos vienen configurados por los
+    request_timeout: int = Field(30, env="REQUEST_TIMEOUT")
-    # defaults de la instancia, use_default_settings: true). Verificados en vivo
+    request_delay: float = Field(1.0, env="REQUEST_DELAY")          # seconds between requests
-    # 2026-07-03: la lista ampliada duplica resultados y dominios únicos (~2x)
+    min_content_length: int = Field(200, env="MIN_CONTENT_LENGTH")  # chars
-    # por ~+2s/query, y da redundancia cuando brave/DDG caen en rate limit o
-    # CAPTCHA. Excluidos: qwant (access denied), yahoo (roto), wikidata (0
-    # resultados en queries de texto), wayback machine (duplica contenido vivo).
-    searxng_engines: str = Field(
-        "duckduckgo,google,bing,brave,startpage,mojeek,presearch,"
-        "google news,bing news,internet archive scholar"
-    )
-    request_timeout: int = Field(30)
-    request_delay: float = Field(1.0)          # seconds between requests
-    min_content_length: int = Field(200)  # chars
 
     # Fuentes opcionales — desactivadas por defecto: la IP del homelab está
     # bloqueada por Reddit (403) y YouTube (transcripts vacíos), eran peso muerto.
-    enable_youtube: bool = Field(False)
+    enable_youtube: bool = Field(False, env="ENABLE_YOUTUBE")
-    enable_reddit: bool = Field(False)
+    enable_reddit: bool = Field(False, env="ENABLE_REDDIT")
-    # Bing News RSS como semilla de actualidad — off hasta activarlo a propósito
-    enable_news_seed: bool = Field(False)
 
     # Processing
-    chunk_size: int = Field(800)                  # tokens per chunk
+    chunk_size: int = Field(800, env="CHUNK_SIZE")                  # tokens per chunk
-    chunk_overlap: int = Field(100)
+    chunk_overlap: int = Field(100, env="CHUNK_OVERLAP")
-    quality_threshold: float = Field(0.3)  # 0-1, chunks below discarded
+    quality_threshold: float = Field(0.3, env="QUALITY_THRESHOLD")  # 0-1, chunks below discarded
 
     # Ghost CMS
-    ghost_url: Optional[str] = Field(None)
+    ghost_url: Optional[str] = Field(None, env="GHOST_URL")
-    ghost_api_key: Optional[str] = Field(None)
+    ghost_api_key: Optional[str] = Field(None, env="GHOST_API_KEY")
-    ghost_url_en: str = Field("")
+    ghost_url_en: str = Field("", env="GHOST_URL_EN")
-    ghost_api_key_en: str = Field("")
+    ghost_api_key_en: str = Field("", env="GHOST_API_KEY_EN")
 
     # SEO autofill — "off" | "on" | "dryrun" (default off).
     #   off    = today's exact behavior (bare draft, no second LLM call).
@@ -73,27 +55,14 @@ class Settings(BaseSettings):
     #   dryrun = runs the full pipeline + reports the proposed SEO to Telegram, but
     #            writes a BARE draft (no seo fields), for inspection before trusting
     #            the write path. A `/generate blog en dry` arg forces dryrun per-call.
-    seo_autofill: str = Field("off")
+    seo_autofill: str = Field("off", env="SEO_AUTOFILL")
-
-    # News monitor (RSS de noticias UAP/OVNI) — inerte hasta NEWS_ENABLED=true.
-    NEWS_ENABLED: bool = Field(False)
-    NEWS_POLL_INTERVAL_HOURS: int = Field(6)
-    # Chat al que el monitor empuja novedades; si None -> 1er TELEGRAM_ALLOWED_USERS
-    # (ver propiedad news_chat_id).
-    NEWS_CHAT_ID: Optional[int] = Field(None)
-    NEWS_KEYWORDS: str = Field(
-        "ovni,ovnis,uap,ufo,desclasificado,desclasificacion,declassified,"
-        "avistamiento,sighting,extraterrestre,non-human,nhi,grusch,"
-        "aaro,pursue,fenomeno aereo,whistleblower",
-    )
-    NEWS_MAX_ITEMS: int = Field(15)
 
     # Alerts
-    cost_alert_threshold: float = Field(0.15)
+    cost_alert_threshold: float = Field(0.15, env="COST_ALERT_THRESHOLD")
 
     # App
-    log_level: str = Field("INFO")
+    log_level: str = Field("INFO", env="LOG_LEVEL")
-    timezone: str = Field("Europe/Madrid")
+    timezone: str = Field("Europe/Madrid", env="TIMEZONE")
 
     @property
     def allowed_user_ids(self) -> list[int]:
@@ -101,15 +70,6 @@ class Settings(BaseSettings):
             return []
         return [int(uid.strip()) for uid in self.telegram_allowed_users.split(",") if uid.strip()]
 
-    @property
-    def news_chat_id(self) -> Optional[int]:
-        """Chat destino del monitor de noticias: NEWS_CHAT_ID explícito o, si no
-        está definido, el primer TELEGRAM_ALLOWED_USERS."""
-        if self.NEWS_CHAT_ID is not None:
-            return self.NEWS_CHAT_ID
-        ids = self.allowed_user_ids
-        return ids[0] if ids else None
-
     @property
     def seo_autofill_enabled(self) -> bool:
         """True when autofill should run (either live 'on' or 'dryrun')."""
@@ -119,7 +79,8 @@ class Settings(BaseSettings):
     def seo_autofill_dryrun(self) -> bool:
         return (self.seo_autofill or "").strip().lower() == "dryrun"
 
-    model_config = SettingsConfigDict(env_file=".env")
+    class Config:
+        env_file = ".env"
 
 
 settings = Settings()

src/db/database.py

@@ -115,22 +115,6 @@ CREATE TABLE IF NOT EXISTS watched_topics (
     created_at REAL NOT NULL,
     UNIQUE(topic, chat_id)
 );
-
-CREATE TABLE IF NOT EXISTS news_seen (
-    id INTEGER PRIMARY KEY,
-    source TEXT NOT NULL,
-    guid TEXT NOT NULL,
-    title TEXT,
-    link TEXT,
-    published_at TEXT,
-    matched_keywords TEXT,
-    seen_at TEXT DEFAULT (datetime('now')),
-    notified INTEGER DEFAULT 0,
-    UNIQUE(source, guid)
-);
-
-CREATE INDEX IF NOT EXISTS idx_news_seen_source ON news_seen(source);
-CREATE INDEX IF NOT EXISTS idx_news_seen_seen ON news_seen(seen_at);
 """
 
 
@@ -512,73 +496,6 @@ class ResearchDB:
         )
         await self.db.commit()
 
-    # --- News monitor ---
-
-    async def source_seeded(self, source: str) -> bool:
-        """True si ya hemos visto algún item de esta fuente (cold-start por fuente:
-        el primer poll siembra sin notificar)."""
-        cursor = await self.db.execute(
-            "SELECT 1 FROM news_seen WHERE source = ? LIMIT 1", (source,)
-        )
-        return await cursor.fetchone() is not None
-
-    async def record_news_item(self, source: str, guid: str, title: str,
-                               link: str, published_at: str, matched: str,
-                               notified: int) -> bool:
-        """INSERT OR IGNORE de un item. Devuelve True solo si es una inserción
-        real (item nuevo); False si ya existía (UNIQUE(source, guid))."""
-        cursor = await self.db.execute(
-            """INSERT OR IGNORE INTO news_seen
-               (source, guid, title, link, published_at, matched_keywords, notified)
-               VALUES (?, ?, ?, ?, ?, ?, ?)""",
-            (source, guid, title, link, published_at, matched, notified)
-        )
-        await self.db.commit()
-        return cursor.rowcount > 0
-
-    async def mark_news_notified(self, ids: list[int]):
-        if not ids:
-            return
-        placeholders = ",".join("?" for _ in ids)
-        await self.db.execute(
-            f"UPDATE news_seen SET notified = 1 WHERE id IN ({placeholders})",
-            tuple(ids)
-        )
-        await self.db.commit()
-
-    async def get_recent_news(self, hours: int = 24) -> list[dict]:
-        """Items matcheados cuya seen_at (o published_at parseable) cae en la
-        ventana. Orden por published_at desc, con seen_at como desempate."""
-        window = f"-{int(hours)} hours"
-        cursor = await self.db.execute(
-            """SELECT * FROM news_seen
-               WHERE seen_at >= datetime('now', ?)
-                  OR (published_at IS NOT NULL AND published_at != ''
-                      AND datetime(published_at) >= datetime('now', ?))
-               ORDER BY published_at DESC, seen_at DESC
-               LIMIT 50""",
-            (window, window)
-        )
-        rows = await cursor.fetchall()
-        return [dict(r) for r in rows]
-
-    async def get_unnotified(self, limit: Optional[int] = None) -> list[dict]:
-        """Items aún sin notificar (notified=0), más recientes primero. El
-        scheduler (F2) los empuja a Telegram y luego los marca con
-        mark_news_notified. Aplica LIMIT solo si `limit` no es None."""
-        sql = (
-            "SELECT id, source, title, link, published_at, matched_keywords "
-            "FROM news_seen WHERE notified = 0 "
-            "ORDER BY published_at DESC NULLS LAST, seen_at DESC"
-        )
-        params: tuple = ()
-        if limit is not None:
-            sql += " LIMIT ?"
-            params = (int(limit),)
-        cursor = await self.db.execute(sql, params)
-        rows = await cursor.fetchall()
-        return [dict(r) for r in rows]
-
     # --- Maintenance ---
 
     async def purge_old_sessions(self, max_age_days: int = 30) -> dict:

src/generator/generator.py

@@ -9,10 +9,9 @@ import json
 import re
 import time
 
-import aiohttp
 import structlog
 
-from src.config import settings, SAFE_ACCEPT_ENCODING
+from src.config import settings
 from src.llm import get_anthropic_client
 from src.processor.processor import OllamaClient, ContentProcessor
 from src.db.database import ResearchDB, OutputType
@@ -285,8 +284,8 @@ def _seo_checklist(slug: str) -> str:
     be all-fail noise. Jose runs `seo-check <slug>` after filling these in."""
     return (
         "\n\n⚠️ Antes de publicar, añade: meta title, meta description (≤145), "
-        "custom excerpt, OG/Twitter, feature image + alt (campo Alt, NO Caption), "
+        "custom excerpt, OG/Twitter, feature image + alt, 2-3 internal links "
-        "2-3 internal links (donde sea natural)\n"
+        "(donde sea natural)\n"
         f"Luego valida:  seo-check {slug}"
     )
 
@@ -313,7 +312,7 @@ def _seo_checklist_slim(slug: str) -> str:
     """Slimmed checklist for the autofill path: meta/OG/excerpt/tags/links are
     already on the draft, so only the human-only bits remain."""
     return (
-        "⚠️ Antes de publicar: añade *feature image + alt* (campo Alt, NO Caption), luego:\n"
+        "⚠️ Antes de publicar: añade *feature image + alt*, luego:\n"
         f"`seo-check {slug}`"
     )
 
@@ -386,24 +385,6 @@ def _seo_dryrun_message(ghost: "GhostPublisher", post: dict, seo: dict, inserted
     )
 
 
-def _norm_title(t: str) -> str:
-    """Normaliza un título para compararlo con lo que Ghost almacenó: colapsa
-    whitespace y trunca a 255 chars (límite de posts.title en Ghost — un H1
-    largo del LLM puede volver truncado/normalizado, y la igualdad exacta
-    fallaría justo en los casos que el guard debe cubrir)."""
-    return " ".join((t or "").split())[:255]
-
-
-def _parse_ghost_ts(ts: str) -> float | None:
-    """created_at de la Admin API (ISO 8601, p.ej. 2026-07-04T20:26:00.000+00:00
-    o sufijo Z) → epoch. None si no parsea."""
-    try:
-        from datetime import datetime
-        return datetime.fromisoformat(ts.replace("Z", "+00:00")).timestamp()
-    except Exception:
-        return None
-
-
 class GhostPublisher:
     def __init__(self, lang: str = "es"):
         self.lang = lang
@@ -438,64 +419,6 @@ class GhostPublisher:
         )
         return f"{signing}.{sig}"
 
-    def _admin_headers(self) -> dict:
-        """Headers canónicos de la Admin API (token JWT fresco por llamada).
-        Único sitio donde se construyen: cualquier cambio (Accept-Version,
-        encoding) se hace aquí y cubre todas las llamadas a Ghost."""
-        return {
-            "Authorization": f"Ghost {self._make_token()}",
-            "Accept-Version": "v5.0",
-            # Nunca heredar el default de aiohttp: con un backend brotli
-            # instalado anunciaría br y el decode roto de aiohttp 3.14 rompe
-            # la lectura de respuestas (incidente 2026-07-04, KNOWN-ISSUES.md).
-            "Accept-Encoding": SAFE_ACCEPT_ENCODING,
-        }
-
-    async def _admin_get(self, query: str, timeout: float = 15) -> dict | None:
-        """GET a {url}/ghost/api/admin/{query} con auth + headers canónicos.
-        None si non-200 (logueado); las excepciones de red/parseo suben tal
-        cual para que cada caller aplique su propia política best-effort."""
-        async with aiohttp.ClientSession(
-                timeout=aiohttp.ClientTimeout(total=timeout)) as sess:
-            async with sess.get(f"{self.url}/ghost/api/admin/{query}",
-                                headers=self._admin_headers()) as resp:
-                if resp.status != 200:
-                    body = await resp.text()
-                    logger.warning("Ghost admin GET non-200", query=query[:80],
-                                   status=resp.status, body=body[:200])
-                    return None
-                return await resp.json()
-
-    async def find_draft_by_title(self, title: str,
-                                  since: float | None = None) -> dict | None:
-        """Busca entre los drafts recientes uno con título exacto.
-
-        Mitad de recuperación del guard anti-duplicados de publish_draft: si el
-        POST fue aceptado por Ghost pero la lectura de la respuesta falló
-        (p.ej. decode roto), el draft existe aunque el caller viera una
-        excepción. Con `since` (epoch) solo casan drafts creados en este
-        intento — un draft viejo del mismo título de una run anterior NO cuenta
-        como "ya publicado". La comparación usa _norm_title (whitespace
-        colapsado + truncado a 255) para sobrevivir la normalización de Ghost.
-        Best-effort: None si no hay match o si falla."""
-        data = await self._admin_get(
-            "posts/?filter=status:draft&order=created_at%20desc&limit=15"
-            "&fields=id,title,slug,created_at"
-        )
-        if data is None:
-            return None
-        wanted = _norm_title(title)
-        for p in data.get("posts", []):
-            if _norm_title(p.get("title") or "") != wanted:
-                continue
-            if since is not None:
-                created = _parse_ghost_ts(p.get("created_at") or "")
-                # 60s de margen por posible desfase de reloj Ghost/bot.
-                if created is None or created < since - 60:
-                    continue
-            return p
-        return None
-
     async def publish_draft(self, title: str, markdown_content: str,
                             tags: list[str] | None = None,
                             seo: dict | None = None,
@@ -509,6 +432,8 @@ class GhostPublisher:
           * seo — a generate_seo_fields dict; when given, its meta/OG/Twitter fields
             are added to the post and its (allow-list) tags are used.
         """
+        import aiohttp as _aio
+
         # Caller-supplied linked HTML wins; otherwise convert markdown as before.
         html = body_html if body_html is not None else self._build_html(markdown_content)
 
@@ -529,6 +454,7 @@ class GhostPublisher:
             "sections": [[10, 0]],
         })
 
+        token = self._make_token()
         # Language-aware default tag (fixes the hardcoded "investigacion" for EN).
         tag_names = tags or [_DEFAULT_TAG.get(self.lang, "investigation")]
         post_obj = {
@@ -548,40 +474,19 @@ class GhostPublisher:
                 "twitter_description": seo["twitter_description"],
             })
         body = {"posts": [post_obj]}
-        attempt_start = time.time()
+        async with _aio.ClientSession() as sess:
-        async with aiohttp.ClientSession() as sess:
             async with sess.post(
                 f"{self.url}/ghost/api/admin/posts/",
                 json=body,
-                headers=self._admin_headers(),
+                headers={
+                    "Authorization": f"Ghost {token}",
+                    "Accept-Version": "v5.0",
+                },
             ) as resp:
                 if resp.status not in (200, 201):
                     text = await resp.text()
                     raise ValueError(f"Ghost API {resp.status}: {text[:300]}")
-                # Guard anti-duplicados (incidente 2026-07-04): el 2xx confirma
+                return await resp.json()
-                # que Ghost YA creó el draft; si la lectura del body falla
-                # (p.ej. decode br roto), recuperamos el draft recién creado en
-                # vez de propagar — propagar hacía que el caller (o el usuario
-                # con /publish) reintentara y duplicara. El guard vive AQUÍ para
-                # cubrir a todos los callers, y solo se activa tras un POST
-                # aceptado: un fallo pre-POST jamás lo dispara.
-                try:
-                    return await resp.json()
-                except Exception as read_err:
-                    existing = None
-                    try:
-                        existing = await self.find_draft_by_title(
-                            title, since=attempt_start)
-                    except Exception as find_err:
-                        logger.warning("publish_draft: draft-exists check failed",
-                                       error=str(find_err))
-                    if existing:
-                        logger.warning(
-                            "publish_draft: POST aceptado pero lectura de la "
-                            "respuesta falló; draft recuperado sin re-publicar",
-                            post_id=existing["id"], error=str(read_err))
-                        return {"posts": [existing]}
-                    raise
 
 
 # ─── Output generation ────────────────────────────────────────────────────────
@@ -649,11 +554,6 @@ class OutputGenerator:
             except Exception as e:
                 logger.warning("SEO autofill failed; falling back to bare draft",
                                error=str(e))
-                # Sin guard anti-duplicados aquí: vive DENTRO de publish_draft
-                # (solo se activa tras un POST aceptado). Un fallo pre-POST
-                # (Ollama caído, menú, links) cae aquí y DEBE publicar el
-                # contenido nuevo — un guard por título casaba con drafts
-                # viejos del mismo topic y lo descartaba en silencio.
                 # fall through to the bare-draft publish below
 
         # OFF mode, or autofill failed → today's exact bare-draft publish.

src/news/monitor.py

@@ -1,207 +0,0 @@
-"""Monitor RSS de noticias UAP/OVNI.
-
-Capa independiente: NUNCA importa de src.bot (el bot importa de aquí). No toca el
-scheduler — F2. Best-effort por diseño: un feed caído se loguea y se salta; nunca
-tumba el run. El estado vive en la tabla news_seen vía los métodos de ResearchDB,
-que se inyectan (no se importan) para mantener el desacople.
-"""
-import asyncio
-import re
-import unicodedata
-from dataclasses import dataclass, field
-
-import feedparser
-import structlog
-
-log = structlog.get_logger()
-
-# Reddit devuelve 403 con el UA por defecto de feedparser/python; uno explícito
-# (y honesto) pasa. El resto de feeds lo aceptan sin problema.
-UA = "ExclusionZone-NewsBot/1.0 (+https://theexclusionzone.com)"
-
-FEEDS: list[dict] = [
-    {"name": "Gran Misterio",      "url": "https://granmisterio.org/feed/",                                                     "lang": "es"},
-    # Reemplazos ES tras retirar Espacio Misterio (XML roto). Verificados 2026-07-03:
-    # MysteryPlanet y Marcianitos publican a diario/casi a diario; El Ojo Crítico es
-    # mensual (poco volumen a propósito). Ufopolis/MundoEsoterico descartados (muerto/roto).
-    {"name": "MysteryPlanet",      "url": "https://mysteryplanet.com.ar/site/feed/",                                            "lang": "es"},
-    {"name": "Marcianitos Verdes", "url": "https://marcianitosverdes.haaan.com/feed/",                                          "lang": "es"},
-    {"name": "El Ojo Crítico",     "url": "https://elojocritico.info/feed/",                                                    "lang": "es"},
-    {"name": "Josep Guijarro (YT)", "url": "https://www.youtube.com/feeds/videos.xml?channel_id=UCIlGy26zi-BbX6zDYAMQ9Qg",      "lang": "es"},
-    {"name": "r/UFOs",             "url": "https://www.reddit.com/r/UFOs/new/.rss",                                             "lang": "en"},
-    {"name": "r/UFOB",             "url": "https://www.reddit.com/r/UFOB/new/.rss",                                             "lang": "en"},
-    {"name": "Liberation Times",   "url": "https://www.liberationtimes.com/?format=rss",                                        "lang": "en"},
-    {"name": "The Debrief",        "url": "https://thedebrief.org/feed/",                                                       "lang": "en"},
-]
-
-# Tokens cortos: word-boundary para evitar falsos positivos dentro de palabras
-# (p.ej. "aaro" en "aaron", "ufo" en "tufo"). Frases multi-palabra y términos
-# largos van por substring. Se decide dinámicamente: sin espacios y len <= 4.
-_SHORT_TOKEN_MAXLEN = 4
-
-# Límite de Telegram 4096; troceamos por debajo con margen.
-_TG_LIMIT = 4000
-
-# Reddit sin OAuth ratelimita por IP con ventana larga (>30s verificado): la 2ª
-# petición seguida a reddit.com devuelve 429 SIEMPRE, así que solo se pollea un
-# feed de reddit por run, rotando. Contador por proceso: tras un reinicio se
-# empieza por el primero, lo cual es inocuo.
-_reddit_turn = 0
-
-
-def _is_reddit(feed: dict) -> bool:
-    return "reddit.com" in feed["url"]
-
-
-@dataclass
-class NewsItem:
-    source: str
-    guid: str
-    title: str
-    link: str
-    published: str
-    summary: str
-    matched: list[str] = field(default_factory=list)
-
-
-def _normalize(text: str) -> str:
-    """minúsculas + NFKD sin marcas combinantes (quita tildes/diacríticos)."""
-    if not text:
-        return ""
-    decomposed = unicodedata.normalize("NFKD", text.lower())
-    return "".join(c for c in decomposed if not unicodedata.combining(c))
-
-
-def _is_short_token(kw: str) -> bool:
-    return " " not in kw and len(kw) <= _SHORT_TOKEN_MAXLEN
-
-
-def _match(title: str, summary: str, keywords: list[str]) -> list[str]:
-    """Devuelve las keywords (en su forma original) que matchean en title+summary,
-    sobre texto normalizado. Word-boundary para tokens cortos, substring para el
-    resto. Sin duplicados, preservando el orden de `keywords`."""
-    text = _normalize(f"{title} {summary}")
-    if not text:
-        return []
-    matched: list[str] = []
-    for kw in keywords:
-        nkw = _normalize(kw)
-        if not nkw:
-            continue
-        if _is_short_token(nkw):
-            if re.search(rf"\b{re.escape(nkw)}\b", text):
-                matched.append(kw)
-        else:
-            if nkw in text:
-                matched.append(kw)
-    return matched
-
-
-async def poll_feeds(db, settings) -> list[NewsItem]:
-    """Recorre los feeds, registra items matcheados en news_seen y devuelve los
-    NUEVOS (no vistos) de fuentes ya sembradas. Cold-start por fuente: el primer
-    poll de cada feed siembra en silencio (notified=1) y no devuelve nada.
-
-    Best-effort: cada feed va en su propio try/except — uno caído nunca tumba el
-    run. feedparser.parse es bloqueante (I/O de red), así que se ejecuta en un
-    hilo para no congelar el event loop del bot durante el poll.
-
-    Reddit: solo se pollea UN feed de reddit.com por run, rotando entre ellos
-    (ver _reddit_turn) — dos peticiones seguidas garantizan un 429 en la 2ª.
-    """
-    global _reddit_turn
-    keywords = [k.strip() for k in settings.NEWS_KEYWORDS.split(",") if k.strip()]
-    new_items: list[NewsItem] = []
-
-    reddit_feeds = [f for f in FEEDS if _is_reddit(f)]
-    skip: set[str] = set()
-    if len(reddit_feeds) > 1:
-        chosen = reddit_feeds[_reddit_turn % len(reddit_feeds)]
-        _reddit_turn += 1
-        skip = {f["name"] for f in reddit_feeds if f["name"] != chosen["name"]}
-
-    for feed in FEEDS:
-        if feed["name"] in skip:
-            continue
-        try:
-            parsed = await asyncio.to_thread(feedparser.parse, feed["url"], agent=UA)
-            status = getattr(parsed, "status", None)
-            # Un 429/403 llega con bozo=False y 0 entries: sin este guard se
-            # tragaba en silencio (así estuvo r/UFOB sin sembrar desde F1).
-            if status and status >= 400:
-                log.warning("feed %s HTTP %s; salto", feed["name"], status)
-                continue
-            if parsed.bozo and not parsed.entries:
-                log.warning("feed bozo sin entries: %s (%s)",
-                            feed["name"], getattr(parsed, "bozo_exception", ""))
-                continue
-
-            seeded = await db.source_seeded(feed["name"])  # cold-start por fuente
-            for e in parsed.entries:
-                guid = getattr(e, "id", None) or getattr(e, "link", "")
-                if not guid:
-                    continue
-                matched = _match(e.get("title", ""), e.get("summary", ""), keywords)
-                if not matched:
-                    continue
-                notified = 1 if not seeded else 0  # 1er run: sembrar sin notificar
-                is_new = await db.record_news_item(
-                    feed["name"], guid, e.get("title", ""), e.get("link", ""),
-                    e.get("published", ""), ",".join(matched), notified,
-                )
-                if is_new and seeded:
-                    new_items.append(NewsItem(
-                        feed["name"], guid, e.get("title", ""), e.get("link", ""),
-                        e.get("published", ""), e.get("summary", ""), matched,
-                    ))
-        except Exception as ex:  # aislado: un feed no tumba el run
-            log.exception("fallo feed %s: %s", feed["name"], ex)
-
-    return new_items
-
-
-def item_from_row(row: dict) -> NewsItem:
-    """Adapta una fila de get_recent_news (dict de news_seen) a NewsItem para que
-    format_digest sea agnóstico de la fuente (poll vs DB)."""
-    matched = row.get("matched_keywords") or ""
-    return NewsItem(
-        source=row.get("source", ""),
-        guid=row.get("guid", ""),
-        title=row.get("title", "") or "",
-        link=row.get("link", "") or "",
-        published=row.get("published_at", "") or "",
-        summary="",
-        matched=[m for m in matched.split(",") if m],
-    )
-
-
-def format_digest(items: list[NewsItem], header: str = "🛸 Novedades UAP/OVNI") -> list[str]:
-    """Agrupa por fuente y formatea cada item como '• <title>\\n  <link>'. Trocea
-    en mensajes < _TG_LIMIT chars (devuelve varios si hace falta). Texto plano
-    (no Markdown): el handler envía sin parse_mode. Lista vacía -> []."""
-    if not items:
-        return []
-
-    # Agrupar por fuente preservando el orden de primera aparición.
-    groups: dict[str, list[NewsItem]] = {}
-    for it in items:
-        groups.setdefault(it.source, []).append(it)
-
-    blocks: list[str] = []  # un bloque por fuente, indivisible salvo que él solo exceda
-    for source, its in groups.items():
-        lines = [f"── {source} ──"]
-        for it in its:
-            title = (it.title or "(sin título)").strip()
-            lines.append(f"• {title}\n  {it.link}".rstrip())
-        blocks.append("\n".join(lines))
-
-    chunks: list[str] = []
-    cur = header
-    for block in blocks:
-        if len(cur) + len(block) + 2 > _TG_LIMIT and cur.strip() != header:
-            chunks.append(cur)
-            cur = f"{header} (cont.)"
-        cur += "\n\n" + block
-    if cur.strip():
-        chunks.append(cur)
-    return chunks

src/scraper/exhaustive.py

@@ -7,7 +7,7 @@ import random
 import re
 import time
 from typing import Optional
-from urllib.parse import urljoin, urlparse, parse_qs, quote, quote_plus
+from urllib.parse import urljoin, urlparse, quote_plus
 
 import aiohttp
 import feedparser
@@ -18,7 +18,7 @@ from duckduckgo_search import DDGS
 from youtube_transcript_api import YouTubeTranscriptApi, NoTranscriptFound
 from tenacity import retry, stop_after_attempt, wait_exponential
 
-from src.config import settings, SAFE_ACCEPT_ENCODING
+from src.config import settings
 from src.db.database import ResearchDB
 
 logger = structlog.get_logger()
@@ -27,13 +27,7 @@ HEADERS = {
     "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
     "Accept-Language": "en-US,en;q=0.9,es;q=0.8",
-    # Sin "br": el descompresor incremental de aiohttp 3.14 falla con streams
+    "Accept-Encoding": "gzip, deflate, br",
-    # brotli válidos según el troceo de chunks (disclosure.org, todaywhy.com;
-    # los mismos bytes descomprimen bien offline). Con gzip el camino zlib es
-    # sólido. OJO: ya NO hay ningún backend brotli instalado (397546a quitó
-    # brotlicffi) — si un servidor responde br sin que se haya anunciado,
-    # aiohttp falla sin recuperación posible y esa fuente se pierde.
-    "Accept-Encoding": SAFE_ACCEPT_ENCODING,
     "DNT": "1",
 }
 
@@ -42,7 +36,7 @@ REDDIT_HEADERS = {
     "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
     "Accept": "application/json, text/javascript, */*; q=0.01",
     "Accept-Language": "en-US,en;q=0.9",
-    "Accept-Encoding": SAFE_ACCEPT_ENCODING,  # sin "br", ver HEADERS
+    "Accept-Encoding": "gzip, deflate, br",
     "Referer": "https://www.reddit.com/",
     "X-Requested-With": "XMLHttpRequest",
 }
@@ -59,10 +53,6 @@ YOUTUBE_RE = re.compile(r"(?:youtube\.com/watch\?v=|youtu\.be/)([a-zA-Z0-9_-]{11
 PDF_RE = re.compile(r"\.pdf(\?|$)", re.IGNORECASE)
 REDDIT_RE = re.compile(r"reddit\.com/(r/\w+/comments/\w+)")
 WIKIPEDIA_RE = re.compile(r"wikipedia\.org/wiki/(.+)")
-# Segmentos de path típicos de feeds, con límite de palabra: el substring puro
-# clasificaba /feedback o /atomic como rss.
-RSS_RE = re.compile(r"/(rss|feeds?|atom)(/|\.xml|\.rss|$)|\.rss$|format=rss",
-                    re.IGNORECASE)
 
 
 async def fetch_with_retry(fetch_fn, source_name: str, max_retries: int = 3):
@@ -92,7 +82,7 @@ def detect_source_type(url: str) -> str:
         return "wikipedia"
     if "arxiv.org" in url:
         return "arxiv"
-    if RSS_RE.search(url):
+    if any(d in url for d in ["rss", "feed", "atom"]):
         return "rss"
     return "web"
 
@@ -110,25 +100,6 @@ def is_blacklisted(url: str) -> bool:
         return True
 
 
-def _unwrap_news_link(link: str) -> str:
-    """Bing News envuelve los links de entry en apiclick.aspx; la URL real del
-    publisher viene en texto plano en el parámetro ?url=. Cualquier otro link
-    pasa sin tocar. Si es un apiclick pero no trae URL usable, devuelve "" —
-    el caller lo descarta por startswith("http") — en vez de dejar pasar el
-    redirect de bing.com, que se rasparía como página de consent/basura."""
-    try:
-        parsed = urlparse(link)
-        if (parsed.netloc.lower().endswith("bing.com")
-                and parsed.path == "/news/apiclick.aspx"):
-            real = parse_qs(parsed.query).get("url", [""])[0].strip()
-            if real.startswith("//"):  # protocol-relative
-                real = "https:" + real
-            return real if real.startswith("http") else ""
-    except Exception:
-        return ""
-    return link
-
-
 def normalize_url(url: str) -> str:
     # Strip only the fragment. NO borrar el query string: rompía URLs de
     # YouTube (watch?v=...) y artículos que enrutan por query (?id=, ?p=).
@@ -196,14 +167,11 @@ class ExhaustiveScraper:
         tasks = [
             self._seed_search(),
             self._seed_wikipedia(),
-            self._seed_archive(),
         ]
         if settings.enable_reddit:
             tasks.append(self._seed_reddit())
         if settings.enable_youtube:
             tasks.append(self._seed_youtube())
-        if settings.enable_news_seed:
-            tasks.append(self._seed_news())
         await asyncio.gather(*tasks, return_exceptions=True)
 
     async def _generate_ddg_queries(self) -> list[str]:
@@ -261,17 +229,13 @@ class ExhaustiveScraper:
         params = {
             "q": query,
             "format": "json",
-            "engines": settings.searxng_engines,
+            "engines": "duckduckgo,google,bing,brave",
             "language": "all",
         }
         headers = {
             "Accept": "application/json",
             "X-Forwarded-For": "127.0.0.1",
             "User-Agent": "ResearchOwl/1.0",
-            # Sin esto la sesión hereda el default de aiohttp, que anunciaría
-            # br si algún día se reinstala un backend brotli — y el decode roto
-            # de 3.14 tumbaría TODAS las búsquedas SearXNG (el camino primario).
-            "Accept-Encoding": SAFE_ACCEPT_ENCODING,
         }
         try:
             async with aiohttp.ClientSession() as session:
@@ -297,21 +261,6 @@ class ExhaustiveScraper:
             logger.warning("SearXNG failed", query=query, error=str(e))
             return []
 
-    # DDGS es síncrono (requests bloqueante por dentro): llamarlo directamente
-    # desde código async congela el event loop entero — bot de Telegram incluido —
-    # mientras dura la búsqueda. Estos helpers deben ejecutarse SIEMPRE vía
-    # run_in_executor.
-
-    @staticmethod
-    def _ddg_text_sync(query: str) -> list[dict]:
-        with DDGS() as ddgs:
-            return list(ddgs.text(query, max_results=settings.max_pages_per_search))
-
-    @staticmethod
-    def _ddg_videos_sync(query: str) -> list[dict]:
-        with DDGS() as ddgs:
-            return list(ddgs.videos(query, max_results=10))
-
     async def _seed_search(self):
         """SearXNG primary + DDG fallback per query"""
         queries = await self._generate_ddg_queries()
@@ -322,10 +271,12 @@ class ExhaustiveScraper:
             if not results:
                 logger.info("SearXNG vacío, usando DDG", query=query)
                 try:
-                    loop = asyncio.get_running_loop()
+                    with DDGS() as ddgs:
-                    results = await loop.run_in_executor(
+                        ddg_results = list(ddgs.text(
-                        None, self._ddg_text_sync, query
+                            query,
-                    )
+                            max_results=settings.max_pages_per_search
+                        ))
+                    results = ddg_results
                     logger.info("DDG fallback ok", query=query, results=len(results))
                 except Exception as e:
                     logger.warning("DDG fallback failed", query=query, error=str(e))
@@ -343,92 +294,30 @@ class ExhaustiveScraper:
             await asyncio.sleep(random.uniform(1, 3))
 
     async def _seed_wikipedia(self):
-        """Siembra Wikipedia en AMBOS idiomas siempre, con búsqueda full-text
+        """Search Wikipedia API for correct article URLs.
-        (list=search). El opensearch anterior era prefix-de-título y devolvía 0
+        Tries English first, falls back to Spanish if no results found."""
-        para topics verbosos ("incidente ovni de Manises 1979") aunque el
-        artículo exista; y el break tras en dejaba es solo como fallback.
-        Cada idioma va aislado: un fallo no afecta al otro."""
         http = await self._get_http()
+        added = 0
 
         for lang in ("en", "es"):
             try:
                 api_url = (
-                    f"https://{lang}.wikipedia.org/w/api.php?action=query"
+                    f"https://{lang}.wikipedia.org/w/api.php?action=opensearch"
-                    f"&list=search&srsearch={quote_plus(self.topic)}"
+                    f"&search={quote_plus(self.topic)}&limit=10&format=json"
-                    f"&srlimit=8&format=json"
                 )
                 async with http.get(api_url) as resp:
                     data = await resp.json()
-                hits = data.get("query", {}).get("search", [])
+                    urls = data[3] if len(data) > 3 else []
-                added = 0
+                    for url in urls:
-                for h in hits:
+                        if url:
-                    title = h.get("title")
+                            await self.db.add_source(self.session_id, url, "wikipedia", depth=0)
-                    if not title:
+                            added += 1
-                        continue
+                logger.info("Wikipedia seed", lang=lang, found=len(urls))
-                    url = (f"https://{lang}.wikipedia.org/wiki/"
+                if added > 0:
-                           f"{quote(title.replace(' ', '_'))}")
+                    break  # English results found — no need to try Spanish
-                    await self.db.add_source(self.session_id, url, "wikipedia",
-                                             depth=0, title=title)
-                    added += 1
-                logger.info("Wikipedia seed", lang=lang, found=added)
             except Exception as e:
                 logger.warning("Wikipedia API seed failed", lang=lang, error=str(e))
 
-    async def _seed_archive(self):
-        """Siembra Internet Archive (advancedsearch, mediatype:texts): documentos
-        FOIA, informes y libros históricos. Se siembra la página /details/<id>;
-        sus PDFs los descubre la recursión (la propia página los enlaza) y los
-        extrae _extract_pdf. Sin fallback de query: el AND estricto de archive.org
-        con el topic completo prima precisión — relajarlo mete ruido (verificado:
-        'Manises' a secas devuelve loza valenciana). Best-effort: nunca rompe el seed."""
-        try:
-            http = await self._get_http()
-            q = quote_plus(f"({self.topic}) AND mediatype:(texts)")
-            url = (
-                f"https://archive.org/advancedsearch.php?q={q}"
-                "&fl[]=identifier&fl[]=title&rows=12&page=1&output=json"
-            )
-            async with http.get(url) as resp:
-                if resp.status != 200:
-                    logger.warning("Archive.org seed non-200", status=resp.status)
-                    return
-                data = await resp.json(content_type=None)
-            docs = data.get("response", {}).get("docs", [])
-            added = 0
-            for d in docs:
-                ident = d.get("identifier")
-                if not ident:
-                    continue
-                item_url = f"https://archive.org/details/{ident}"
-                title = d.get("title")
-                await self.db.add_source(
-                    self.session_id, item_url, "web", depth=0,
-                    title=title if isinstance(title, str) else None
-                )
-                added += 1
-            logger.info("Archive.org seed", found=len(docs), added=added)
-        except Exception as e:
-            logger.warning("Archive.org seed failed", error=str(e))
-
-    async def _seed_news(self):
-        """Siembra Bing News RSS para cobertura de actualidad. Solo registra
-        el feed como fuente tipo rss: _extract_rss (dispatch del pipeline) lo
-        parsea y siembra sus entries — sin duplicar lógica de parseo aquí.
-        Bing y no Google News: los links de Google (news.google.com/rss/articles/
-        CBMi…) van a un muro de consent + redirect cifrado solo resoluble vía su
-        API interna batchexecute (verificado 2026-07-04); los de Bing llevan la
-        URL real del publisher en ?url= (ver _unwrap_news_link).
-        Best-effort: nunca rompe el seed."""
-        try:
-            url = f"https://www.bing.com/news/search?q={quote_plus(self.topic)}&format=rss"
-            await self.db.add_source(
-                self.session_id, url, "rss", depth=0,
-                title=f"Bing News: {self.topic}"
-            )
-            logger.info("News seed added", topic=self.topic)
-        except Exception as e:
-            logger.warning("News seed failed", error=str(e))
-
     async def _seed_reddit(self):
         """Search Reddit — sequential to avoid rate limiting"""
         try:
@@ -458,18 +347,18 @@ class ExhaustiveScraper:
     async def _seed_youtube(self):
         """Search YouTube via DDG for video transcripts"""
         try:
-            loop = asyncio.get_running_loop()
+            with DDGS() as ddgs:
-            results = await loop.run_in_executor(
+                results = list(ddgs.videos(
-                None, self._ddg_videos_sync,
+                    f"{self.topic} documentary explanation",
-                f"{self.topic} documentary explanation"
+                    max_results=10
-            )
+                ))
-            for r in results:
+                for r in results:
-                url = r.get("content", "")
+                    url = r.get("content", "")
-                if "youtube.com" in url or "youtu.be" in url:
+                    if "youtube.com" in url or "youtu.be" in url:
-                    await self.db.add_source(
+                        await self.db.add_source(
-                        self.session_id, url, "youtube", depth=0,
+                            self.session_id, url, "youtube", depth=0,
-                        title=r.get("title")
+                            title=r.get("title")
-                    )
+                        )
         except Exception as e:
             logger.warning("YouTube seed failed", error=str(e))
 
@@ -583,9 +472,6 @@ class ExhaustiveScraper:
                         lambda: self._extract_reddit(url), url
                     )
                     await asyncio.sleep(settings.request_delay)
-                elif source_type == "rss":
-                    # El feed no tiene contenido propio: es puro descubrimiento.
-                    return await self._extract_rss(source_id, url, source["depth"])
                 elif source_type == "pdf":
                     content, title = await fetch_with_retry(
                         lambda: self._extract_pdf(url), url
@@ -783,42 +669,6 @@ class ExhaustiveScraper:
             logger.warning("Reddit extraction failed", url=url, error=str(e))
             return None, None
 
-    async def _extract_rss(self, source_id: int, url: str, depth: int) -> int:
-        """RSS/Atom: siembra las entries del feed como fuentes nuevas (filtradas
-        por relevancia de título/URL) y marca el feed como skipped — no tiene
-        contenido propio que trocear. feedparser.parse es bloqueante: va en hilo,
-        igual que en el monitor de noticias. Devuelve nº de fuentes añadidas."""
-        parsed = await asyncio.to_thread(feedparser.parse, url,
-                                         agent=HEADERS["User-Agent"])
-        entries = parsed.entries or []
-        added = 0
-        if depth < settings.max_depth:
-            # Se escanean TODAS las entries y el cap aplica a las añadidas:
-            # capar antes de filtrar perdía las relevantes que no caen entre
-            # las 20 más recientes (verificado con thedebrief: 0/20 vs 3/100).
-            for e in entries:
-                if added >= 20:  # cap, como los 30 links/página de web
-                    break
-                link = normalize_url(_unwrap_news_link(getattr(e, "link", "") or ""))
-                title = getattr(e, "title", "") or ""
-                if (not link.startswith("http") or is_blacklisted(link)
-                        or not self._url_is_relevant(link, title)):
-                    continue
-                if await self.db.source_exists(self.session_id, link):
-                    continue
-                await self.db.add_source(
-                    self.session_id, link, detect_source_type(link),
-                    depth=depth + 1, title=title or None
-                )
-                added += 1
-        await self.db.update_source(
-            source_id, status="skipped",
-            error=f"rss feed: {added} entries sembradas"
-        )
-        logger.info("RSS feed seeded", url=url[:60],
-                    entries=len(entries), added=added)
-        return added
-
     async def _extract_pdf(self, url: str) -> tuple[Optional[str], Optional[str]]:
         """Download and extract PDF text"""
         import pdfplumber

src/seo/autofill.py

@@ -37,19 +37,14 @@ SITE_BY_LANG = {
 
 # Tag allow-list — the model may ONLY pick from these; invented tags are dropped
 # deterministically after the call (never trust the LLM to self-limit). EN is the
-# live vocabulary Jose confirmed; ES is the live core taxonomy on zonadeexclusion
+# live vocabulary Jose confirmed; ES is not yet constrained, so we don't enforce
-# (fetched 2026-07-03), mirroring EN. Deliberately excluded from ES: the one-post
+# it (model picks freely there until an ES vocab is pinned down).
-# long-tail tags the model invented before constraining, and "investigacion-2"
-# (a Ghost case-collision duplicate of "investigacion").
 ALLOWED_TAGS = {
     "en": [
         "uap", "declassified", "military-cases", "classic-cases",
         "investigation", "spain-cases", "latin-america",
     ],
-    "es": [
+    # "es": [...]  # TODO: pin the live ES tag vocabulary before constraining.
-        "uap", "desclasificados", "casos-militares", "casos-clasicos",
-        "investigacion", "casos-espana",
-    ],
 }
 
 # Language-aware default tag when nothing from the allow-list fits / the model
@@ -75,20 +70,33 @@ async def fetch_published_menu(lang: str) -> list[dict]:
     try:
         # Lazy import to avoid a heavy/circular import at module load.
         from src.generator.generator import GhostPublisher
+        import aiohttp as _aio
 
         pub = GhostPublisher(lang=lang)
         if not pub.is_configured():
             logger.warning("seo.menu: Ghost not configured for lang", lang=lang)
             return []
 
-        # _admin_get lleva los headers canónicos (auth, Accept-Version y el
+        token = pub._make_token()
-        # Accept-Encoding sin br — ver KNOWN-ISSUES.md) y loguea los non-200.
+        url = (
-        data = await pub._admin_get(
+            f"{pub.url}/ghost/api/admin/posts/"
-            "posts/?filter=status:published&fields=id,slug,title&limit=all",
+            "?filter=status:published&fields=id,slug,title&limit=all"
-            timeout=30,
         )
-        if data is None:
+        timeout = _aio.ClientTimeout(total=30)
-            return []
+        async with _aio.ClientSession(timeout=timeout) as sess:
+            async with sess.get(
+                url,
+                headers={
+                    "Authorization": f"Ghost {token}",
+                    "Accept-Version": "v5.0",
+                },
+            ) as resp:
+                if resp.status != 200:
+                    body = await resp.text()
+                    logger.warning("seo.menu: non-200 from Ghost",
+                                   status=resp.status, body=body[:200])
+                    return []
+                data = await resp.json()
 
         menu = [
             {"slug": p["slug"], "title": p.get("title", "")}
@@ -108,13 +116,9 @@ async def fetch_published_menu(lang: str) -> list[dict]:
 def _system_prompt(lang: str) -> str:
     allow = ", ".join(ALLOWED_TAGS.get(lang, []))
     out_lang = "SPANISH" if lang == "es" else "ENGLISH"
-    # The anti-legacy "never use investigacion" rule is EN-only: on the ES site
-    # "investigacion" IS the canonical tag (and the allow-list already excludes
-    # the "investigacion-2" duplicate).
-    legacy_clause = ", never use \"investigacion\"" if lang == "en" else ""
     tag_clause = (
         f"TAGS: choose 2-4 tags ONLY from this exact list (never invent a tag, "
-        f"never translate it{legacy_clause}): {allow}.\n"
+        f"never translate it, never use \"investigacion\"): {allow}.\n"
         if allow else
         "TAGS: 2-4 lowercase-hyphenated topical tags appropriate to the article.\n"
     )

tests/test_scraper.py

@@ -11,16 +11,6 @@ def test_detect_source_type():
     assert detect_source_type("https://example.com/article") == "web"
 
 
-def test_detect_source_type_rss():
-    assert detect_source_type("https://thedebrief.org/feed/") == "rss"
-    assert detect_source_type("https://example.com/rss") == "rss"
-    assert detect_source_type("https://example.com/feeds/all.atom.xml") == "rss"
-    assert detect_source_type("https://www.liberationtimes.com/?format=rss") == "rss"
-    # Falsos positivos del substring viejo: no son feeds.
-    assert detect_source_type("https://example.com/feedback") == "web"
-    assert detect_source_type("https://example.com/atomic-theory") == "web"
-
-
 def test_is_blacklisted():
     assert is_blacklisted("https://facebook.com/something") == True
     assert is_blacklisted("https://en.wikipedia.org/wiki/Test") == False
@@ -36,22 +26,3 @@ def test_simple_chunk():
     chunks = simple_chunk(text, chunk_size=100, overlap=20)
     assert len(chunks) > 1
     assert all(isinstance(c, str) for c in chunks)
-
-
-def test_unwrap_news_link():
-    from src.scraper.exhaustive import _unwrap_news_link
-    wrapped = ("http://www.bing.com/news/apiclick.aspx?ref=FexRss&aid=&tid=x"
-               "&url=https://example.com/article&c=1&mkt=es-es")
-    assert _unwrap_news_link(wrapped) == "https://example.com/article"
-    # Links normales pasan intactos
-    assert _unwrap_news_link("https://thedebrief.org/foo") == "https://thedebrief.org/foo"
-    # url= protocol-relative se normaliza a https
-    assert _unwrap_news_link(
-        "http://www.bing.com/news/apiclick.aspx?url=//example.com/article"
-    ) == "https://example.com/article"
-    # apiclick sin url= usable se descarta ("" → el caller lo salta), nunca se
-    # deja pasar el redirect de bing.com como fuente
-    assert _unwrap_news_link("http://www.bing.com/news/apiclick.aspx?ref=x") == ""
-    assert _unwrap_news_link(
-        "http://www.bing.com/news/apiclick.aspx?url=javascript:alert(1)"
-    ) == ""

tests/test_seo_autofill.py

@@ -1,39 +0,0 @@
-from src.seo.autofill import ALLOWED_TAGS, DEFAULT_TAG, _coerce, _system_prompt
-
-BASE = {
-    "meta_title": "t",
-    "meta_description": "d",
-    "custom_excerpt": "e",
-    "image_query": "q",
-    "image_context": "c",
-}
-
-
-def test_coerce_es_drops_invented_tags():
-    obj = dict(BASE, tags=["uap", "humanoides", "Desclasificados", "investigacion-2"])
-    out = _coerce(obj, "es")
-    assert out["tags"] == ["uap", "desclasificados"]
-
-
-def test_coerce_es_falls_back_to_default():
-    obj = dict(BASE, tags=["pentagono", "encuentros-cercanos"])
-    out = _coerce(obj, "es")
-    assert out["tags"] == [DEFAULT_TAG["es"]]
-
-
-def test_coerce_en_still_constrained():
-    obj = dict(BASE, tags=["uap", "investigacion"])
-    out = _coerce(obj, "en")
-    assert out["tags"] == ["uap"]
-
-
-def test_system_prompt_lists_allowed_tags_per_lang():
-    es = _system_prompt("es")
-    en = _system_prompt("en")
-    for tag in ALLOWED_TAGS["es"]:
-        assert tag in es
-    # La regla anti-legacy 'never use "investigacion"' es solo para EN: en ES
-    # "investigacion" es el tag canónico del allow-list.
-    assert 'never use "investigacion"' in en
-    assert 'never use "investigacion"' not in es
-    assert "ONLY from this exact list" in es