diff --git a/src/bot/bot.py b/src/bot/bot.py index f7a5a96..20de42d 100644 --- a/src/bot/bot.py +++ b/src/bot/bot.py @@ -22,6 +22,7 @@ from src.db.database import get_db, close_db, ResearchDB, ResearchStatus, Output from src.scraper.exhaustive import ExhaustiveScraper from src.processor.processor import OllamaClient, ContentProcessor from src.generator.generator import OutputGenerator +from src.news.monitor import poll_feeds, item_from_row, format_digest logger = structlog.get_logger() @@ -503,6 +504,33 @@ async def cmd_outputs(update: Update, ctx: ContextTypes.DEFAULT_TYPE): await db_conn.close() +async def cmd_news(update: Update, ctx: ContextTypes.DEFAULT_TYPE): + """Monitor RSS manual (sin scheduler — F2 lo automatiza). Refresca news_seen + (siembra en cold-start) y muestra las novedades matcheadas de las últimas 24h.""" + if not is_authorized(update.effective_user.id): + return + + db_conn = await get_db() + db = ResearchDB(db_conn) + try: + await update.message.reply_text("📡 Buscando novedades UAP/OVNI…") + try: + await poll_feeds(db, settings) # refresca tabla (siembra en cold-start) + except Exception: + logger.exception("news poll failed") # best-effort: nunca rompe el comando + + recent = await db.get_recent_news(24) + if not recent: + await update.message.reply_text("Sin novedades UAP/OVNI en las últimas 24h.") + return + + items = [item_from_row(r) for r in recent] + for chunk in format_digest(items): + await update.message.reply_text(chunk, disable_web_page_preview=False) + finally: + await db_conn.close() + + async def cmd_costs(update: Update, ctx: ContextTypes.DEFAULT_TYPE): if not is_authorized(update.effective_user.id): return @@ -1292,6 +1320,7 @@ def create_bot() -> Application: app.add_handler(CommandHandler("generate", cmd_generate)) app.add_handler(CommandHandler("sources", cmd_sources)) app.add_handler(CommandHandler("outputs", cmd_outputs)) + app.add_handler(CommandHandler("news", cmd_news)) app.add_handler(CommandHandler("export", cmd_export)) app.add_handler(CommandHandler("costs", cmd_costs)) app.add_handler(CommandHandler("watch", cmd_watch)) diff --git a/src/config.py b/src/config.py index 75b3c8b..4ff517a 100644 --- a/src/config.py +++ b/src/config.py @@ -65,8 +65,8 @@ class Settings(BaseSettings): NEWS_CHAT_ID: Optional[int] = Field(None, env="NEWS_CHAT_ID") NEWS_KEYWORDS: str = Field( "ovni,ovnis,uap,ufo,desclasificado,desclasificacion,declassified," - "avistamiento,sighting,extraterrestre,non-human,nhi,grusch,pentagono," - "pentagon,aaro,pursue,fenomeno aereo,whistleblower", + "avistamiento,sighting,extraterrestre,non-human,nhi,grusch," + "aaro,pursue,fenomeno aereo,whistleblower", env="NEWS_KEYWORDS", ) NEWS_MAX_ITEMS: int = Field(15, env="NEWS_MAX_ITEMS") diff --git a/src/news/__init__.py b/src/news/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/news/monitor.py b/src/news/monitor.py new file mode 100644 index 0000000..8eaaa37 --- /dev/null +++ b/src/news/monitor.py @@ -0,0 +1,173 @@ +"""Monitor RSS de noticias UAP/OVNI. + +Capa independiente: NUNCA importa de src.bot (el bot importa de aquí). No toca el +scheduler — F2. Best-effort por diseño: un feed caído se loguea y se salta; nunca +tumba el run. El estado vive en la tabla news_seen vía los métodos de ResearchDB, +que se inyectan (no se importan) para mantener el desacople. +""" +import asyncio +import re +import unicodedata +from dataclasses import dataclass, field + +import feedparser +import structlog + +log = structlog.get_logger() + +# Reddit devuelve 403 con el UA por defecto de feedparser/python; uno explícito +# (y honesto) pasa. El resto de feeds lo aceptan sin problema. +UA = "ExclusionZone-NewsBot/1.0 (+https://theexclusionzone.com)" + +FEEDS: list[dict] = [ + {"name": "Gran Misterio", "url": "https://granmisterio.org/feed/", "lang": "es"}, + {"name": "Espacio Misterio", "url": "https://www.espaciomisterio.com/feed/", "lang": "es"}, + {"name": "Josep Guijarro (YT)", "url": "https://www.youtube.com/feeds/videos.xml?channel_id=UCIlGy26zi-BbX6zDYAMQ9Qg", "lang": "es"}, + {"name": "r/UFOs", "url": "https://www.reddit.com/r/UFOs/new/.rss", "lang": "en"}, + {"name": "r/UFOB", "url": "https://www.reddit.com/r/UFOB/new/.rss", "lang": "en"}, + {"name": "Liberation Times", "url": "https://www.liberationtimes.com/?format=rss", "lang": "en"}, + {"name": "The Debrief", "url": "https://thedebrief.org/feed/", "lang": "en"}, +] + +# Tokens cortos: word-boundary para evitar falsos positivos dentro de palabras +# (p.ej. "aaro" en "aaron", "ufo" en "tufo"). Frases multi-palabra y términos +# largos van por substring. Se decide dinámicamente: sin espacios y len <= 4. +_SHORT_TOKEN_MAXLEN = 4 + +# Límite de Telegram 4096; troceamos por debajo con margen. +_TG_LIMIT = 4000 + + +@dataclass +class NewsItem: + source: str + guid: str + title: str + link: str + published: str + summary: str + matched: list[str] = field(default_factory=list) + + +def _normalize(text: str) -> str: + """minúsculas + NFKD sin marcas combinantes (quita tildes/diacríticos).""" + if not text: + return "" + decomposed = unicodedata.normalize("NFKD", text.lower()) + return "".join(c for c in decomposed if not unicodedata.combining(c)) + + +def _is_short_token(kw: str) -> bool: + return " " not in kw and len(kw) <= _SHORT_TOKEN_MAXLEN + + +def _match(title: str, summary: str, keywords: list[str]) -> list[str]: + """Devuelve las keywords (en su forma original) que matchean en title+summary, + sobre texto normalizado. Word-boundary para tokens cortos, substring para el + resto. Sin duplicados, preservando el orden de `keywords`.""" + text = _normalize(f"{title} {summary}") + if not text: + return [] + matched: list[str] = [] + for kw in keywords: + nkw = _normalize(kw) + if not nkw: + continue + if _is_short_token(nkw): + if re.search(rf"\b{re.escape(nkw)}\b", text): + matched.append(kw) + else: + if nkw in text: + matched.append(kw) + return matched + + +async def poll_feeds(db, settings) -> list[NewsItem]: + """Recorre los feeds, registra items matcheados en news_seen y devuelve los + NUEVOS (no vistos) de fuentes ya sembradas. Cold-start por fuente: el primer + poll de cada feed siembra en silencio (notified=1) y no devuelve nada. + + Best-effort: cada feed va en su propio try/except — uno caído nunca tumba el + run. feedparser.parse es bloqueante (I/O de red), así que se ejecuta en un + hilo para no congelar el event loop del bot durante el poll. + """ + keywords = [k.strip() for k in settings.NEWS_KEYWORDS.split(",") if k.strip()] + new_items: list[NewsItem] = [] + + for feed in FEEDS: + try: + parsed = await asyncio.to_thread(feedparser.parse, feed["url"], agent=UA) + if parsed.bozo and not parsed.entries: + log.warning("feed bozo sin entries: %s (%s)", + feed["name"], getattr(parsed, "bozo_exception", "")) + continue + + seeded = await db.source_seeded(feed["name"]) # cold-start por fuente + for e in parsed.entries: + guid = getattr(e, "id", None) or getattr(e, "link", "") + if not guid: + continue + matched = _match(e.get("title", ""), e.get("summary", ""), keywords) + if not matched: + continue + notified = 1 if not seeded else 0 # 1er run: sembrar sin notificar + is_new = await db.record_news_item( + feed["name"], guid, e.get("title", ""), e.get("link", ""), + e.get("published", ""), ",".join(matched), notified, + ) + if is_new and seeded: + new_items.append(NewsItem( + feed["name"], guid, e.get("title", ""), e.get("link", ""), + e.get("published", ""), e.get("summary", ""), matched, + )) + except Exception as ex: # aislado: un feed no tumba el run + log.exception("fallo feed %s: %s", feed["name"], ex) + + return new_items + + +def item_from_row(row: dict) -> NewsItem: + """Adapta una fila de get_recent_news (dict de news_seen) a NewsItem para que + format_digest sea agnóstico de la fuente (poll vs DB).""" + matched = row.get("matched_keywords") or "" + return NewsItem( + source=row.get("source", ""), + guid=row.get("guid", ""), + title=row.get("title", "") or "", + link=row.get("link", "") or "", + published=row.get("published_at", "") or "", + summary="", + matched=[m for m in matched.split(",") if m], + ) + + +def format_digest(items: list[NewsItem], header: str = "🛸 Novedades UAP/OVNI") -> list[str]: + """Agrupa por fuente y formatea cada item como '• \\n <link>'. Trocea + en mensajes < _TG_LIMIT chars (devuelve varios si hace falta). Texto plano + (no Markdown): el handler envía sin parse_mode. Lista vacía -> [].""" + if not items: + return [] + + # Agrupar por fuente preservando el orden de primera aparición. + groups: dict[str, list[NewsItem]] = {} + for it in items: + groups.setdefault(it.source, []).append(it) + + blocks: list[str] = [] # un bloque por fuente, indivisible salvo que él solo exceda + for source, its in groups.items(): + lines = [f"── {source} ──"] + for it in its: + title = (it.title or "(sin título)").strip() + lines.append(f"• {title}\n {it.link}".rstrip()) + blocks.append("\n".join(lines)) + + chunks: list[str] = [] + cur = header + for block in blocks: + if len(cur) + len(block) + 2 > _TG_LIMIT and cur.strip() != header: + chunks.append(cur) + cur = f"{header} (cont.)" + cur += "\n\n" + block + if cur.strip(): + chunks.append(cur) + return chunks