diff --git a/src/config.py b/src/config.py index 6424e98..4a802b6 100644 --- a/src/config.py +++ b/src/config.py @@ -45,6 +45,8 @@ class Settings(BaseSettings): # bloqueada por Reddit (403) y YouTube (transcripts vacíos), eran peso muerto. enable_youtube: bool = Field(False) enable_reddit: bool = Field(False) + # Bing News RSS como semilla de actualidad — off hasta activarlo a propósito + enable_news_seed: bool = Field(False) # Processing chunk_size: int = Field(800) # tokens per chunk diff --git a/src/scraper/exhaustive.py b/src/scraper/exhaustive.py index a8c23ff..eb95ba8 100644 --- a/src/scraper/exhaustive.py +++ b/src/scraper/exhaustive.py @@ -7,7 +7,7 @@ import random import re import time from typing import Optional -from urllib.parse import urljoin, urlparse, quote, quote_plus +from urllib.parse import urljoin, urlparse, parse_qs, quote, quote_plus import aiohttp import feedparser @@ -104,6 +104,22 @@ def is_blacklisted(url: str) -> bool: return True +def _unwrap_news_link(link: str) -> str: + """Bing News envuelve los links de entry en apiclick.aspx; la URL real del + publisher viene en texto plano en el parámetro ?url=. Cualquier otro link + pasa sin tocar.""" + try: + parsed = urlparse(link) + if (parsed.netloc.lower().endswith("bing.com") + and parsed.path == "/news/apiclick.aspx"): + real = parse_qs(parsed.query).get("url", [""])[0] + if real.startswith("http"): + return real + except Exception: + pass + return link + + def normalize_url(url: str) -> str: # Strip only the fragment. NO borrar el query string: rompía URLs de # YouTube (watch?v=...) y artículos que enrutan por query (?id=, ?p=). @@ -177,6 +193,8 @@ class ExhaustiveScraper: tasks.append(self._seed_reddit()) if settings.enable_youtube: tasks.append(self._seed_youtube()) + if settings.enable_news_seed: + tasks.append(self._seed_news()) await asyncio.gather(*tasks, return_exceptions=True) async def _generate_ddg_queries(self) -> list[str]: @@ -379,6 +397,25 @@ class ExhaustiveScraper: except Exception as e: logger.warning("Archive.org seed failed", error=str(e)) + async def _seed_news(self): + """Siembra Bing News RSS para cobertura de actualidad. Solo registra + el feed como fuente tipo rss: _extract_rss (dispatch del pipeline) lo + parsea y siembra sus entries — sin duplicar lógica de parseo aquí. + Bing y no Google News: los links de Google (news.google.com/rss/articles/ + CBMi…) van a un muro de consent + redirect cifrado solo resoluble vía su + API interna batchexecute (verificado 2026-07-04); los de Bing llevan la + URL real del publisher en ?url= (ver _unwrap_news_link). + Best-effort: nunca rompe el seed.""" + try: + url = f"https://www.bing.com/news/search?q={quote_plus(self.topic)}&format=rss" + await self.db.add_source( + self.session_id, url, "rss", depth=0, + title=f"Bing News: {self.topic}" + ) + logger.info("News seed added", topic=self.topic) + except Exception as e: + logger.warning("News seed failed", error=str(e)) + async def _seed_reddit(self): """Search Reddit — sequential to avoid rate limiting""" try: @@ -749,7 +786,7 @@ class ExhaustiveScraper: for e in entries: if added >= 20: # cap, como los 30 links/página de web break - link = normalize_url(getattr(e, "link", "") or "") + link = normalize_url(_unwrap_news_link(getattr(e, "link", "") or "")) title = getattr(e, "title", "") or "" if (not link.startswith("http") or is_blacklisted(link) or not self._url_is_relevant(link, title)): diff --git a/tests/test_scraper.py b/tests/test_scraper.py index edbaa1a..dcdfeda 100644 --- a/tests/test_scraper.py +++ b/tests/test_scraper.py @@ -36,3 +36,15 @@ def test_simple_chunk(): chunks = simple_chunk(text, chunk_size=100, overlap=20) assert len(chunks) > 1 assert all(isinstance(c, str) for c in chunks) + + +def test_unwrap_news_link(): + from src.scraper.exhaustive import _unwrap_news_link + wrapped = ("http://www.bing.com/news/apiclick.aspx?ref=FexRss&aid=&tid=x" + "&url=https://example.com/article&c=1&mkt=es-es") + assert _unwrap_news_link(wrapped) == "https://example.com/article" + # Links normales pasan intactos + assert _unwrap_news_link("https://thedebrief.org/foo") == "https://thedebrief.org/foo" + # apiclick sin url= no revienta + assert _unwrap_news_link("http://www.bing.com/news/apiclick.aspx?ref=x") \ + == "http://www.bing.com/news/apiclick.aspx?ref=x"