diff --git a/src/scraper/exhaustive.py b/src/scraper/exhaustive.py index 8cc6179..8c124e2 100644 --- a/src/scraper/exhaustive.py +++ b/src/scraper/exhaustive.py @@ -113,16 +113,19 @@ def is_blacklisted(url: str) -> bool: def _unwrap_news_link(link: str) -> str: """Bing News envuelve los links de entry en apiclick.aspx; la URL real del publisher viene en texto plano en el parámetro ?url=. Cualquier otro link - pasa sin tocar.""" + pasa sin tocar. Si es un apiclick pero no trae URL usable, devuelve "" — + el caller lo descarta por startswith("http") — en vez de dejar pasar el + redirect de bing.com, que se rasparía como página de consent/basura.""" try: parsed = urlparse(link) if (parsed.netloc.lower().endswith("bing.com") and parsed.path == "/news/apiclick.aspx"): - real = parse_qs(parsed.query).get("url", [""])[0] - if real.startswith("http"): - return real + real = parse_qs(parsed.query).get("url", [""])[0].strip() + if real.startswith("//"): # protocol-relative + real = "https:" + real + return real if real.startswith("http") else "" except Exception: - pass + return "" return link diff --git a/tests/test_scraper.py b/tests/test_scraper.py index dcdfeda..1f72019 100644 --- a/tests/test_scraper.py +++ b/tests/test_scraper.py @@ -45,6 +45,13 @@ def test_unwrap_news_link(): assert _unwrap_news_link(wrapped) == "https://example.com/article" # Links normales pasan intactos assert _unwrap_news_link("https://thedebrief.org/foo") == "https://thedebrief.org/foo" - # apiclick sin url= no revienta - assert _unwrap_news_link("http://www.bing.com/news/apiclick.aspx?ref=x") \ - == "http://www.bing.com/news/apiclick.aspx?ref=x" + # url= protocol-relative se normaliza a https + assert _unwrap_news_link( + "http://www.bing.com/news/apiclick.aspx?url=//example.com/article" + ) == "https://example.com/article" + # apiclick sin url= usable se descarta ("" → el caller lo salta), nunca se + # deja pasar el redirect de bing.com como fuente + assert _unwrap_news_link("http://www.bing.com/news/apiclick.aspx?ref=x") == "" + assert _unwrap_news_link( + "http://www.bing.com/news/apiclick.aspx?url=javascript:alert(1)" + ) == ""