From fbe8ae188583c7cd464c1c8da1476d44aba31a21 Mon Sep 17 00:00:00 2001
From: ChemaVX <jmivanez@gmail.com>
Date: Sun, 5 Jul 2026 15:58:40 +0000
Subject: [PATCH] fix(scraper): _unwrap_news_link normaliza //url y descarta
 apiclick sin URL usable
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Un ?url= protocol-relative (//publisher.com/...) caía por el startswith('http')
y devolvía el link apiclick.aspx crudo, que pasaba blacklist y relevancia y
acababa raspado como página de redirect/consent de Bing. Ahora se antepone
https: a los protocol-relative y, si no se puede extraer URL válida de un
apiclick, se devuelve '' para que el caller lo salte.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---
 src/scraper/exhaustive.py | 13 ++++++++-----
 tests/test_scraper.py     | 13 ++++++++++---
 2 files changed, 18 insertions(+), 8 deletions(-)

diff --git a/src/scraper/exhaustive.py b/src/scraper/exhaustive.py
index 8cc6179..8c124e2 100644
--- a/src/scraper/exhaustive.py
+++ b/src/scraper/exhaustive.py
@@ -113,16 +113,19 @@ def is_blacklisted(url: str) -> bool:
 def _unwrap_news_link(link: str) -> str:
     """Bing News envuelve los links de entry en apiclick.aspx; la URL real del
     publisher viene en texto plano en el parámetro ?url=. Cualquier otro link
-    pasa sin tocar."""
+    pasa sin tocar. Si es un apiclick pero no trae URL usable, devuelve "" —
+    el caller lo descarta por startswith("http") — en vez de dejar pasar el
+    redirect de bing.com, que se rasparía como página de consent/basura."""
     try:
         parsed = urlparse(link)
         if (parsed.netloc.lower().endswith("bing.com")
                 and parsed.path == "/news/apiclick.aspx"):
-            real = parse_qs(parsed.query).get("url", [""])[0]
-            if real.startswith("http"):
-                return real
+            real = parse_qs(parsed.query).get("url", [""])[0].strip()
+            if real.startswith("//"):  # protocol-relative
+                real = "https:" + real
+            return real if real.startswith("http") else ""
     except Exception:
-        pass
+        return ""
     return link
 
 
diff --git a/tests/test_scraper.py b/tests/test_scraper.py
index dcdfeda..1f72019 100644
--- a/tests/test_scraper.py
+++ b/tests/test_scraper.py
@@ -45,6 +45,13 @@ def test_unwrap_news_link():
     assert _unwrap_news_link(wrapped) == "https://example.com/article"
     # Links normales pasan intactos
     assert _unwrap_news_link("https://thedebrief.org/foo") == "https://thedebrief.org/foo"
-    # apiclick sin url= no revienta
-    assert _unwrap_news_link("http://www.bing.com/news/apiclick.aspx?ref=x") \
-        == "http://www.bing.com/news/apiclick.aspx?ref=x"
+    # url= protocol-relative se normaliza a https
+    assert _unwrap_news_link(
+        "http://www.bing.com/news/apiclick.aspx?url=//example.com/article"
+    ) == "https://example.com/article"
+    # apiclick sin url= usable se descarta ("" → el caller lo salta), nunca se
+    # deja pasar el redirect de bing.com como fuente
+    assert _unwrap_news_link("http://www.bing.com/news/apiclick.aspx?ref=x") == ""
+    assert _unwrap_news_link(
+        "http://www.bing.com/news/apiclick.aspx?url=javascript:alert(1)"
+    ) == ""