ChemaVXandClaude Fable 5
fbe8ae1885
Un ?url= protocol-relative (//publisher.com/...) caía por el startswith('http')
y devolvía el link apiclick.aspx crudo, que pasaba blacklist y relevancia y
acababa raspado como página de redirect/consent de Bing. Ahora se antepone
https: a los protocol-relative y, si no se puede extraer URL válida de un
apiclick, se devuelve '' para que el caller lo salte.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
58 lines
2.6 KiB
Python
58 lines
2.6 KiB
Python
import pytest
|
|
from src.scraper.exhaustive import detect_source_type, is_blacklisted, normalize_url
|
|
from src.processor.processor import simple_chunk
|
|
|
|
|
|
def test_detect_source_type():
|
|
assert detect_source_type("https://youtube.com/watch?v=dQw4w9WgXcY") == "youtube"
|
|
assert detect_source_type("https://reddit.com/r/test/comments/abc") == "reddit"
|
|
assert detect_source_type("https://en.wikipedia.org/wiki/Roswell") == "wikipedia"
|
|
assert detect_source_type("https://example.com/doc.pdf") == "pdf"
|
|
assert detect_source_type("https://example.com/article") == "web"
|
|
|
|
|
|
def test_detect_source_type_rss():
|
|
assert detect_source_type("https://thedebrief.org/feed/") == "rss"
|
|
assert detect_source_type("https://example.com/rss") == "rss"
|
|
assert detect_source_type("https://example.com/feeds/all.atom.xml") == "rss"
|
|
assert detect_source_type("https://www.liberationtimes.com/?format=rss") == "rss"
|
|
# Falsos positivos del substring viejo: no son feeds.
|
|
assert detect_source_type("https://example.com/feedback") == "web"
|
|
assert detect_source_type("https://example.com/atomic-theory") == "web"
|
|
|
|
|
|
def test_is_blacklisted():
|
|
assert is_blacklisted("https://facebook.com/something") == True
|
|
assert is_blacklisted("https://en.wikipedia.org/wiki/Test") == False
|
|
|
|
|
|
def test_normalize_url():
|
|
assert normalize_url("https://example.com/page#section") == "https://example.com/page"
|
|
assert normalize_url("https://example.com/page/") == "https://example.com/page"
|
|
|
|
|
|
def test_simple_chunk():
|
|
text = "\n\n".join([f"Paragraph {i} with some content here." for i in range(50)])
|
|
chunks = simple_chunk(text, chunk_size=100, overlap=20)
|
|
assert len(chunks) > 1
|
|
assert all(isinstance(c, str) for c in chunks)
|
|
|
|
|
|
def test_unwrap_news_link():
|
|
from src.scraper.exhaustive import _unwrap_news_link
|
|
wrapped = ("http://www.bing.com/news/apiclick.aspx?ref=FexRss&aid=&tid=x"
|
|
"&url=https://example.com/article&c=1&mkt=es-es")
|
|
assert _unwrap_news_link(wrapped) == "https://example.com/article"
|
|
# Links normales pasan intactos
|
|
assert _unwrap_news_link("https://thedebrief.org/foo") == "https://thedebrief.org/foo"
|
|
# url= protocol-relative se normaliza a https
|
|
assert _unwrap_news_link(
|
|
"http://www.bing.com/news/apiclick.aspx?url=//example.com/article"
|
|
) == "https://example.com/article"
|
|
# apiclick sin url= usable se descarta ("" → el caller lo salta), nunca se
|
|
# deja pasar el redirect de bing.com como fuente
|
|
assert _unwrap_news_link("http://www.bing.com/news/apiclick.aspx?ref=x") == ""
|
|
assert _unwrap_news_link(
|
|
"http://www.bing.com/news/apiclick.aspx?url=javascript:alert(1)"
|
|
) == ""
|