diff --git a/src/config.py b/src/config.py index dabee50..6424e98 100644 --- a/src/config.py +++ b/src/config.py @@ -27,6 +27,16 @@ class Settings(BaseSettings): searxng_url: str = Field( "http://searxng-svc.researchowl.svc.cluster.local:8080/search", ) + # Motores que el scraper pide a SearXNG (todos vienen configurados por los + # defaults de la instancia, use_default_settings: true). Verificados en vivo + # 2026-07-03: la lista ampliada duplica resultados y dominios Ășnicos (~2x) + # por ~+2s/query, y da redundancia cuando brave/DDG caen en rate limit o + # CAPTCHA. Excluidos: qwant (access denied), yahoo (roto), wikidata (0 + # resultados en queries de texto), wayback machine (duplica contenido vivo). + searxng_engines: str = Field( + "duckduckgo,google,bing,brave,startpage,mojeek,presearch," + "google news,bing news,internet archive scholar" + ) request_timeout: int = Field(30) request_delay: float = Field(1.0) # seconds between requests min_content_length: int = Field(200) # chars diff --git a/src/scraper/exhaustive.py b/src/scraper/exhaustive.py index 3cc186b..2bbc06c 100644 --- a/src/scraper/exhaustive.py +++ b/src/scraper/exhaustive.py @@ -229,7 +229,7 @@ class ExhaustiveScraper: params = { "q": query, "format": "json", - "engines": "duckduckgo,google,bing,brave", + "engines": settings.searxng_engines, "language": "all", } headers = {