diff --git a/src/scraper/exhaustive.py b/src/scraper/exhaustive.py index 2bbc06c..48dd803 100644 --- a/src/scraper/exhaustive.py +++ b/src/scraper/exhaustive.py @@ -7,7 +7,7 @@ import random import re import time from typing import Optional -from urllib.parse import urljoin, urlparse, quote_plus +from urllib.parse import urljoin, urlparse, quote, quote_plus import aiohttp import feedparser @@ -294,27 +294,34 @@ class ExhaustiveScraper: await asyncio.sleep(random.uniform(1, 3)) async def _seed_wikipedia(self): - """Search Wikipedia API for correct article URLs. - Tries English first, falls back to Spanish if no results found.""" + """Siembra Wikipedia en AMBOS idiomas siempre, con búsqueda full-text + (list=search). El opensearch anterior era prefix-de-título y devolvía 0 + para topics verbosos ("incidente ovni de Manises 1979") aunque el + artículo exista; y el break tras en dejaba es solo como fallback. + Cada idioma va aislado: un fallo no afecta al otro.""" http = await self._get_http() - added = 0 for lang in ("en", "es"): try: api_url = ( - f"https://{lang}.wikipedia.org/w/api.php?action=opensearch" - f"&search={quote_plus(self.topic)}&limit=10&format=json" + f"https://{lang}.wikipedia.org/w/api.php?action=query" + f"&list=search&srsearch={quote_plus(self.topic)}" + f"&srlimit=8&format=json" ) async with http.get(api_url) as resp: data = await resp.json() - urls = data[3] if len(data) > 3 else [] - for url in urls: - if url: - await self.db.add_source(self.session_id, url, "wikipedia", depth=0) - added += 1 - logger.info("Wikipedia seed", lang=lang, found=len(urls)) - if added > 0: - break # English results found — no need to try Spanish + hits = data.get("query", {}).get("search", []) + added = 0 + for h in hits: + title = h.get("title") + if not title: + continue + url = (f"https://{lang}.wikipedia.org/wiki/" + f"{quote(title.replace(' ', '_'))}") + await self.db.add_source(self.session_id, url, "wikipedia", + depth=0, title=title) + added += 1 + logger.info("Wikipedia seed", lang=lang, found=added) except Exception as e: logger.warning("Wikipedia API seed failed", lang=lang, error=str(e))