diff --git a/src/scraper/exhaustive.py b/src/scraper/exhaustive.py index 4cc2044..28228bf 100644 --- a/src/scraper/exhaustive.py +++ b/src/scraper/exhaustive.py @@ -171,6 +171,7 @@ class ExhaustiveScraper: tasks = [ self._seed_search(), self._seed_wikipedia(), + self._seed_archive(), ] if settings.enable_reddit: tasks.append(self._seed_reddit()) @@ -329,6 +330,42 @@ class ExhaustiveScraper: except Exception as e: logger.warning("Wikipedia API seed failed", lang=lang, error=str(e)) + async def _seed_archive(self): + """Siembra Internet Archive (advancedsearch, mediatype:texts): documentos + FOIA, informes y libros históricos. Se siembra la página /details/; + sus PDFs los descubre la recursión (la propia página los enlaza) y los + extrae _extract_pdf. Sin fallback de query: el AND estricto de archive.org + con el topic completo prima precisión — relajarlo mete ruido (verificado: + 'Manises' a secas devuelve loza valenciana). Best-effort: nunca rompe el seed.""" + try: + http = await self._get_http() + q = quote_plus(f"({self.topic}) AND mediatype:(texts)") + url = ( + f"https://archive.org/advancedsearch.php?q={q}" + "&fl[]=identifier&fl[]=title&rows=12&page=1&output=json" + ) + async with http.get(url) as resp: + if resp.status != 200: + logger.warning("Archive.org seed non-200", status=resp.status) + return + data = await resp.json(content_type=None) + docs = data.get("response", {}).get("docs", []) + added = 0 + for d in docs: + ident = d.get("identifier") + if not ident: + continue + item_url = f"https://archive.org/details/{ident}" + title = d.get("title") + await self.db.add_source( + self.session_id, item_url, "web", depth=0, + title=title if isinstance(title, str) else None + ) + added += 1 + logger.info("Archive.org seed", found=len(docs), added=added) + except Exception as e: + logger.warning("Archive.org seed failed", error=str(e)) + async def _seed_reddit(self): """Search Reddit — sequential to avoid rate limiting""" try: