diff --git a/src/scraper/exhaustive.py b/src/scraper/exhaustive.py index 74ff47d..aeb007f 100644 --- a/src/scraper/exhaustive.py +++ b/src/scraper/exhaustive.py @@ -181,6 +181,7 @@ class ExhaustiveScraper: try: import anthropic + logger.info("Generating DDG queries with Claude", topic=self.topic) client = anthropic.AsyncAnthropic(api_key=settings.anthropic_api_key) prompt = ( f'Generate exactly 8 DuckDuckGo search queries to research: "{self.topic}"\n\n' @@ -207,7 +208,8 @@ class ExhaustiveScraper: logger.info("DDG queries generated by Claude", queries=queries) return queries except Exception as e: - logger.warning("Claude query generation failed, using fallback", error=str(e)) + logger.warning("Claude query generation failed, using fallback", + error=str(e), error_type=type(e).__name__) return fallback async def _seed_duckduckgo(self): @@ -232,7 +234,7 @@ class ExhaustiveScraper: logger.info("DDG query ok", query=query, results=len(results)) except Exception as e: logger.warning("DDG query failed", query=query, error=str(e)) - await asyncio.sleep(settings.request_delay * 2) + await asyncio.sleep(random.uniform(3, 8)) async def _seed_wikipedia(self): """Search Wikipedia API for correct article URLs.