feat: retry+backoff en scraper, ProgressReporter en bot

src/scraper/exhaustive.py

@@ -3,6 +3,7 @@ ResearchOwl Exhaustive Scraper
 Core engine: discovers, expands, and evaluates sources recursively
 """
 import asyncio
+import random
 import re
 import time
 from typing import Optional
@@ -54,6 +55,22 @@ REDDIT_RE = re.compile(r"reddit\.com/(r/\w+/comments/\w+)")
 WIKIPEDIA_RE = re.compile(r"wikipedia\.org/wiki/(.+)")
 
 
+async def fetch_with_retry(fetch_fn, source_name: str, max_retries: int = 3):
+    last_exc = None
+    for attempt in range(max_retries):
+        try:
+            return await fetch_fn()
+        except Exception as e:
+            last_exc = e
+            if attempt < max_retries - 1:
+                wait = 2 ** attempt + random.random()
+                logger.debug("fetch_with_retry backoff", source=source_name[:60],
+                             attempt=attempt + 1, wait=round(wait, 1), error=str(e))
+                await asyncio.sleep(wait)
+    logger.warning("fetch_with_retry exhausted", source=source_name[:60], error=str(last_exc))
+    raise last_exc
+
+
 def detect_source_type(url: str) -> str:
     if YOUTUBE_RE.search(url):
         return "youtube"
@@ -290,12 +307,7 @@ class ExhaustiveScraper:
             )
 
             if self.progress_callback:
-                await self.progress_callback(
-                    iteration=self.iteration,
-                    total=self.total_sources,
-                    new_this_round=new_sources,
-                    stats=stats
-                )
+                await self.progress_callback(self.iteration, self.total_sources)
 
             # Saturation check: if we found very few new URLs, we're done
             if new_sources < 3 and self.iteration > 2:
@@ -317,9 +329,13 @@ class ExhaustiveScraper:
 
             try:
                 if source_type == "youtube":
-                    content, title = await self._extract_youtube(url)
+                    content, title = await fetch_with_retry(
+                        lambda: self._extract_youtube(url), url
+                    )
                 elif source_type == "wikipedia":
-                    content, title, new_urls = await self._extract_wikipedia(url)
+                    content, title, new_urls = await fetch_with_retry(
+                        lambda: self._extract_wikipedia(url), url
+                    )
                     added = 0
                     for new_url in (new_urls or []):
                         if self._url_is_relevant(new_url):
@@ -331,13 +347,18 @@ class ExhaustiveScraper:
                     await self._mark_scraped(source_id, content, title, url)
                     return added
                 elif source_type == "reddit":
-                    content, title = await self._extract_reddit(url)
-                    # Small delay between Reddit requests to avoid rate limiting
+                    content, title = await fetch_with_retry(
+                        lambda: self._extract_reddit(url), url
+                    )
                     await asyncio.sleep(settings.request_delay)
                 elif source_type == "pdf":
-                    content, title = await self._extract_pdf(url)
+                    content, title = await fetch_with_retry(
+                        lambda: self._extract_pdf(url), url
+                    )
                 else:
-                    content, title, new_urls = await self._extract_web(url, source["depth"])
+                    content, title, new_urls = await fetch_with_retry(
+                        lambda: self._extract_web(url, source["depth"]), url
+                    )
                     added = 0
                     for new_url in (new_urls or []):
                         if self._url_is_relevant(new_url):