fix: add /process command, log quality filtering, improve Reddit headers

- bot.py: add cmd_process handler to manually trigger chunk processing
  on the last session; register CommandHandler("process")
- processor.py: log exceptions from asyncio.gather instead of silently
  dropping them; add per-chunk quality score debug logging; warn when
  all chunks filtered by quality threshold with actionable hint;
  raise fallback score to 0.6 so Ollama failures don't filter chunks
- exhaustive.py: replace bot User-Agent with full browser UA + headers
  for REDDIT_HEADERS; downgrade Reddit 403 from warning to info since
  server IPs are routinely blocked; use content_type=None on json()
  to avoid aiohttp content-type mismatch errors

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

src/scraper/exhaustive.py

@@ -23,9 +23,21 @@ from src.db.database import ResearchDB
 logger = structlog.get_logger()
 
 HEADERS = {
-    "User-Agent": "Mozilla/5.0 (compatible; ResearchOwl/1.0; +https://chemavx.xyz)",
-    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
     "Accept-Language": "en-US,en;q=0.9,es;q=0.8",
+    "Accept-Encoding": "gzip, deflate, br",
+    "DNT": "1",
+}
+
+# Reddit requires its own headers — generic bots get 403
+REDDIT_HEADERS = {
+    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Accept": "application/json, text/javascript, */*; q=0.01",
+    "Accept-Language": "en-US,en;q=0.9",
+    "Accept-Encoding": "gzip, deflate, br",
+    "Referer": "https://www.reddit.com/",
+    "X-Requested-With": "XMLHttpRequest",
 }
 
 # Domains to skip — not useful for research
@@ -177,10 +189,10 @@ class ExhaustiveScraper:
         """Search Reddit — sequential to avoid rate limiting"""
         try:
             http = await self._get_http()
-            url = f"https://www.reddit.com/search.json?q={quote_plus(self.topic)}&sort=top&limit=15"
-            async with http.get(url, headers={**HEADERS, "User-Agent": "ResearchOwl/1.0"}) as resp:
+            url = f"https://www.reddit.com/search.json?q={quote_plus(self.topic)}&sort=top&limit=15&type=link"
+            async with http.get(url, headers=REDDIT_HEADERS) as resp:
                 if resp.status == 200:
-                    data = await resp.json()
+                    data = await resp.json(content_type=None)
                     posts = data.get("data", {}).get("children", [])
                     for post in posts:
                         post_data = post.get("data", {})
@@ -192,6 +204,8 @@ class ExhaustiveScraper:
                                 title=post_data.get("title")
                             )
                     logger.info("Reddit seed", found=len(posts), status=resp.status)
+                elif resp.status == 403:
+                    logger.info("Reddit seed blocked (403) — server IP likely blocked by Reddit; skipping")
                 else:
                     logger.warning("Reddit seed non-200", status=resp.status)
         except Exception as e:
@@ -446,14 +460,14 @@ class ExhaustiveScraper:
         json_url = url.rstrip("/") + ".json?limit=100&sort=top"
         http = await self._get_http()
         try:
-            async with http.get(
-                json_url,
-                headers={**HEADERS, "User-Agent": "ResearchOwl/1.0"}
-            ) as resp:
+            async with http.get(json_url, headers=REDDIT_HEADERS) as resp:
+                if resp.status == 403:
+                    logger.info("Reddit post blocked (403) — skipping", url=url[:60])
+                    return None, None
                 if resp.status != 200:
                     logger.debug("Reddit non-200", status=resp.status, url=url[:60])
                     return None, None
-                data = await resp.json()
+                data = await resp.json(content_type=None)
 
             post = data[0]["data"]["children"][0]["data"]
             title = post.get("title", "")