chemavx/researchowl
1
0
Fork 0
Code Issues Pull Requests 4 Actions Packages Projects Releases Wiki Activity

fix: delay DDG 3-8s aleatorio, logging mejorado en query generation
Build & Deploy ResearchOwl / build-and-push (push) Successful in 6s
Details

Browse Source
This commit is contained in:
ChemaVX
2026-05-04 13:28:54 +00:00
parent 4bef9d2d17
commit ba2b366534
1 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/scraper/exhaustive.py
+4 -2
View File
@@ -181,6 +181,7 @@ class ExhaustiveScraper:
try: try:
import anthropic import anthropic
logger.info("Generating DDG queries with Claude", topic=self.topic)
client = anthropic.AsyncAnthropic(api_key=settings.anthropic_api_key) client = anthropic.AsyncAnthropic(api_key=settings.anthropic_api_key)
prompt = ( prompt = (
f'Generate exactly 8 DuckDuckGo search queries to research: "{self.topic}"\n\n' f'Generate exactly 8 DuckDuckGo search queries to research: "{self.topic}"\n\n'
@@ -207,7 +208,8 @@ class ExhaustiveScraper:
logger.info("DDG queries generated by Claude", queries=queries) logger.info("DDG queries generated by Claude", queries=queries)
return queries return queries
except Exception as e: except Exception as e:
logger.warning("Claude query generation failed, using fallback", error=str(e)) logger.warning("Claude query generation failed, using fallback",
error=str(e), error_type=type(e).__name__)
return fallback return fallback
async def _seed_duckduckgo(self): async def _seed_duckduckgo(self):
@@ -232,7 +234,7 @@ class ExhaustiveScraper:
logger.info("DDG query ok", query=query, results=len(results)) logger.info("DDG query ok", query=query, results=len(results))
except Exception as e: except Exception as e:
logger.warning("DDG query failed", query=query, error=str(e)) logger.warning("DDG query failed", query=query, error=str(e))
await asyncio.sleep(settings.request_delay * 2) await asyncio.sleep(random.uniform(3, 8))
async def _seed_wikipedia(self): async def _seed_wikipedia(self):
"""Search Wikipedia API for correct article URLs. """Search Wikipedia API for correct article URLs.