feat(manifold): add persistent cooldowns to reduce redundant evaluations
CI/CD / build-and-push (push) Successful in 8s
CI/CD / build-and-push (push) Successful in 8s
The trading loop re-evaluated the same ~22 politics/tech markets every ~60s,
flooding manifold_match_audit with ~76k rows (~3,500 attempts/market) of which
none carried new information, making the metrics uninterpretable.
Add per-market persistent cooldowns:
- New table manifold_eval_cooldown (poly_market_id PK, last_evaluated_at,
last_status, retry_after, cooldown_reason) created via run_migrations.
- bayesian.evaluate() consults the cooldown BEFORE calling the matcher and skips
the call entirely while now() < retry_after — no matcher call, no audit row,
no signal (equivalent to no_results). After a real evaluation it upserts the
cooldown with a verdict-dependent backoff: no_results/low_score/outcome_mismatch/
ambiguous_inversion -> 24h, conditional_market -> 7d, accepted -> 1h.
- manifold.py stays a pure client/matcher with no DB access.
- New db methods get_manifold_cooldown / upsert_manifold_cooldown.
- /api/metrics/manifold-matches summary gains unique_markets
{evaluated, accepted, coverage_rate} for per-market (not per-attempt) coverage.
Matching logic, MANIFOLD_MATCHER_VERSION, MANIFOLD_LOGODDS_WEIGHT, edge/exposure
thresholds and existing audit rows are unchanged.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
chemavx
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
df988a36b6
commit
98abd96fd2
+125
-59
@@ -14,7 +14,7 @@ import logging
|
||||
import math
|
||||
import uuid
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import Optional, TYPE_CHECKING
|
||||
|
||||
from bot.data.polymarket import Market, market_family_key
|
||||
@@ -66,6 +66,34 @@ MANIFOLD_LOGODDS_WEIGHT = 0.6
|
||||
MAX_NEWS_QUERIES_PER_CYCLE = 5
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Manifold evaluation cooldown
|
||||
#
|
||||
# Per-market backoff so the trading loop stops re-querying Manifold (and flooding
|
||||
# manifold_match_audit) for markets whose verdict is stable. Computed from the
|
||||
# match result; longer for verdicts that essentially never change.
|
||||
# no_results → 24 h (Manifold has no market on this topic)
|
||||
# rejected/low_score → 24 h (best candidate below Jaccard threshold)
|
||||
# rejected/outcome_mism. → 24 h (outcome types differ)
|
||||
# rejected/ambiguous → 24 h (party named but inversion unverifiable)
|
||||
# rejected/conditional → 7 d (premise-gated market; structural, won't change)
|
||||
# accepted → 1 h (signal is live; refresh probability hourly)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
def _cooldown_for(result: ManifoldMatchResult) -> tuple[timedelta, str]:
|
||||
"""Map a Manifold match result to (retry_delay, cooldown_reason)."""
|
||||
if result.status == "accepted":
|
||||
return timedelta(hours=1), "accepted"
|
||||
if result.status == "no_results":
|
||||
return timedelta(hours=24), "no_results"
|
||||
# rejected — classify by the reason text the matcher produced
|
||||
reason = result.match_reason or "rejected"
|
||||
if "conditional_market" in reason:
|
||||
return timedelta(days=7), reason
|
||||
# outcome_mismatch, ambiguous_inversion, and low_score (jaccard<threshold)
|
||||
# all settle in 24 h.
|
||||
return timedelta(hours=24), reason
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Phase 4 — Regime-based minimum edge (uses edge_NET, not edge_gross)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
@@ -444,68 +472,106 @@ class BayesianStrategy:
|
||||
audit_id: Optional[str] = None
|
||||
|
||||
if (is_politics or is_tech) and self._manifold is not None:
|
||||
manifold_result = await self._manifold.get_match(market.question)
|
||||
|
||||
# Persist audit record for ALL outcomes (accepted / rejected / no_results)
|
||||
if self._db is not None:
|
||||
if not market.id:
|
||||
log.error(
|
||||
"MANIFOLD_AUDIT: market.id is None/empty — skipping audit save | "
|
||||
"question=%r", market.question[:60],
|
||||
# ── Cooldown gate ────────────────────────────────────────────────
|
||||
# Skip markets whose Manifold verdict was recently settled to a
|
||||
# stable value. A skip is equivalent to a no-signal: the matcher is
|
||||
# NOT called and NO manifold_match_audit row is written, so only real
|
||||
# evaluations are recorded. See _cooldown_for() and the
|
||||
# manifold_eval_cooldown table.
|
||||
in_cooldown = False
|
||||
if self._db is not None and market.id:
|
||||
try:
|
||||
cd = await self._db.get_manifold_cooldown(market.id)
|
||||
except Exception as exc:
|
||||
log.warning("Failed to read manifold cooldown: %s", exc)
|
||||
cd = None
|
||||
if cd is not None and datetime.now(timezone.utc) < cd["retry_after"]:
|
||||
in_cooldown = True
|
||||
log.info(
|
||||
"MANIFOLD_COOLDOWN skip market=%s | last_status=%s "
|
||||
"retry_after=%s | %s",
|
||||
market.id, cd["last_status"],
|
||||
cd["retry_after"].isoformat(), market.question[:50],
|
||||
)
|
||||
else:
|
||||
audit_id = str(uuid.uuid4())
|
||||
try:
|
||||
await self._db.save_manifold_audit(
|
||||
audit_id=audit_id,
|
||||
poly_market_id=market.id,
|
||||
poly_question=market.question,
|
||||
search_query=manifold_result.search_query,
|
||||
mfld_market_id=manifold_result.market_id,
|
||||
mfld_market_title=manifold_result.market_title,
|
||||
mfld_market_url=manifold_result.market_url,
|
||||
prob_raw=manifold_result.prob_raw,
|
||||
prob_final=manifold_result.prob_final,
|
||||
inverted=manifold_result.inverted,
|
||||
match_score=manifold_result.match_score,
|
||||
match_reason=manifold_result.match_reason,
|
||||
match_status=manifold_result.status,
|
||||
poly_outcome_type=manifold_result.poly_outcome_type,
|
||||
mfld_outcome_type=manifold_result.mfld_outcome_type,
|
||||
matcher_version=MANIFOLD_MATCHER_VERSION,
|
||||
|
||||
if not in_cooldown:
|
||||
manifold_result = await self._manifold.get_match(market.question)
|
||||
|
||||
# Persist audit record for ALL outcomes (accepted / rejected / no_results)
|
||||
if self._db is not None:
|
||||
if not market.id:
|
||||
log.error(
|
||||
"MANIFOLD_AUDIT: market.id is None/empty — skipping audit save | "
|
||||
"question=%r", market.question[:60],
|
||||
)
|
||||
except Exception as exc:
|
||||
log.warning("Failed to save manifold audit: %s", exc)
|
||||
audit_id = None
|
||||
else:
|
||||
audit_id = str(uuid.uuid4())
|
||||
try:
|
||||
await self._db.save_manifold_audit(
|
||||
audit_id=audit_id,
|
||||
poly_market_id=market.id,
|
||||
poly_question=market.question,
|
||||
search_query=manifold_result.search_query,
|
||||
mfld_market_id=manifold_result.market_id,
|
||||
mfld_market_title=manifold_result.market_title,
|
||||
mfld_market_url=manifold_result.market_url,
|
||||
prob_raw=manifold_result.prob_raw,
|
||||
prob_final=manifold_result.prob_final,
|
||||
inverted=manifold_result.inverted,
|
||||
match_score=manifold_result.match_score,
|
||||
match_reason=manifold_result.match_reason,
|
||||
match_status=manifold_result.status,
|
||||
poly_outcome_type=manifold_result.poly_outcome_type,
|
||||
mfld_outcome_type=manifold_result.mfld_outcome_type,
|
||||
matcher_version=MANIFOLD_MATCHER_VERSION,
|
||||
)
|
||||
except Exception as exc:
|
||||
log.warning("Failed to save manifold audit: %s", exc)
|
||||
audit_id = None
|
||||
|
||||
# Structured log — both forms for compatibility
|
||||
log.info(
|
||||
"MANIFOLD_MATCH poly='%s' mfld='%s' score=%s raw=%s final=%s"
|
||||
" inverted=%s status=%s reason=%s",
|
||||
market.question, manifold_result.market_title,
|
||||
manifold_result.match_score, manifold_result.prob_raw,
|
||||
manifold_result.prob_final, manifold_result.inverted,
|
||||
manifold_result.status, manifold_result.match_reason,
|
||||
)
|
||||
log.info("MANIFOLD_MATCH", extra={
|
||||
"poly_question": market.question,
|
||||
"mfld_title": manifold_result.market_title,
|
||||
"score": manifold_result.match_score,
|
||||
"prob_raw": manifold_result.prob_raw,
|
||||
"prob_final": manifold_result.prob_final,
|
||||
"inverted": manifold_result.inverted,
|
||||
"status": manifold_result.status,
|
||||
"reason": manifold_result.match_reason,
|
||||
})
|
||||
# Record the cooldown so this market is not re-queried every
|
||||
# cycle. Written even if the audit save above failed — we
|
||||
# still performed a real evaluation.
|
||||
if market.id:
|
||||
delay, cd_reason = _cooldown_for(manifold_result)
|
||||
try:
|
||||
await self._db.upsert_manifold_cooldown(
|
||||
poly_market_id=market.id,
|
||||
last_status=manifold_result.status,
|
||||
retry_after=datetime.now(timezone.utc) + delay,
|
||||
cooldown_reason=cd_reason,
|
||||
)
|
||||
except Exception as exc:
|
||||
log.warning("Failed to save manifold cooldown: %s", exc)
|
||||
|
||||
if manifold_result.status == "accepted" and manifold_result.prob_final is not None:
|
||||
manifold_used = True
|
||||
self._manifold_fetched += 1
|
||||
m_clamped = max(0.05, min(0.95, manifold_result.prob_final))
|
||||
m_log = math.log(m_clamped / (1 - m_clamped))
|
||||
p_log = math.log(prior / (1 - prior))
|
||||
manifold_log_adj = (m_log - p_log) * MANIFOLD_LOGODDS_WEIGHT
|
||||
sources.append(f"Manifold:{manifold_result.prob_final:.2f}")
|
||||
# Structured log — both forms for compatibility
|
||||
log.info(
|
||||
"MANIFOLD_MATCH poly='%s' mfld='%s' score=%s raw=%s final=%s"
|
||||
" inverted=%s status=%s reason=%s",
|
||||
market.question, manifold_result.market_title,
|
||||
manifold_result.match_score, manifold_result.prob_raw,
|
||||
manifold_result.prob_final, manifold_result.inverted,
|
||||
manifold_result.status, manifold_result.match_reason,
|
||||
)
|
||||
log.info("MANIFOLD_MATCH", extra={
|
||||
"poly_question": market.question,
|
||||
"mfld_title": manifold_result.market_title,
|
||||
"score": manifold_result.match_score,
|
||||
"prob_raw": manifold_result.prob_raw,
|
||||
"prob_final": manifold_result.prob_final,
|
||||
"inverted": manifold_result.inverted,
|
||||
"status": manifold_result.status,
|
||||
"reason": manifold_result.match_reason,
|
||||
})
|
||||
|
||||
if manifold_result.status == "accepted" and manifold_result.prob_final is not None:
|
||||
manifold_used = True
|
||||
self._manifold_fetched += 1
|
||||
m_clamped = max(0.05, min(0.95, manifold_result.prob_final))
|
||||
m_log = math.log(m_clamped / (1 - m_clamped))
|
||||
p_log = math.log(prior / (1 - prior))
|
||||
manifold_log_adj = (m_log - p_log) * MANIFOLD_LOGODDS_WEIGHT
|
||||
sources.append(f"Manifold:{manifold_result.prob_final:.2f}")
|
||||
|
||||
# Confidence cap: macro/politics/tech signals are weaker proxies
|
||||
confidence_cap = 0.65 if (is_macro or is_politics or is_tech or is_events) else 0.90
|
||||
|
||||
Reference in New Issue
Block a user