chemavxandClaude Opus 4.8
54fc8fa11a
Two minor faults found during the GNews capture/prioritisation diagnostic: 1. Hyphens/dashes reached the GNews query verbatim. '-' is GNews's exclusion operator, so a token like "El-Sayed" returned HTTP 400 and wasted a query. _PUNCT_RE now strips '-', en dash and em dash to spaces. 2. The per-cycle GNews budget counter incremented in evaluate() before get_sentiment() checked the API key, so with no key configured the [CYCLE SUMMARY] reported a phantom "gnews_queries_used: 5/5" with zero real requests. Added NewsClient.enabled and gated the GNews block on it; with no key the counter stays 0/5 and no spurious SKIP_GNEWS_PRIORITY is logged. No behaviour change when a key is present. Prioritisation itself was confirmed correct and is left untouched: politics markets are sorted by gnews_priority DESC and prior-extreme markets return before the budget is consumed, so no query is ever spent on a market that cannot trade. Tests: tests/test_news_query.py (4 new); full suite 66 passed. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
797 lines
41 KiB
Python
797 lines
41 KiB
Python
"""
|
||
Bayesian Market Making Strategy.
|
||
|
||
Core idea:
|
||
1. Compute a prior probability for a market outcome using external data
|
||
2. Compare with Polymarket's current price
|
||
3. If divergence > threshold + confidence is high enough → generate signal
|
||
|
||
For crypto markets: if BTC is up 5% and fear/greed is 75 (greed),
|
||
a market asking "Will BTC be above $X?" should be priced higher than
|
||
Polymarket might reflect in a slow-moving order book.
|
||
"""
|
||
import logging
|
||
import math
|
||
import os
|
||
import re
|
||
import uuid
|
||
from dataclasses import dataclass, field
|
||
from datetime import datetime, timedelta, timezone
|
||
from typing import Optional, TYPE_CHECKING
|
||
|
||
from bot.data.polymarket import Market, market_family_key
|
||
from bot.data.external import ExternalSignals
|
||
from bot.data.manifold import MANIFOLD_MATCHER_VERSION, ManifoldMatchResult
|
||
|
||
if TYPE_CHECKING:
|
||
from bot.data.news import NewsClient
|
||
from bot.data.manifold import ManifoldClient
|
||
from bot.data.db import Database
|
||
|
||
log = logging.getLogger(__name__)
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Cost constants (Phase 1 — heuristics, not exact Polymarket exchange costs)
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# spread_estimate: approximate half-spread for medium-liquidity Polymarket
|
||
# markets. Real spread varies by market and time; 0.02 is a conservative
|
||
# starting estimate. Replace with live order-book data when available.
|
||
SPREAD_ESTIMATE: float = 0.02
|
||
|
||
# commission_rate: Polymarket taker fee approximation. Current Polymarket fee
|
||
# is 0% on CLOB but was 2% historically; keeping 2% as a conservative buffer
|
||
# against future fee changes and exchange rate effects.
|
||
COMMISSION_RATE: float = 0.02
|
||
|
||
# Combined cost floor deducted from edge_gross to get edge_net.
|
||
# edge_net = edge_gross - SPREAD_ESTIMATE - COMMISSION_RATE
|
||
TOTAL_COST_RATE: float = SPREAD_ESTIMATE + COMMISSION_RATE # 0.04
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Other strategy constants
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
MIN_CONFIDENCE = 0.55 # Minimum confidence to generate a signal
|
||
|
||
# Log-odds weight applied to the GNews sentiment score (range ±1.0).
|
||
# A weight of 1.5 means a fully negative/positive signal shifts log-odds by ±1.5,
|
||
# which moves a 50% prior to ~18%/82% — strong but not overwhelming.
|
||
NEWS_LOGODDS_WEIGHT = 1.5
|
||
|
||
# Log-odds weight applied to Manifold cross-market probability signal.
|
||
# Weight 0.6: a 30 pp divergence (Manifold 0.75 vs Poly 0.45) produces
|
||
# edge_gross ≈ 0.19, clearing politics far-horizon regime_min=0.12 after costs.
|
||
# Weaker than NEWS_LOGODDS_WEIGHT because Manifold can have illiquid/stale markets.
|
||
MANIFOLD_LOGODDS_WEIGHT = 0.6
|
||
|
||
|
||
def _env_bool(name: str, default: bool) -> bool:
|
||
return os.getenv(name, str(default)).strip().lower() in ("1", "true", "yes", "on")
|
||
|
||
|
||
# ── Manifold activation flags ──────────────────────────────────────────────────
|
||
# Manifold has been retired as an ACTIVE trading signal: a per-category coverage
|
||
# audit (see /api/metrics/manifold-coverage) showed coverage_rate=0.0 across every
|
||
# category in the bot's current universe, so any edge it produced was false edge.
|
||
#
|
||
# MANIFOLD_SIGNAL_ENABLED (default False): when False, Manifold is observational
|
||
# only — its probability never touches the edge model: no manifold_log_adj, no
|
||
# confidence bump, feat_mfld_lo stays 0.0 (so it can never be the dominant
|
||
# feature), and it never contributes to a trade.
|
||
# MANIFOLD_AUDIT_ENABLED (default True): when True the matcher still runs and
|
||
# audit/coverage rows + cooldowns are written, preserving the trail so we can
|
||
# decide later whether to reactivate Manifold in a universe with real coverage.
|
||
# The matcher is only called when at least one flag is on.
|
||
MANIFOLD_SIGNAL_ENABLED = _env_bool("MANIFOLD_SIGNAL_ENABLED", False)
|
||
MANIFOLD_AUDIT_ENABLED = _env_bool("MANIFOLD_AUDIT_ENABLED", True)
|
||
|
||
# GNews free tier: 100 req/day. We limit to 5 queries per trading cycle
|
||
# (politics markets only) and rely on 6 h cache to stay within budget.
|
||
MAX_NEWS_QUERIES_PER_CYCLE = 5
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Manifold evaluation cooldown
|
||
#
|
||
# Per-market backoff so the trading loop stops re-querying Manifold (and flooding
|
||
# manifold_match_audit) for markets whose verdict is stable. Computed from the
|
||
# match result; longer for verdicts that essentially never change.
|
||
# no_results → 24 h (Manifold has no market on this topic)
|
||
# rejected/low_score → 24 h (best candidate below Jaccard threshold)
|
||
# rejected/outcome_mism. → 24 h (outcome types differ)
|
||
# rejected/ambiguous → 24 h (party named but inversion unverifiable)
|
||
# rejected/conditional → 7 d (premise-gated market; structural, won't change)
|
||
# accepted → 1 h (signal is live; refresh probability hourly)
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
def _cooldown_for(result: ManifoldMatchResult) -> tuple[timedelta, str]:
|
||
"""Map a Manifold match result to (retry_delay, cooldown_reason)."""
|
||
if result.status == "accepted":
|
||
return timedelta(hours=1), "accepted"
|
||
if result.status == "no_results":
|
||
return timedelta(hours=24), "no_results"
|
||
# rejected — classify by the reason text the matcher produced
|
||
reason = result.match_reason or "rejected"
|
||
if "conditional_market" in reason:
|
||
return timedelta(days=7), reason
|
||
# outcome_mismatch, ambiguous_inversion, and low_score (jaccard<threshold)
|
||
# all settle in 24 h.
|
||
return timedelta(hours=24), reason
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Phase 4 — Regime-based minimum edge (uses edge_NET, not edge_gross)
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
def _regime_min_edge(category: str, days_to_resolution: int) -> float:
|
||
"""
|
||
Return the minimum edge_net required to execute a trade.
|
||
|
||
Thresholds are higher for far-future politics markets (less signal, more
|
||
noise) and lower for near-term politics (time pressure makes any edge
|
||
actionable). Tech/crypto use a flat threshold.
|
||
|
||
category | days_to_resolution | min_edge_net
|
||
──────────────────────┼────────────────────┼─────────────
|
||
politics | > 60 d | 0.12
|
||
politics | 30–60 d | 0.10
|
||
politics | < 30 d | 0.08
|
||
tech / crypto/finance | any | 0.10
|
||
other / unknown | any | 0.10
|
||
"""
|
||
if category == "politics":
|
||
if days_to_resolution > 60:
|
||
return 0.12
|
||
if days_to_resolution > 30:
|
||
return 0.10
|
||
return 0.08
|
||
return 0.10 # tech, crypto/finance, events, default
|
||
|
||
|
||
def _days_to_resolution(end_date: str) -> int:
|
||
"""Return calendar days until market resolution, or 30 if unknown."""
|
||
if not end_date:
|
||
return 30 # conservative: treat as medium-term
|
||
try:
|
||
dt = datetime.fromisoformat(end_date.replace("Z", "+00:00"))
|
||
if dt.tzinfo is None:
|
||
dt = dt.replace(tzinfo=timezone.utc)
|
||
days = (dt - datetime.now(timezone.utc)).days
|
||
return max(0, days)
|
||
except (ValueError, TypeError):
|
||
return 30
|
||
|
||
|
||
def has_token(text: str, token: str) -> bool:
|
||
"""
|
||
True if `token` appears in `text` as a standalone word.
|
||
|
||
Short crypto tickers (eth, sol, ada, …) must NOT match inside ordinary
|
||
words — "Seth", "dissolved", "Canada" — but must still match the usual
|
||
market phrasings: "ETH", "$ETH", "ETH/USD", "SOL reach $200". Boundaries
|
||
are any non-alphanumeric character (or start/end of string), so "$" and
|
||
"/" delimit correctly.
|
||
"""
|
||
return re.search(
|
||
rf"(?<![A-Za-z0-9]){re.escape(token)}(?![A-Za-z0-9])", text, re.IGNORECASE
|
||
) is not None
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Phase 3 — GNews priority scoring
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
def gnews_priority(market: Market, news: "NewsClient") -> float:
|
||
"""
|
||
Score a market for GNews query priority (higher = more valuable to query).
|
||
|
||
Formula: priority = uncertainty × volume_score × freshness
|
||
|
||
uncertainty = 1 - |prior - 0.5| × 2 (1.0 at 50%, 0.0 at 0%/100%)
|
||
volume_score = min(volume_24h / 10_000, 1.0)
|
||
freshness = NewsClient.get_freshness(question)
|
||
(1.0 never queried → 0.10 queried <2h ago)
|
||
|
||
Markets with occupied families, or that have already been queried recently,
|
||
score lower and receive GNews budget only if capacity remains.
|
||
"""
|
||
prior = max(0.05, min(0.95, market.yes_price))
|
||
uncertainty = 1.0 - abs(prior - 0.5) * 2
|
||
volume_score = min(market.volume_24h / 10_000, 1.0)
|
||
freshness = news.get_freshness(market.question)
|
||
return uncertainty * volume_score * freshness
|
||
|
||
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
# Signal and strategy classes
|
||
# ─────────────────────────────────────────────────────────────────────────────
|
||
|
||
@dataclass
|
||
class TradingSignal:
|
||
market_id: str
|
||
question: str
|
||
polymarket_price: float # Current market price for YES (0-1)
|
||
estimated_prob: float # Our Bayesian estimate (0-1)
|
||
edge: float # Kept for backward compat — equals edge_gross
|
||
confidence: float # How confident we are (0-1)
|
||
direction: str # "BUY_YES" | "BUY_NO"
|
||
reasoning: str # Human-readable explanation for logging
|
||
sources: list[str] # Data sources used
|
||
# ── Phase 1: edge neto ───────────────────────────────────────────────────
|
||
edge_gross: float = 0.0 # |estimated_prob - polymarket_price|
|
||
edge_net: float = 0.0 # edge_gross - SPREAD_ESTIMATE - COMMISSION_RATE
|
||
prior_prob: float = 0.0 # market.yes_price clamped to [0.05, 0.95]
|
||
final_prob: float = 0.0 # estimated_prob (explicit alias)
|
||
# mid_price: (bid+ask)/2 from order book when available; falls back to
|
||
# market.yes_price. Order-book fetching is a future enhancement — using
|
||
# yes_price here is conservative (already the ask side).
|
||
mid_price: float = 0.0
|
||
spread_estimate: float = SPREAD_ESTIMATE
|
||
# ── Phase 2: market families ─────────────────────────────────────────────
|
||
family_key: str = ""
|
||
# ── Phase 4: regime ──────────────────────────────────────────────────────
|
||
regime_min_edge: float = 0.10
|
||
# ── Phase 6: per-feature log-odds contributions ───────────────────────────
|
||
# All values are in log-odds space for direct comparability.
|
||
# feat_fg_lo / feat_mom_lo: probability-delta × 2 → log-odds.
|
||
# feat_news_lo / feat_mfld_lo: already log-odds (no scaling).
|
||
# feat_btc_dom_lo: btc-dominance probability-delta × 2 → log-odds.
|
||
feat_fg_lo: float = 0.0
|
||
feat_mom_lo: float = 0.0
|
||
feat_news_lo: float = 0.0
|
||
feat_mfld_lo: float = 0.0
|
||
feat_btc_dom_lo: float = 0.0
|
||
# ── Manifold match audit (propagated → Order → Trade → DB) ───────────────
|
||
# mfld_audit_id: UUID of the manifold_match_audit row; used to mark
|
||
# used_in_trade=TRUE after executor confirms the trade was executed.
|
||
mfld_audit_id: Optional[str] = None
|
||
mfld_market_id: Optional[str] = None
|
||
mfld_market_title: Optional[str] = None
|
||
mfld_market_url: Optional[str] = None
|
||
mfld_prob_raw: Optional[float] = None
|
||
mfld_prob_final: Optional[float] = None
|
||
mfld_inverted: bool = False
|
||
mfld_match_score: Optional[float] = None
|
||
mfld_match_reason: Optional[str] = None
|
||
mfld_match_status: Optional[str] = None
|
||
|
||
|
||
class BayesianStrategy:
|
||
"""
|
||
Estimates true probability using external signals and Bayesian updating.
|
||
|
||
Prior: Polymarket's current YES price (market consensus — not 0.5)
|
||
Likelihood updates from:
|
||
- BTC/ETH price momentum
|
||
- Fear & Greed index
|
||
- Market cap trend / BTC dominance
|
||
- GNews sentiment (politics only, capped at MAX_NEWS_QUERIES_PER_CYCLE)
|
||
|
||
Execution gate (Phase 1 + 4):
|
||
- Compute edge_net = edge_gross - SPREAD_ESTIMATE - COMMISSION_RATE
|
||
- Only trade when edge_net > regime_min_edge(category, days_to_resolution)
|
||
|
||
Family deduplication (Phase 2):
|
||
- At most 1 open position per market family per cycle.
|
||
- Caller passes occupied_families; this method skips and logs SKIP_FAMILY.
|
||
|
||
GNews prioritisation (Phase 3):
|
||
- Caller pre-sorts politics markets by gnews_priority() (desc) so the
|
||
highest-value markets consume the GNews budget first.
|
||
- Within evaluate(), the per-cycle cap is enforced.
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
news: Optional["NewsClient"] = None,
|
||
manifold: Optional["ManifoldClient"] = None,
|
||
db: Optional["Database"] = None,
|
||
) -> None:
|
||
self._signal_count = 0
|
||
self._news = news
|
||
self._manifold = manifold
|
||
self._db = db
|
||
self._news_queries_this_cycle = 0
|
||
# Per-cycle counters — reset by reset_cycle(), read by get_cycle_stats()
|
||
self._skip_family: int = 0
|
||
self._skip_prior_extreme: int = 0
|
||
self._skip_edge_net_nonpositive: int = 0 # edge_net <= 0
|
||
self._skip_edge_net_below_regime: int = 0 # 0 < edge_net < regime_min
|
||
self._manifold_fetched: int = 0 # markets where Manifold prob was retrieved
|
||
self._manifold_on_trade: int = 0 # subset of above that ended in a trade signal
|
||
# (edge_gross, edge_net, regime_min) for every market that reached the
|
||
# edge computation stage (passed prior-extreme, family, unsupported filters)
|
||
self._evaluated_edges: list[tuple[float, float, float]] = []
|
||
|
||
def reset_cycle(self) -> None:
|
||
"""Call once at the start of each trading cycle to reset per-cycle counters."""
|
||
self._news_queries_this_cycle = 0
|
||
self._skip_family = 0
|
||
self._skip_prior_extreme = 0
|
||
self._skip_edge_net_nonpositive = 0
|
||
self._skip_edge_net_below_regime = 0
|
||
self._manifold_fetched = 0
|
||
self._manifold_on_trade = 0
|
||
self._evaluated_edges = []
|
||
|
||
def get_cycle_stats(self) -> dict:
|
||
"""Return per-cycle counters for the [CYCLE SUMMARY] log block."""
|
||
edges = self._evaluated_edges
|
||
all_gross = [g for g, n, r in edges]
|
||
all_net = [n for g, n, r in edges]
|
||
return {
|
||
"skip_family": self._skip_family,
|
||
"skip_prior_extreme": self._skip_prior_extreme,
|
||
"skip_edge_net_nonpositive": self._skip_edge_net_nonpositive,
|
||
"skip_edge_net_below_regime": self._skip_edge_net_below_regime,
|
||
"gnews_queries_used": self._news_queries_this_cycle,
|
||
"max_edge_gross": max(all_gross) if all_gross else 0.0,
|
||
"max_edge_net": max(all_net) if all_net else 0.0,
|
||
"evaluated_count": len(edges),
|
||
"gross_gt_002": sum(1 for g in all_gross if g > 0.02),
|
||
"gross_gt_004": sum(1 for g in all_gross if g > 0.04),
|
||
"manifold_matches_accepted": self._manifold_on_trade,
|
||
"manifold_matches_rejected": self._manifold_fetched - self._manifold_on_trade,
|
||
}
|
||
|
||
async def evaluate(
|
||
self,
|
||
market: Market,
|
||
ext: ExternalSignals,
|
||
occupied_families: set[str],
|
||
) -> Optional[TradingSignal]:
|
||
"""
|
||
Evaluate a market and return a TradingSignal if actionable.
|
||
|
||
Returns None with a structured log line in all skip cases.
|
||
Skip reasons (Phase 5 observability):
|
||
SKIP_UNSUPPORTED — category not supported
|
||
SKIP_NO_SIGNALS — external data unavailable
|
||
SKIP_PRIOR_EXTREME — prior < 0.08 or > 0.92
|
||
SKIP_FAMILY — family already has an open/pending position
|
||
SKIP_EDGE_NET — edge_net < regime_min_edge
|
||
SKIP_CONFIDENCE — confidence < MIN_CONFIDENCE
|
||
"""
|
||
question_lower = market.question.lower()
|
||
category = market.category
|
||
|
||
# ── Classify market type ─────────────────────────────────────────────
|
||
is_price_above = any(w in question_lower for w in [
|
||
"above", "over", "exceed", "higher", "atleast", "reach",
|
||
])
|
||
is_price_below = any(w in question_lower for w in [
|
||
"below", "under", "less than", "lower", "drop",
|
||
])
|
||
|
||
# Short tickers need word boundaries: "Seth" contains "eth",
|
||
# "dissolved" contains "sol", "Canada" contains "ada". Long
|
||
# unambiguous names (bitcoin, ethereum, …) stay as substrings.
|
||
is_btc = has_token(question_lower, "btc") or "bitcoin" in question_lower
|
||
is_eth = has_token(question_lower, "eth") or "ethereum" in question_lower
|
||
is_sol = has_token(question_lower, "sol") or "solana" in question_lower
|
||
is_xrp = has_token(question_lower, "xrp") or "ripple" in question_lower
|
||
is_doge = has_token(question_lower, "doge") or "dogecoin" in question_lower
|
||
is_altcoin = is_sol or is_xrp or is_doge or any(
|
||
has_token(question_lower, t) for t in ["ltc", "bnb", "ada", "avax"]
|
||
) or any(
|
||
w in question_lower for w in ["litecoin", "cardano", "avalanche"]
|
||
)
|
||
is_general_crypto = any(
|
||
w in question_lower for w in ["crypto", "market cap", "total market", "altcoin", "defi"]
|
||
)
|
||
is_macro = any(
|
||
w in question_lower for w in [
|
||
"nasdaq", "s&p", "sp500", "inflation", "fed rate", "interest rate", "tariff",
|
||
]
|
||
)
|
||
is_politics = category == "politics"
|
||
is_tech = category == "tech"
|
||
is_events = category == "events"
|
||
|
||
is_any_supported = (
|
||
is_btc or is_eth or is_altcoin or is_general_crypto or is_macro
|
||
or is_politics or is_tech or is_events
|
||
)
|
||
if not is_any_supported:
|
||
log.info(
|
||
"SKIP_UNSUPPORTED %-50s | cat=%r",
|
||
market.question[:50], category,
|
||
)
|
||
return None
|
||
|
||
if not ext.valid:
|
||
log.info(
|
||
"SKIP_NO_SIGNALS %-50s | reason=external data unavailable",
|
||
market.question[:50],
|
||
)
|
||
return None
|
||
|
||
# ── Phase 1: prior + prior-extreme filter ────────────────────────────
|
||
prior = max(0.05, min(0.95, market.yes_price))
|
||
|
||
if market.yes_price < 0.08:
|
||
self._skip_prior_extreme += 1
|
||
log.info(
|
||
"SKIP_PRIOR_EXTREME %-50s | cat=%-12s | prior=%.3f | reason=prior<0.08",
|
||
market.question[:50], category, market.yes_price,
|
||
)
|
||
return None
|
||
if market.yes_price > 0.92:
|
||
self._skip_prior_extreme += 1
|
||
log.info(
|
||
"SKIP_PRIOR_EXTREME %-50s | cat=%-12s | prior=%.3f | reason=prior>0.92",
|
||
market.question[:50], category, market.yes_price,
|
||
)
|
||
return None
|
||
|
||
# ── Phase 2: family deduplication ────────────────────────────────────
|
||
family = market_family_key(market)
|
||
if family in occupied_families:
|
||
self._skip_family += 1
|
||
log.info(
|
||
"SKIP_FAMILY %-50s | cat=%-12s | family=%s",
|
||
market.question[:50], category, family,
|
||
)
|
||
return None
|
||
|
||
# ── Phase 4: regime min-edge ─────────────────────────────────────────
|
||
days = _days_to_resolution(market.end_date)
|
||
regime_min = _regime_min_edge(category, days)
|
||
|
||
# ── Bayesian probability estimation ──────────────────────────────────
|
||
sources: list[str] = [f"Prior=poly({prior:.3f})"]
|
||
adjustments: list[float] = []
|
||
|
||
# Momentum and Fear & Greed only make sense for price markets, where
|
||
# is_price_above gives the adjustment a meaningful sign. For
|
||
# politics/tech/events there is no above/below notion — is_price_above
|
||
# defaults to False (or flips on accidental wording like "reach"), so
|
||
# applying these signals just injected sign noise. Skip them entirely;
|
||
# their contributions stay 0.0 → feat_mom_lo / feat_fg_lo = 0.0.
|
||
is_non_price = is_politics or is_tech or is_events
|
||
|
||
# Signal 1: price momentum (asset-specific; price markets only)
|
||
_momentum_contribution = 0.0
|
||
if not is_non_price:
|
||
if is_btc:
|
||
momentum = ext.btc_change_24h
|
||
asset_label = "BTC"
|
||
elif is_eth:
|
||
momentum = ext.eth_change_24h
|
||
asset_label = "ETH"
|
||
else:
|
||
momentum = ext.total_market_cap_change
|
||
asset_label = "total mktcap"
|
||
|
||
if abs(momentum) > 2:
|
||
momentum_adj = math.tanh(momentum / 20) * 0.15
|
||
_momentum_contribution = momentum_adj if is_price_above else -momentum_adj
|
||
adjustments.append(_momentum_contribution)
|
||
sources.append(f"{asset_label} 24h: {momentum:+.1f}%")
|
||
|
||
# Signal 2: Fear & Greed (price markets only)
|
||
_fg_contribution = 0.0
|
||
if not is_non_price:
|
||
fg = ext.fear_greed_index
|
||
if fg > 70:
|
||
fg_adj = 0.06
|
||
sources.append(f"Fear&Greed: {fg} (greed)")
|
||
elif fg < 30:
|
||
fg_adj = -0.06
|
||
sources.append(f"Fear&Greed: {fg} (fear)")
|
||
else:
|
||
fg_adj = (fg - 50) / 50 * 0.04
|
||
sources.append(f"Fear&Greed: {fg} (neutral)")
|
||
_fg_contribution = fg_adj if is_price_above else -fg_adj
|
||
adjustments.append(_fg_contribution)
|
||
|
||
# Signal 3: BTC dominance — hurts altcoins when high (price markets only)
|
||
# Like momentum and Fear & Greed above: no demonstrated causality for
|
||
# politics/tech/events, even when they legitimately mention a ticker
|
||
# ("Will the ETH ETF be approved?"). For non-price markets the
|
||
# contribution stays 0.0 → feat_btc_dom_lo = 0.0.
|
||
_btc_dom_contribution = 0.0
|
||
if not is_non_price:
|
||
if (is_eth or is_altcoin or is_general_crypto) and ext.btc_dominance > 55:
|
||
_btc_dom_contribution = -0.03 if is_price_above else 0.03
|
||
adjustments.append(_btc_dom_contribution)
|
||
sources.append(f"BTC dom: {ext.btc_dominance:.1f}% (high → alt pressure)")
|
||
elif (is_eth or is_altcoin or is_general_crypto) and ext.btc_dominance < 45:
|
||
_btc_dom_contribution = 0.03 if is_price_above else -0.03
|
||
adjustments.append(_btc_dom_contribution)
|
||
sources.append(f"BTC dom: {ext.btc_dominance:.1f}% (low → alt season)")
|
||
|
||
# Signal 4: GNews sentiment (politics only, budget-gated)
|
||
# Phase 3: caller has pre-sorted markets by gnews_priority() so the
|
||
# highest-value markets reach this block first.
|
||
news_log_adj = 0.0
|
||
# self._news.enabled gates the whole block: with no GNews API key the
|
||
# client is a no-op, so we must not consume (or report) query budget for
|
||
# it — see NewsClient.enabled.
|
||
if is_politics and self._news is not None and self._news.enabled:
|
||
if self._news_queries_this_cycle < MAX_NEWS_QUERIES_PER_CYCLE:
|
||
self._news_queries_this_cycle += 1
|
||
sentiment = await self._news.get_sentiment(market.question)
|
||
if abs(sentiment) > 0.05:
|
||
news_log_adj = sentiment * NEWS_LOGODDS_WEIGHT
|
||
sources.append(f"GNews: {sentiment:+.2f}")
|
||
else:
|
||
log.info(
|
||
"SKIP_GNEWS_PRIORITY %-50s | reason=cycle budget %d reached",
|
||
market.question[:50], MAX_NEWS_QUERIES_PER_CYCLE,
|
||
)
|
||
|
||
# Signal 5: Manifold cross-market probability (politics + tech)
|
||
# Applies a log-odds adjustment proportional to divergence from prior.
|
||
# No query budget — 30 min cache means network cost is paid once per cycle.
|
||
# Now uses ManifoldMatchResult for stricter semantic validation and audit.
|
||
manifold_log_adj = 0.0
|
||
manifold_used = False
|
||
manifold_result: Optional[ManifoldMatchResult] = None
|
||
audit_id: Optional[str] = None
|
||
|
||
if ((is_politics or is_tech) and self._manifold is not None
|
||
and (MANIFOLD_AUDIT_ENABLED or MANIFOLD_SIGNAL_ENABLED)):
|
||
# ── Cooldown gate ────────────────────────────────────────────────
|
||
# Skip markets whose Manifold verdict was recently settled to a
|
||
# stable value. A skip is equivalent to a no-signal: the matcher is
|
||
# NOT called and NO manifold_match_audit row is written, so only real
|
||
# evaluations are recorded. See _cooldown_for() and the
|
||
# manifold_eval_cooldown table.
|
||
in_cooldown = False
|
||
if self._db is not None and market.id:
|
||
try:
|
||
cd = await self._db.get_manifold_cooldown(market.id)
|
||
except Exception as exc:
|
||
log.warning("Failed to read manifold cooldown: %s", exc)
|
||
cd = None
|
||
if cd is not None and datetime.now(timezone.utc) < cd["retry_after"]:
|
||
in_cooldown = True
|
||
log.info(
|
||
"MANIFOLD_COOLDOWN skip market=%s | last_status=%s "
|
||
"retry_after=%s | %s",
|
||
market.id, cd["last_status"],
|
||
cd["retry_after"].isoformat(), market.question[:50],
|
||
)
|
||
|
||
if not in_cooldown:
|
||
manifold_result = await self._manifold.get_match(market.question)
|
||
|
||
# Persist audit record for ALL outcomes (accepted / rejected / no_results).
|
||
# Gated by MANIFOLD_AUDIT_ENABLED so the audit/coverage trail and
|
||
# cooldowns can be kept even while Manifold is observational-only.
|
||
if MANIFOLD_AUDIT_ENABLED and self._db is not None:
|
||
if not market.id:
|
||
log.error(
|
||
"MANIFOLD_AUDIT: market.id is None/empty — skipping audit save | "
|
||
"question=%r", market.question[:60],
|
||
)
|
||
else:
|
||
audit_id = str(uuid.uuid4())
|
||
try:
|
||
await self._db.save_manifold_audit(
|
||
audit_id=audit_id,
|
||
poly_market_id=market.id,
|
||
poly_question=market.question,
|
||
search_query=manifold_result.search_query,
|
||
mfld_market_id=manifold_result.market_id,
|
||
mfld_market_title=manifold_result.market_title,
|
||
mfld_market_url=manifold_result.market_url,
|
||
prob_raw=manifold_result.prob_raw,
|
||
prob_final=manifold_result.prob_final,
|
||
inverted=manifold_result.inverted,
|
||
match_score=manifold_result.match_score,
|
||
match_reason=manifold_result.match_reason,
|
||
match_status=manifold_result.status,
|
||
poly_outcome_type=manifold_result.poly_outcome_type,
|
||
mfld_outcome_type=manifold_result.mfld_outcome_type,
|
||
matcher_version=MANIFOLD_MATCHER_VERSION,
|
||
)
|
||
except Exception as exc:
|
||
log.warning("Failed to save manifold audit: %s", exc)
|
||
audit_id = None
|
||
|
||
# Record the cooldown so this market is not re-queried every
|
||
# cycle. Written even if the audit save above failed — we
|
||
# still performed a real evaluation.
|
||
if market.id:
|
||
delay, cd_reason = _cooldown_for(manifold_result)
|
||
try:
|
||
await self._db.upsert_manifold_cooldown(
|
||
poly_market_id=market.id,
|
||
last_status=manifold_result.status,
|
||
retry_after=datetime.now(timezone.utc) + delay,
|
||
cooldown_reason=cd_reason,
|
||
)
|
||
except Exception as exc:
|
||
log.warning("Failed to save manifold cooldown: %s", exc)
|
||
|
||
# Structured log — both forms for compatibility
|
||
log.info(
|
||
"MANIFOLD_MATCH poly='%s' mfld='%s' score=%s raw=%s final=%s"
|
||
" inverted=%s status=%s reason=%s",
|
||
market.question, manifold_result.market_title,
|
||
manifold_result.match_score, manifold_result.prob_raw,
|
||
manifold_result.prob_final, manifold_result.inverted,
|
||
manifold_result.status, manifold_result.match_reason,
|
||
)
|
||
log.info("MANIFOLD_MATCH", extra={
|
||
"poly_question": market.question,
|
||
"mfld_title": manifold_result.market_title,
|
||
"score": manifold_result.match_score,
|
||
"prob_raw": manifold_result.prob_raw,
|
||
"prob_final": manifold_result.prob_final,
|
||
"inverted": manifold_result.inverted,
|
||
"status": manifold_result.status,
|
||
"reason": manifold_result.match_reason,
|
||
})
|
||
|
||
if (MANIFOLD_SIGNAL_ENABLED
|
||
and manifold_result.status == "accepted"
|
||
and manifold_result.prob_final is not None):
|
||
# ACTIVE signal path — only when explicitly enabled.
|
||
manifold_used = True
|
||
self._manifold_fetched += 1
|
||
m_clamped = max(0.05, min(0.95, manifold_result.prob_final))
|
||
m_log = math.log(m_clamped / (1 - m_clamped))
|
||
p_log = math.log(prior / (1 - prior))
|
||
manifold_log_adj = (m_log - p_log) * MANIFOLD_LOGODDS_WEIGHT
|
||
sources.append(f"Manifold:{manifold_result.prob_final:.2f}")
|
||
elif not MANIFOLD_SIGNAL_ENABLED:
|
||
# Observational-only: matched/audited but NEVER fed to the edge
|
||
# model. manifold_log_adj stays 0.0 → no confidence bump,
|
||
# feat_mfld_lo=0.0 (cannot be dominant), no trade contribution.
|
||
log.info(
|
||
"Manifold: observational_only — signal disabled "
|
||
"(MANIFOLD_SIGNAL_ENABLED=false) | market=%s status=%s",
|
||
market.id, manifold_result.status,
|
||
)
|
||
sources.append("Manifold: observational_only")
|
||
|
||
# Confidence cap: macro/politics/tech signals are weaker proxies
|
||
confidence_cap = 0.65 if (is_macro or is_politics or is_tech or is_events) else 0.90
|
||
|
||
# Posterior via log-odds updating
|
||
log_odds_prior = math.log(prior / (1 - prior))
|
||
total_adj = sum(adjustments)
|
||
estimated_prob = _sigmoid(log_odds_prior + total_adj * 2 + news_log_adj + manifold_log_adj)
|
||
estimated_prob = max(0.05, min(0.95, estimated_prob))
|
||
|
||
# ── Phase 1: edge_gross and edge_net ─────────────────────────────────
|
||
raw_edge = estimated_prob - market.yes_price
|
||
direction = "BUY_YES" if raw_edge > 0 else "BUY_NO"
|
||
edge_gross = abs(raw_edge)
|
||
# NOTE: commission/size_usdc = COMMISSION_RATE always (constant fraction).
|
||
edge_net = edge_gross - SPREAD_ESTIMATE - COMMISSION_RATE
|
||
# mid_price falls back to yes_price; live order-book data is a future enhancement
|
||
mid_price = market.yes_price
|
||
|
||
# Record for cycle summary — every market that reached edge computation
|
||
self._evaluated_edges.append((edge_gross, edge_net, regime_min))
|
||
|
||
# Confidence based on signal agreement
|
||
agreement = sum(1 for a in adjustments if (a > 0) == (total_adj > 0))
|
||
confidence = min(confidence_cap, 0.4 + (agreement / max(len(adjustments), 1)) * 0.5)
|
||
if news_log_adj != 0.0:
|
||
confidence = min(confidence_cap, confidence + 0.10)
|
||
if manifold_log_adj != 0.0:
|
||
confidence = min(confidence_cap, confidence + 0.08)
|
||
|
||
# Per-feature log-odds contributions (Phase 6).
|
||
# fg / mom / btc_dom: probability-delta × 2 → log-odds.
|
||
# news / mfld: already log-odds (LOGODDS_WEIGHT already applied).
|
||
feat_fg_lo = _fg_contribution * 2
|
||
feat_mom_lo = _momentum_contribution * 2
|
||
feat_news_lo = news_log_adj
|
||
feat_mfld_lo = manifold_log_adj
|
||
feat_btc_dom_lo = _btc_dom_contribution * 2
|
||
|
||
feat_str = (
|
||
f"fg_lo={feat_fg_lo:+.4f} mom_lo={feat_mom_lo:+.4f} "
|
||
f"news_lo={feat_news_lo:+.4f} mfld_lo={feat_mfld_lo:+.4f} "
|
||
f"btc_dom_lo={feat_btc_dom_lo:+.4f}"
|
||
)
|
||
|
||
# ── Phase 5: structured audit log ────────────────────────────────────
|
||
passed_gross = edge_gross >= regime_min
|
||
passed_net = edge_net >= regime_min
|
||
can_trade = passed_net and confidence >= MIN_CONFIDENCE
|
||
|
||
if not can_trade:
|
||
# Increment the appropriate edge-net counter
|
||
if edge_net <= 0:
|
||
self._skip_edge_net_nonpositive += 1
|
||
else:
|
||
self._skip_edge_net_below_regime += 1
|
||
skip_parts: list[str] = []
|
||
if not passed_gross:
|
||
skip_parts.append(f"edge_gross={edge_gross:.3f}<{regime_min:.2f}(regime)")
|
||
elif not passed_net:
|
||
skip_parts.append(
|
||
f"edge_net={edge_net:.3f}<{regime_min:.2f}(regime) "
|
||
f"[gross={edge_gross:.3f} pass]"
|
||
)
|
||
if confidence < MIN_CONFIDENCE:
|
||
skip_parts.append(f"conf={confidence:.2f}<{MIN_CONFIDENCE}")
|
||
log.info(
|
||
"SKIP_EDGE_NET %-50s | cat=%-12s | family=%-28s | "
|
||
"prior=%.3f | est=%.3f | gross=%+.3f | net=%+.3f | "
|
||
"regime=%.2f | days=%d | conf=%.2f | %s | signals=%s | %s",
|
||
market.question[:50], category, family,
|
||
prior, estimated_prob, edge_gross, edge_net,
|
||
regime_min, days, confidence, feat_str,
|
||
", ".join(sources[1:]) or "none",
|
||
" | ".join(skip_parts),
|
||
)
|
||
return None
|
||
|
||
reasoning = (
|
||
f"Prior=poly({prior:.3f}) → estimate={estimated_prob:.3f} | "
|
||
f"Poly price={market.yes_price:.3f} | "
|
||
f"edge_gross={edge_gross:+.3f} | edge_net={edge_net:+.3f} | "
|
||
f"regime_min={regime_min:.2f} | days={days} | "
|
||
f"family={family} | "
|
||
f"Direction={direction} | "
|
||
f"{feat_str} | "
|
||
f"Signals: {', '.join(sources[1:])}"
|
||
)
|
||
|
||
log.info(
|
||
"TRADE %-50s | cat=%-12s | family=%-28s | "
|
||
"prior=%.3f | est=%.3f | gross=%+.3f | net=%+.3f | "
|
||
"regime=%.2f | days=%d | conf=%.2f | dir=%-8s | %s | signals=%s",
|
||
market.question[:50], category, family,
|
||
prior, estimated_prob, edge_gross, edge_net,
|
||
regime_min, days, confidence, direction, feat_str,
|
||
", ".join(sources[1:]) or "none",
|
||
)
|
||
|
||
self._signal_count += 1
|
||
if manifold_used:
|
||
self._manifold_on_trade += 1
|
||
return TradingSignal(
|
||
market_id=market.id,
|
||
question=market.question,
|
||
polymarket_price=market.yes_price,
|
||
estimated_prob=estimated_prob,
|
||
edge=edge_gross, # backward compat — same as edge_gross
|
||
confidence=confidence,
|
||
direction=direction,
|
||
reasoning=reasoning,
|
||
sources=sources,
|
||
# Phase 1 new fields
|
||
edge_gross=edge_gross,
|
||
edge_net=edge_net,
|
||
prior_prob=prior,
|
||
final_prob=estimated_prob,
|
||
mid_price=mid_price,
|
||
spread_estimate=SPREAD_ESTIMATE,
|
||
# Phase 2 new fields
|
||
family_key=family,
|
||
# Phase 4 new fields
|
||
regime_min_edge=regime_min,
|
||
# Phase 6 new fields — all in log-odds space
|
||
feat_fg_lo=feat_fg_lo,
|
||
feat_mom_lo=feat_mom_lo,
|
||
feat_news_lo=feat_news_lo,
|
||
feat_mfld_lo=feat_mfld_lo,
|
||
feat_btc_dom_lo=feat_btc_dom_lo,
|
||
# Manifold match audit — propagated through Order → Trade → DB.
|
||
# mfld_audit_id is the hook main.py uses to flip the audit row's
|
||
# used_in_trade=TRUE; suppress it when observational so the trail
|
||
# truthfully shows Manifold drove no trades. The mfld_* fields below
|
||
# stay as observational record (feat_mfld_lo is already 0.0).
|
||
mfld_audit_id=(audit_id if MANIFOLD_SIGNAL_ENABLED else None),
|
||
mfld_market_id=manifold_result.market_id if manifold_result else None,
|
||
mfld_market_title=manifold_result.market_title if manifold_result else None,
|
||
mfld_market_url=manifold_result.market_url if manifold_result else None,
|
||
mfld_prob_raw=manifold_result.prob_raw if manifold_result else None,
|
||
mfld_prob_final=manifold_result.prob_final if manifold_result else None,
|
||
mfld_inverted=manifold_result.inverted if manifold_result else False,
|
||
mfld_match_score=manifold_result.match_score if manifold_result else None,
|
||
mfld_match_reason=manifold_result.match_reason if manifold_result else None,
|
||
mfld_match_status=manifold_result.status if manifold_result else None,
|
||
)
|
||
|
||
|
||
def _sigmoid(x: float) -> float:
|
||
return 1 / (1 + math.exp(-x))
|