"""
Bayesian Market Making Strategy.

Core idea:
1. Compute a prior probability for a market outcome using external data
2. Compare with Polymarket's current price
3. If divergence > threshold + confidence is high enough → generate signal

For crypto markets: if BTC is up 5% and fear/greed is 75 (greed),
a market asking "Will BTC be above $X?" should be priced higher than
Polymarket might reflect in a slow-moving order book.
"""
import logging
import math
import os
import re
import uuid
from dataclasses import dataclass, field
from datetime import datetime, timedelta, timezone
from typing import Optional, TYPE_CHECKING

from bot.data.polymarket import Market, market_family_key
from bot.data.external import ExternalSignals
from bot.data.manifold import MANIFOLD_MATCHER_VERSION, ManifoldMatchResult

if TYPE_CHECKING:
    from bot.data.news import NewsClient
    from bot.data.manifold import ManifoldClient
    from bot.data.db import Database

log = logging.getLogger(__name__)

# ─────────────────────────────────────────────────────────────────────────────
# Cost constants (Phase 1 — heuristics, not exact Polymarket exchange costs)
# ─────────────────────────────────────────────────────────────────────────────
# spread_estimate: approximate half-spread for medium-liquidity Polymarket
#   markets.  Real spread varies by market and time; 0.02 is a conservative
#   starting estimate.  Replace with live order-book data when available.
SPREAD_ESTIMATE: float = 0.02

# commission_rate: Polymarket taker fee approximation.  Current Polymarket fee
#   is 0% on CLOB but was 2% historically; keeping 2% as a conservative buffer
#   against future fee changes and exchange rate effects.
COMMISSION_RATE: float = 0.02

# Combined cost floor deducted from edge_gross to get edge_net.
# edge_net = edge_gross - SPREAD_ESTIMATE - COMMISSION_RATE
TOTAL_COST_RATE: float = SPREAD_ESTIMATE + COMMISSION_RATE  # 0.04

# ─────────────────────────────────────────────────────────────────────────────
# Other strategy constants
# ─────────────────────────────────────────────────────────────────────────────
MIN_CONFIDENCE = 0.55   # Minimum confidence to generate a signal

# Log-odds weight applied to the GNews sentiment score (range ±1.0).
# A weight of 1.5 means a fully negative/positive signal shifts log-odds by ±1.5,
# which moves a 50% prior to ~18%/82% — strong but not overwhelming.
NEWS_LOGODDS_WEIGHT = 1.5

# Log-odds weight applied to Manifold cross-market probability signal.
# Weight 0.6: a 30 pp divergence (Manifold 0.75 vs Poly 0.45) produces
# edge_gross ≈ 0.19, clearing politics far-horizon regime_min=0.12 after costs.
# Weaker than NEWS_LOGODDS_WEIGHT because Manifold can have illiquid/stale markets.
MANIFOLD_LOGODDS_WEIGHT = 0.6


def _env_bool(name: str, default: bool) -> bool:
    return os.getenv(name, str(default)).strip().lower() in ("1", "true", "yes", "on")


# ── Manifold activation flags ──────────────────────────────────────────────────
# Manifold has been retired as an ACTIVE trading signal: a per-category coverage
# audit (see /api/metrics/manifold-coverage) showed coverage_rate=0.0 across every
# category in the bot's current universe, so any edge it produced was false edge.
#
# MANIFOLD_SIGNAL_ENABLED (default False): when False, Manifold is observational
#   only — its probability never touches the edge model: no manifold_log_adj, no
#   confidence bump, feat_mfld_lo stays 0.0 (so it can never be the dominant
#   feature), and it never contributes to a trade.
# MANIFOLD_AUDIT_ENABLED (default True): when True the matcher still runs and
#   audit/coverage rows + cooldowns are written, preserving the trail so we can
#   decide later whether to reactivate Manifold in a universe with real coverage.
# The matcher is only called when at least one flag is on.
MANIFOLD_SIGNAL_ENABLED = _env_bool("MANIFOLD_SIGNAL_ENABLED", False)
MANIFOLD_AUDIT_ENABLED  = _env_bool("MANIFOLD_AUDIT_ENABLED", True)

# ── GNews guardrail (catastrophic fuse) ────────────────────────────────────────
# Post-mortem NVIDIA 631181: a single strong signal (legacy Manifold 0.13 at
# weight 0.6) flipped a 0.845 market to 0.431 and lost.  With Manifold now
# observational-only and macro signals gated behind is_non_price, GNews
# (weight 1.5) is the only live signal that can move politics markets 20-30 pp
# against the order-book consensus.  This is NOT a fine calibration — it is a
# fuse against the extreme case: one uncorroborated signal violently inverting
# the market.
#
# NEWS_GUARDRAIL_ENABLED: master switch for the fuse.
# MAX_NEWS_ONLY_PROB_SHIFT: when GNews is the ONLY material signal, the final
#   probability is clamped to prior ± this value.  0.25 still allows a 25 pp
#   move (edge_net 0.21 after costs) — trades still happen, sizing is bounded.
# NEWS_MATERIAL_LOGODDS_THRESHOLD: a signal counts as *material* iff its
#   |log-odds contribution| >= this value.  Below it, a signal is noise and
#   does NOT count as corroboration.  If ANY other signal (fg, momentum,
#   btc_dom, manifold) is material, the fuse does not apply.
NEWS_GUARDRAIL_ENABLED = _env_bool("NEWS_GUARDRAIL_ENABLED", True)
MAX_NEWS_ONLY_PROB_SHIFT = float(os.getenv("MAX_NEWS_ONLY_PROB_SHIFT", "0.25"))
NEWS_MATERIAL_LOGODDS_THRESHOLD = float(os.getenv("NEWS_MATERIAL_LOGODDS_THRESHOLD", "0.10"))

# GNews free tier: 100 req/day.  We limit to 5 queries per trading cycle
# (politics markets only) and rely on 6 h cache to stay within budget.
MAX_NEWS_QUERIES_PER_CYCLE = 5


# ─────────────────────────────────────────────────────────────────────────────
# Manifold evaluation cooldown
#
# Per-market backoff so the trading loop stops re-querying Manifold (and flooding
# manifold_match_audit) for markets whose verdict is stable.  Computed from the
# match result; longer for verdicts that essentially never change.
#   no_results            → 24 h  (Manifold has no market on this topic)
#   rejected/low_score    → 24 h  (best candidate below Jaccard threshold)
#   rejected/outcome_mism. → 24 h  (outcome types differ)
#   rejected/ambiguous    → 24 h  (party named but inversion unverifiable)
#   rejected/conditional  →  7 d  (premise-gated market; structural, won't change)
#   accepted              →  1 h  (signal is live; refresh probability hourly)
# ─────────────────────────────────────────────────────────────────────────────
def _cooldown_for(result: ManifoldMatchResult) -> tuple[timedelta, str]:
    """Map a Manifold match result to (retry_delay, cooldown_reason)."""
    if result.status == "accepted":
        return timedelta(hours=1), "accepted"
    if result.status == "no_results":
        return timedelta(hours=24), "no_results"
    # rejected — classify by the reason text the matcher produced
    reason = result.match_reason or "rejected"
    if "conditional_market" in reason:
        return timedelta(days=7), reason
    # outcome_mismatch, ambiguous_inversion, and low_score (jaccard<threshold)
    # all settle in 24 h.
    return timedelta(hours=24), reason


# ─────────────────────────────────────────────────────────────────────────────
# Phase 4 — Regime-based minimum edge (uses edge_NET, not edge_gross)
# ─────────────────────────────────────────────────────────────────────────────

def _regime_min_edge(category: str, days_to_resolution: int) -> float:
    """
    Return the minimum edge_net required to execute a trade.

    Thresholds are higher for far-future politics markets (less signal, more
    noise) and lower for near-term politics (time pressure makes any edge
    actionable).  Tech/crypto use a flat threshold.

    category              | days_to_resolution | min_edge_net
    ──────────────────────┼────────────────────┼─────────────
    politics              | > 60 d             | 0.12
    politics              | 30–60 d            | 0.10
    politics              | < 30 d             | 0.08
    tech / crypto/finance | any                | 0.10
    other / unknown       | any                | 0.10
    """
    if category == "politics":
        if days_to_resolution > 60:
            return 0.12
        if days_to_resolution > 30:
            return 0.10
        return 0.08
    return 0.10  # tech, crypto/finance, events, default


def _days_to_resolution(end_date: str, as_of: Optional[datetime] = None) -> int:
    """Return calendar days until market resolution, or 30 if unknown.

    as_of (Replay R1): reference clock for the computation.  None (production)
    means wall-clock now; a replay run passes the archived cycle_ts so
    days-to-resolution — and therefore the regime edge threshold — is computed
    against the moment the decision was originally made.
    """
    if not end_date:
        return 30  # conservative: treat as medium-term
    try:
        dt = datetime.fromisoformat(end_date.replace("Z", "+00:00"))
        if dt.tzinfo is None:
            dt = dt.replace(tzinfo=timezone.utc)
        now = as_of if as_of is not None else datetime.now(timezone.utc)
        days = (dt - now).days
        return max(0, days)
    except (ValueError, TypeError):
        return 30


def has_token(text: str, token: str) -> bool:
    """
    True if `token` appears in `text` as a standalone word.

    Short crypto tickers (eth, sol, ada, …) must NOT match inside ordinary
    words — "Seth", "dissolved", "Canada" — but must still match the usual
    market phrasings: "ETH", "$ETH", "ETH/USD", "SOL reach $200".  Boundaries
    are any non-alphanumeric character (or start/end of string), so "$" and
    "/" delimit correctly.
    """
    return re.search(
        rf"(?<![A-Za-z0-9]){re.escape(token)}(?![A-Za-z0-9])", text, re.IGNORECASE
    ) is not None


# ─────────────────────────────────────────────────────────────────────────────
# Phase 3 — GNews priority scoring
# ─────────────────────────────────────────────────────────────────────────────

def apply_news_guardrail(
    prior: float,
    raw_final_prob: float,
    feat_news_lo: float,
    other_feats_lo: tuple[float, ...],
) -> tuple[float, bool]:
    """
    GNews guardrail (catastrophic fuse).

    Clamp raw_final_prob to prior ± MAX_NEWS_ONLY_PROB_SHIFT when ALL hold:
      1. NEWS_GUARDRAIL_ENABLED
      2. |feat_news_lo| >= NEWS_MATERIAL_LOGODDS_THRESHOLD (news is material)
      3. every other signal's |log-odds contribution| is below the threshold
         (GNews is the ONLY material signal — no corroboration)

    Returns (final_prob, guardrail_applied).  guardrail_applied is True only
    when the clamp actually changed the value; a raw_final_prob already inside
    the band passes through untouched with applied=False.

    Module globals are read at call time so tests can monkeypatch them.
    """
    if not NEWS_GUARDRAIL_ENABLED:
        return raw_final_prob, False
    if abs(feat_news_lo) < NEWS_MATERIAL_LOGODDS_THRESHOLD:
        return raw_final_prob, False
    if any(abs(v) >= NEWS_MATERIAL_LOGODDS_THRESHOLD for v in other_feats_lo):
        return raw_final_prob, False  # corroborated — fuse does not apply
    clamped = min(
        max(raw_final_prob, prior - MAX_NEWS_ONLY_PROB_SHIFT),
        prior + MAX_NEWS_ONLY_PROB_SHIFT,
    )
    if clamped == raw_final_prob:
        return raw_final_prob, False
    return clamped, True


def gnews_priority(market: Market, news: "NewsClient") -> float:
    """
    Score a market for GNews query priority (higher = more valuable to query).

    Formula:  priority = uncertainty × volume_score × freshness

      uncertainty  = 1 - |prior - 0.5| × 2   (1.0 at 50%, 0.0 at 0%/100%)
      volume_score = min(volume_24h / 10_000, 1.0)
      freshness    = NewsClient.get_freshness(question)
                     (1.0 never queried → 0.10 queried <2h ago)

    Markets with occupied families, or that have already been queried recently,
    score lower and receive GNews budget only if capacity remains.
    """
    prior = max(0.05, min(0.95, market.yes_price))
    uncertainty = 1.0 - abs(prior - 0.5) * 2
    volume_score = min(market.volume_24h / 10_000, 1.0)
    freshness = news.get_freshness(market.question)
    return uncertainty * volume_score * freshness


# ─────────────────────────────────────────────────────────────────────────────
# Signal and strategy classes
# ─────────────────────────────────────────────────────────────────────────────

@dataclass
class TradingSignal:
    market_id: str
    question: str
    polymarket_price: float     # Current market price for YES (0-1)
    estimated_prob: float       # Our Bayesian estimate (0-1)
    edge: float                 # Kept for backward compat — equals edge_gross
    confidence: float           # How confident we are (0-1)
    direction: str              # "BUY_YES" | "BUY_NO"
    reasoning: str              # Human-readable explanation for logging
    sources: list[str]          # Data sources used
    # ── Phase 1: edge neto ───────────────────────────────────────────────────
    edge_gross: float = 0.0         # |estimated_prob - polymarket_price|
    edge_net: float = 0.0           # edge_gross - SPREAD_ESTIMATE - COMMISSION_RATE
    prior_prob: float = 0.0         # market.yes_price clamped to [0.05, 0.95]
    final_prob: float = 0.0         # estimated_prob (explicit alias)
    # mid_price: (bid+ask)/2 from order book when available; falls back to
    # market.yes_price.  Order-book fetching is a future enhancement — using
    # yes_price here is conservative (already the ask side).
    mid_price: float = 0.0
    spread_estimate: float = SPREAD_ESTIMATE
    # ── Phase 2: market families ─────────────────────────────────────────────
    family_key: str = ""
    # ── Phase 4: regime ──────────────────────────────────────────────────────
    regime_min_edge: float = 0.10
    # ── Phase 6: per-feature log-odds contributions ───────────────────────────
    # All values are in log-odds space for direct comparability.
    # feat_fg_lo / feat_mom_lo: probability-delta × 2 → log-odds.
    # feat_news_lo / feat_mfld_lo: already log-odds (no scaling).
    # feat_btc_dom_lo: btc-dominance probability-delta × 2 → log-odds.
    feat_fg_lo:      float = 0.0
    feat_mom_lo:     float = 0.0
    feat_news_lo:    float = 0.0
    feat_mfld_lo:    float = 0.0
    feat_btc_dom_lo: float = 0.0
    # ── Manifold match audit (propagated → Order → Trade → DB) ───────────────
    # mfld_audit_id: UUID of the manifold_match_audit row; used to mark
    #   used_in_trade=TRUE after executor confirms the trade was executed.
    mfld_audit_id:    Optional[str]   = None
    mfld_market_id:   Optional[str]   = None
    mfld_market_title: Optional[str]  = None
    mfld_market_url:  Optional[str]   = None
    mfld_prob_raw:    Optional[float] = None
    mfld_prob_final:  Optional[float] = None
    mfld_inverted:    bool            = False
    mfld_match_score: Optional[float] = None
    mfld_match_reason: Optional[str]  = None
    mfld_match_status: Optional[str]  = None


class BayesianStrategy:
    """
    Estimates true probability using external signals and Bayesian updating.

    Prior: Polymarket's current YES price (market consensus — not 0.5)
    Likelihood updates from:
    - BTC/ETH price momentum
    - Fear & Greed index
    - Market cap trend / BTC dominance
    - GNews sentiment (politics only, capped at MAX_NEWS_QUERIES_PER_CYCLE)

    Execution gate (Phase 1 + 4):
    - Compute edge_net = edge_gross - SPREAD_ESTIMATE - COMMISSION_RATE
    - Only trade when edge_net > regime_min_edge(category, days_to_resolution)

    Family deduplication (Phase 2):
    - At most 1 open position per market family per cycle.
    - Caller passes occupied_families; this method skips and logs SKIP_FAMILY.

    GNews prioritisation (Phase 3):
    - Caller pre-sorts politics markets by gnews_priority() (desc) so the
      highest-value markets consume the GNews budget first.
    - Within evaluate(), the per-cycle cap is enforced.
    """

    def __init__(
        self,
        news: Optional["NewsClient"] = None,
        manifold: Optional["ManifoldClient"] = None,
        db: Optional["Database"] = None,
    ) -> None:
        self._signal_count = 0
        self._news = news
        self._manifold = manifold
        self._db = db
        self._news_queries_this_cycle = 0
        # Per-cycle counters — reset by reset_cycle(), read by get_cycle_stats()
        self._skip_family: int = 0
        self._skip_prior_extreme: int = 0
        self._skip_edge_net_nonpositive: int = 0   # edge_net <= 0
        self._skip_edge_net_below_regime: int = 0  # 0 < edge_net < regime_min
        self._manifold_fetched: int = 0   # markets where Manifold prob was retrieved
        self._manifold_on_trade: int = 0  # subset of above that ended in a trade signal
        # (edge_gross, edge_net, regime_min) for every market that reached the
        # edge computation stage (passed prior-extreme, family, unsupported filters)
        self._evaluated_edges: list[tuple[float, float, float]] = []
        # GNews guardrail observability — only markets with material news
        self._news_shifts: list[float] = []       # final_prob - prior, signed
        self._news_guardrail_applied: int = 0
        self._news_changed_decisions: int = 0
        # Replay R0: per-(market, cycle) decision records, drained by main.py
        # into the signals table after each evaluation loop.
        self._cycle_records: list[dict] = []

    def reset_cycle(self) -> None:
        """Call once at the start of each trading cycle to reset per-cycle counters."""
        self._news_queries_this_cycle = 0
        self._skip_family = 0
        self._skip_prior_extreme = 0
        self._skip_edge_net_nonpositive = 0
        self._skip_edge_net_below_regime = 0
        self._manifold_fetched = 0
        self._manifold_on_trade = 0
        self._evaluated_edges = []
        self._news_shifts = []
        self._news_guardrail_applied = 0
        self._news_changed_decisions = 0
        self._cycle_records = []

    def record_skip(self, market: Market, skip_reason: str) -> None:
        """Record a skip decided OUTSIDE evaluate() (e.g. reentry_guard in main)."""
        self._record(market, skip_reason=skip_reason)

    def drain_cycle_records(self) -> list[dict]:
        """Return and clear this cycle's decision records (Replay R0)."""
        records, self._cycle_records = self._cycle_records, []
        return records

    def _record(self, market: Market, skip_reason: Optional[str], **fields) -> None:
        """Append one decision record. Early skips leave most fields None —
        the archive still shows the market existed and why it went no further."""
        rec = {
            "market_id": market.id,
            "polymarket_price": market.yes_price,
            "category": market.category,
            "volume_24h": market.volume_24h,
            "skip_reason": skip_reason,
            "family_key": None,
            "prior_prob": None,
            "estimated_prob": None,
            "raw_final_prob": None,
            "edge_gross": None,
            "edge_net": None,
            "regime_min_edge": None,
            "days_to_resolution": None,
            "confidence": None,
            "direction": None,
            "passed_gross": None,
            "passed_net": None,
            "news_sentiment": None,
            "news_budget_skipped": None,
            "guardrail_applied": None,
            "guardrail_changed_decision": None,
            "feat_fg_lo": None,
            "feat_mom_lo": None,
            "feat_news_lo": None,
            "feat_mfld_lo": None,
            "feat_btc_dom_lo": None,
            "acted_on": False,
        }
        rec.update(fields)
        self._cycle_records.append(rec)

    def get_cycle_stats(self) -> dict:
        """Return per-cycle counters for the [CYCLE SUMMARY] log block."""
        edges = self._evaluated_edges
        all_gross = [g for g, n, r in edges]
        all_net   = [n for g, n, r in edges]
        return {
            "skip_family":                self._skip_family,
            "skip_prior_extreme":         self._skip_prior_extreme,
            "skip_edge_net_nonpositive":  self._skip_edge_net_nonpositive,
            "skip_edge_net_below_regime": self._skip_edge_net_below_regime,
            "gnews_queries_used":         self._news_queries_this_cycle,
            "max_edge_gross": max(all_gross) if all_gross else 0.0,
            "max_edge_net":   max(all_net)   if all_net   else 0.0,
            "evaluated_count": len(edges),
            "gross_gt_002": sum(1 for g in all_gross if g > 0.02),
            "gross_gt_004": sum(1 for g in all_gross if g > 0.04),
            "manifold_matches_accepted": self._manifold_on_trade,
            "manifold_matches_rejected": self._manifold_fetched - self._manifold_on_trade,
            # GNews guardrail — markets with |news_lo| >= NEWS_MATERIAL_LOGODDS_THRESHOLD
            "news_with_material":     len(self._news_shifts),
            "news_avg_shift":         (sum(self._news_shifts) / len(self._news_shifts))
                                      if self._news_shifts else 0.0,
            "news_max_shift":         max(self._news_shifts, key=abs)
                                      if self._news_shifts else 0.0,
            "news_guardrail_applied": self._news_guardrail_applied,
            "news_changed_decisions": self._news_changed_decisions,
        }

    async def evaluate(
        self,
        market: Market,
        ext: ExternalSignals,
        occupied_families: set[str],
        as_of: Optional[datetime] = None,
    ) -> Optional[TradingSignal]:
        """
        Evaluate a market and return a TradingSignal if actionable.

        as_of (Replay R1): clock injection — None in production (wall-clock
        now); a replay passes the archived cycle_ts so the regime threshold
        matches the original decision moment.  Only days-to-resolution
        depends on the clock; everything else is a pure function of
        (market, ext, occupied_families) and the news/manifold clients.

        Returns None with a structured log line in all skip cases.
        Skip reasons (Phase 5 observability):
          SKIP_UNSUPPORTED  — category not supported
          SKIP_NO_SIGNALS   — external data unavailable
          SKIP_PRIOR_EXTREME — prior < 0.08 or > 0.92
          SKIP_FAMILY        — family already has an open/pending position
          SKIP_EDGE_NET      — edge_net < regime_min_edge
          SKIP_CONFIDENCE    — confidence < MIN_CONFIDENCE
        """
        question_lower = market.question.lower()
        category = market.category

        # ── Classify market type ─────────────────────────────────────────────
        is_price_above = any(w in question_lower for w in [
            "above", "over", "exceed", "higher", "atleast", "reach",
        ])
        is_price_below = any(w in question_lower for w in [
            "below", "under", "less than", "lower", "drop",
        ])

        # Short tickers need word boundaries: "Seth" contains "eth",
        # "dissolved" contains "sol", "Canada" contains "ada".  Long
        # unambiguous names (bitcoin, ethereum, …) stay as substrings.
        is_btc = has_token(question_lower, "btc") or "bitcoin" in question_lower
        is_eth = has_token(question_lower, "eth") or "ethereum" in question_lower
        is_sol = has_token(question_lower, "sol") or "solana" in question_lower
        is_xrp = has_token(question_lower, "xrp") or "ripple" in question_lower
        is_doge = has_token(question_lower, "doge") or "dogecoin" in question_lower
        is_altcoin = is_sol or is_xrp or is_doge or any(
            has_token(question_lower, t) for t in ["ltc", "bnb", "ada", "avax"]
        ) or any(
            w in question_lower for w in ["litecoin", "cardano", "avalanche"]
        )
        is_general_crypto = any(
            w in question_lower for w in ["crypto", "market cap", "total market", "altcoin", "defi"]
        )
        is_macro = any(
            w in question_lower for w in [
                "nasdaq", "s&p", "sp500", "inflation", "fed rate", "interest rate", "tariff",
            ]
        )
        is_politics = category == "politics"
        is_tech = category == "tech"
        is_events = category == "events"

        is_any_supported = (
            is_btc or is_eth or is_altcoin or is_general_crypto or is_macro
            or is_politics or is_tech or is_events
        )
        if not is_any_supported:
            log.info(
                "SKIP_UNSUPPORTED  %-50s | cat=%r",
                market.question[:50], category,
            )
            self._record(market, skip_reason="unsupported")
            return None

        if not ext.valid:
            log.info(
                "SKIP_NO_SIGNALS   %-50s | reason=external data unavailable",
                market.question[:50],
            )
            self._record(market, skip_reason="no_signals")
            return None

        # ── Phase 1: prior + prior-extreme filter ────────────────────────────
        prior = max(0.05, min(0.95, market.yes_price))

        if market.yes_price < 0.08:
            self._skip_prior_extreme += 1
            log.info(
                "SKIP_PRIOR_EXTREME %-50s | cat=%-12s | prior=%.3f | reason=prior<0.08",
                market.question[:50], category, market.yes_price,
            )
            self._record(market, skip_reason="prior_extreme", prior_prob=prior)
            return None
        if market.yes_price > 0.92:
            self._skip_prior_extreme += 1
            log.info(
                "SKIP_PRIOR_EXTREME %-50s | cat=%-12s | prior=%.3f | reason=prior>0.92",
                market.question[:50], category, market.yes_price,
            )
            self._record(market, skip_reason="prior_extreme", prior_prob=prior)
            return None

        # ── Phase 2: family deduplication ────────────────────────────────────
        family = market_family_key(market)
        if family in occupied_families:
            self._skip_family += 1
            log.info(
                "SKIP_FAMILY        %-50s | cat=%-12s | family=%s",
                market.question[:50], category, family,
            )
            self._record(market, skip_reason="family", prior_prob=prior, family_key=family)
            return None

        # ── Phase 4: regime min-edge ─────────────────────────────────────────
        days = _days_to_resolution(market.end_date, as_of)
        regime_min = _regime_min_edge(category, days)

        # ── Bayesian probability estimation ──────────────────────────────────
        sources: list[str] = [f"Prior=poly({prior:.3f})"]
        adjustments: list[float] = []

        # Momentum and Fear & Greed only make sense for price markets, where
        # is_price_above gives the adjustment a meaningful sign.  For
        # politics/tech/events there is no above/below notion — is_price_above
        # defaults to False (or flips on accidental wording like "reach"), so
        # applying these signals just injected sign noise.  Skip them entirely;
        # their contributions stay 0.0 → feat_mom_lo / feat_fg_lo = 0.0.
        is_non_price = is_politics or is_tech or is_events

        # Signal 1: price momentum (asset-specific; price markets only)
        _momentum_contribution = 0.0
        if not is_non_price:
            if is_btc:
                momentum = ext.btc_change_24h
                asset_label = "BTC"
            elif is_eth:
                momentum = ext.eth_change_24h
                asset_label = "ETH"
            else:
                momentum = ext.total_market_cap_change
                asset_label = "total mktcap"

            if abs(momentum) > 2:
                momentum_adj = math.tanh(momentum / 20) * 0.15
                _momentum_contribution = momentum_adj if is_price_above else -momentum_adj
                adjustments.append(_momentum_contribution)
                sources.append(f"{asset_label} 24h: {momentum:+.1f}%")

        # Signal 2: Fear & Greed (price markets only)
        _fg_contribution = 0.0
        if not is_non_price:
            fg = ext.fear_greed_index
            if fg > 70:
                fg_adj = 0.06
                sources.append(f"Fear&Greed: {fg} (greed)")
            elif fg < 30:
                fg_adj = -0.06
                sources.append(f"Fear&Greed: {fg} (fear)")
            else:
                fg_adj = (fg - 50) / 50 * 0.04
                sources.append(f"Fear&Greed: {fg} (neutral)")
            _fg_contribution = fg_adj if is_price_above else -fg_adj
            adjustments.append(_fg_contribution)

        # Signal 3: BTC dominance — hurts altcoins when high (price markets only)
        # Like momentum and Fear & Greed above: no demonstrated causality for
        # politics/tech/events, even when they legitimately mention a ticker
        # ("Will the ETH ETF be approved?").  For non-price markets the
        # contribution stays 0.0 → feat_btc_dom_lo = 0.0.
        _btc_dom_contribution = 0.0
        if not is_non_price:
            if (is_eth or is_altcoin or is_general_crypto) and ext.btc_dominance > 55:
                _btc_dom_contribution = -0.03 if is_price_above else 0.03
                adjustments.append(_btc_dom_contribution)
                sources.append(f"BTC dom: {ext.btc_dominance:.1f}% (high → alt pressure)")
            elif (is_eth or is_altcoin or is_general_crypto) and ext.btc_dominance < 45:
                _btc_dom_contribution = 0.03 if is_price_above else -0.03
                adjustments.append(_btc_dom_contribution)
                sources.append(f"BTC dom: {ext.btc_dominance:.1f}% (low → alt season)")

        # Signal 4: GNews sentiment (politics only, budget-gated)
        # Phase 3: caller has pre-sorted markets by gnews_priority() so the
        # highest-value markets reach this block first.
        news_log_adj = 0.0
        news_sentiment = 0.0
        # Replay R0: True when GNews was never consulted for this market this
        # cycle (budget exhausted) — a replay must not read feat_news_lo=0.0 as
        # "there was no news".
        news_budget_skipped = False
        # self._news.enabled gates the whole block: with no GNews API key the
        # client is a no-op, so we must not consume (or report) query budget for
        # it — see NewsClient.enabled.
        if is_politics and self._news is not None and self._news.enabled:
            if self._news_queries_this_cycle < MAX_NEWS_QUERIES_PER_CYCLE:
                self._news_queries_this_cycle += 1
                sentiment = await self._news.get_sentiment(market.question)
                if abs(sentiment) > 0.05:
                    news_sentiment = sentiment
                    news_log_adj = sentiment * NEWS_LOGODDS_WEIGHT
                    sources.append(f"GNews: {sentiment:+.2f}")
            else:
                news_budget_skipped = True
                log.info(
                    "SKIP_GNEWS_PRIORITY %-50s | reason=cycle budget %d reached",
                    market.question[:50], MAX_NEWS_QUERIES_PER_CYCLE,
                )

        # Signal 5: Manifold cross-market probability (politics + tech)
        # Applies a log-odds adjustment proportional to divergence from prior.
        # No query budget — 30 min cache means network cost is paid once per cycle.
        # Now uses ManifoldMatchResult for stricter semantic validation and audit.
        manifold_log_adj = 0.0
        manifold_used = False
        manifold_result: Optional[ManifoldMatchResult] = None
        audit_id: Optional[str] = None

        if ((is_politics or is_tech) and self._manifold is not None
                and (MANIFOLD_AUDIT_ENABLED or MANIFOLD_SIGNAL_ENABLED)):
            # ── Cooldown gate ────────────────────────────────────────────────
            # Skip markets whose Manifold verdict was recently settled to a
            # stable value.  A skip is equivalent to a no-signal: the matcher is
            # NOT called and NO manifold_match_audit row is written, so only real
            # evaluations are recorded.  See _cooldown_for() and the
            # manifold_eval_cooldown table.
            in_cooldown = False
            if self._db is not None and market.id:
                try:
                    cd = await self._db.get_manifold_cooldown(market.id)
                except Exception as exc:
                    log.warning("Failed to read manifold cooldown: %s", exc)
                    cd = None
                if cd is not None and datetime.now(timezone.utc) < cd["retry_after"]:
                    in_cooldown = True
                    log.info(
                        "MANIFOLD_COOLDOWN skip market=%s | last_status=%s "
                        "retry_after=%s | %s",
                        market.id, cd["last_status"],
                        cd["retry_after"].isoformat(), market.question[:50],
                    )

            if not in_cooldown:
                manifold_result = await self._manifold.get_match(market.question)

                # Persist audit record for ALL outcomes (accepted / rejected / no_results).
                # Gated by MANIFOLD_AUDIT_ENABLED so the audit/coverage trail and
                # cooldowns can be kept even while Manifold is observational-only.
                if MANIFOLD_AUDIT_ENABLED and self._db is not None:
                    if not market.id:
                        log.error(
                            "MANIFOLD_AUDIT: market.id is None/empty — skipping audit save | "
                            "question=%r", market.question[:60],
                        )
                    else:
                        audit_id = str(uuid.uuid4())
                        try:
                            await self._db.save_manifold_audit(
                                audit_id=audit_id,
                                poly_market_id=market.id,
                                poly_question=market.question,
                                search_query=manifold_result.search_query,
                                mfld_market_id=manifold_result.market_id,
                                mfld_market_title=manifold_result.market_title,
                                mfld_market_url=manifold_result.market_url,
                                prob_raw=manifold_result.prob_raw,
                                prob_final=manifold_result.prob_final,
                                inverted=manifold_result.inverted,
                                match_score=manifold_result.match_score,
                                match_reason=manifold_result.match_reason,
                                match_status=manifold_result.status,
                                poly_outcome_type=manifold_result.poly_outcome_type,
                                mfld_outcome_type=manifold_result.mfld_outcome_type,
                                matcher_version=MANIFOLD_MATCHER_VERSION,
                            )
                        except Exception as exc:
                            log.warning("Failed to save manifold audit: %s", exc)
                            audit_id = None

                    # Record the cooldown so this market is not re-queried every
                    # cycle.  Written even if the audit save above failed — we
                    # still performed a real evaluation.
                    if market.id:
                        delay, cd_reason = _cooldown_for(manifold_result)
                        try:
                            await self._db.upsert_manifold_cooldown(
                                poly_market_id=market.id,
                                last_status=manifold_result.status,
                                retry_after=datetime.now(timezone.utc) + delay,
                                cooldown_reason=cd_reason,
                            )
                        except Exception as exc:
                            log.warning("Failed to save manifold cooldown: %s", exc)

                # Structured log — both forms for compatibility
                log.info(
                    "MANIFOLD_MATCH poly='%s' mfld='%s' score=%s raw=%s final=%s"
                    " inverted=%s status=%s reason=%s",
                    market.question, manifold_result.market_title,
                    manifold_result.match_score, manifold_result.prob_raw,
                    manifold_result.prob_final, manifold_result.inverted,
                    manifold_result.status, manifold_result.match_reason,
                )
                log.info("MANIFOLD_MATCH", extra={
                    "poly_question": market.question,
                    "mfld_title":    manifold_result.market_title,
                    "score":         manifold_result.match_score,
                    "prob_raw":      manifold_result.prob_raw,
                    "prob_final":    manifold_result.prob_final,
                    "inverted":      manifold_result.inverted,
                    "status":        manifold_result.status,
                    "reason":        manifold_result.match_reason,
                })

                if (MANIFOLD_SIGNAL_ENABLED
                        and manifold_result.status == "accepted"
                        and manifold_result.prob_final is not None):
                    # ACTIVE signal path — only when explicitly enabled.
                    manifold_used = True
                    self._manifold_fetched += 1
                    m_clamped = max(0.05, min(0.95, manifold_result.prob_final))
                    m_log = math.log(m_clamped / (1 - m_clamped))
                    p_log = math.log(prior / (1 - prior))
                    manifold_log_adj = (m_log - p_log) * MANIFOLD_LOGODDS_WEIGHT
                    sources.append(f"Manifold:{manifold_result.prob_final:.2f}")
                elif not MANIFOLD_SIGNAL_ENABLED:
                    # Observational-only: matched/audited but NEVER fed to the edge
                    # model.  manifold_log_adj stays 0.0 → no confidence bump,
                    # feat_mfld_lo=0.0 (cannot be dominant), no trade contribution.
                    log.info(
                        "Manifold: observational_only — signal disabled "
                        "(MANIFOLD_SIGNAL_ENABLED=false) | market=%s status=%s",
                        market.id, manifold_result.status,
                    )
                    sources.append("Manifold: observational_only")

        # Confidence cap: macro/politics/tech signals are weaker proxies
        confidence_cap = 0.65 if (is_macro or is_politics or is_tech or is_events) else 0.90

        # Posterior via log-odds updating
        log_odds_prior = math.log(prior / (1 - prior))
        total_adj = sum(adjustments)
        # raw_final_prob: posterior BEFORE the news guardrail.
        raw_final_prob = _sigmoid(log_odds_prior + total_adj * 2 + news_log_adj + manifold_log_adj)
        raw_final_prob = max(0.05, min(0.95, raw_final_prob))

        # Per-feature log-odds contributions (Phase 6) — computed here (not
        # after the edge gate) because the guardrail below needs them to decide
        # signal materiality.
        # fg / mom / btc_dom: probability-delta × 2 → log-odds.
        # news / mfld: already log-odds (LOGODDS_WEIGHT already applied).
        feat_fg_lo      = _fg_contribution * 2
        feat_mom_lo     = _momentum_contribution * 2
        feat_news_lo    = news_log_adj
        feat_mfld_lo    = manifold_log_adj
        feat_btc_dom_lo = _btc_dom_contribution * 2

        # ── GNews guardrail (catastrophic fuse) ──────────────────────────────
        # When GNews is the ONLY material signal, clamp the posterior to
        # prior ± MAX_NEWS_ONLY_PROB_SHIFT.  estimated_prob (post-guardrail) is
        # what edge/trading uses; raw_final_prob is kept for observability.
        estimated_prob, news_guardrail_applied = apply_news_guardrail(
            prior,
            raw_final_prob,
            feat_news_lo,
            (feat_fg_lo, feat_mom_lo, feat_btc_dom_lo, feat_mfld_lo),
        )

        # ── Phase 1: edge_gross and edge_net ─────────────────────────────────
        raw_edge = estimated_prob - market.yes_price
        direction = "BUY_YES" if raw_edge > 0 else "BUY_NO"
        edge_gross = abs(raw_edge)
        # NOTE: commission/size_usdc = COMMISSION_RATE always (constant fraction).
        edge_net = edge_gross - SPREAD_ESTIMATE - COMMISSION_RATE
        # mid_price falls back to yes_price; live order-book data is a future enhancement
        mid_price = market.yes_price

        # Record for cycle summary — every market that reached edge computation
        self._evaluated_edges.append((edge_gross, edge_net, regime_min))

        # Confidence based on signal agreement
        agreement = sum(1 for a in adjustments if (a > 0) == (total_adj > 0))
        confidence = min(confidence_cap, 0.4 + (agreement / max(len(adjustments), 1)) * 0.5)
        if news_log_adj != 0.0:
            confidence = min(confidence_cap, confidence + 0.10)
        if manifold_log_adj != 0.0:
            confidence = min(confidence_cap, confidence + 0.08)

        feat_str = (
            f"fg_lo={feat_fg_lo:+.4f} mom_lo={feat_mom_lo:+.4f} "
            f"news_lo={feat_news_lo:+.4f} mfld_lo={feat_mfld_lo:+.4f} "
            f"btc_dom_lo={feat_btc_dom_lo:+.4f}"
        )

        # ── Phase 5: structured audit log ────────────────────────────────────
        passed_gross = edge_gross >= regime_min
        passed_net = edge_net >= regime_min
        can_trade = passed_net and confidence >= MIN_CONFIDENCE

        # ── Guardrail decision impact ────────────────────────────────────────
        # True when the un-clamped posterior's edge crossed the regime gate but
        # the clamped one no longer does — i.e. the fuse PREVENTED a trade.
        # Confidence is invariant under the clamp (it depends only on signal
        # agreement), so the edge gate is the only component that can flip.
        guardrail_changed_trade_decision = False
        if news_guardrail_applied:
            raw_edge_net = abs(raw_final_prob - market.yes_price) - TOTAL_COST_RATE
            guardrail_changed_trade_decision = (
                raw_edge_net >= regime_min and edge_net < regime_min
            )

        # ── Guardrail observability — ONLY markets with material news ───────
        # Gated on materiality so the ~145 markets/cycle without news don't
        # flood the logs.  posterior_before_news = everything except GNews.
        news_is_material = abs(feat_news_lo) >= NEWS_MATERIAL_LOGODDS_THRESHOLD
        if news_is_material:
            posterior_before_news = max(0.05, min(0.95, _sigmoid(
                log_odds_prior + total_adj * 2 + manifold_log_adj
            )))
            self._news_shifts.append(estimated_prob - prior)
            if news_guardrail_applied:
                self._news_guardrail_applied += 1
            if guardrail_changed_trade_decision:
                self._news_changed_decisions += 1
            log.info(
                "NEWS_MATERIAL      %-50s | cat=%-12s | family=%-28s | "
                "prior=%.3f | before_news=%.3f | raw=%.3f | final=%.3f | "
                "sent=%+.2f | news_lo=%+.4f | "
                "edge_before_news=%.3f | edge_after_raw=%.3f | edge_after_guardrail=%.3f | "
                "guardrail=%s | changed_decision=%s | max_shift=%.2f",
                market.question[:50], category, family,
                prior, posterior_before_news, raw_final_prob, estimated_prob,
                news_sentiment, feat_news_lo,
                abs(posterior_before_news - market.yes_price),
                abs(raw_final_prob - market.yes_price),
                edge_gross,
                "applied" if news_guardrail_applied else "none",
                str(guardrail_changed_trade_decision).lower(),
                MAX_NEWS_ONLY_PROB_SHIFT,
            )

        # Replay R0: full decision record — same fields for skip and trade paths.
        # skip_reason granularity: "edge_net" when the edge gate failed,
        # "confidence" when only the confidence gate blocked the trade.
        self._record(
            market,
            skip_reason=(
                None if can_trade
                else ("edge_net" if not passed_net else "confidence")
            ),
            family_key=family,
            prior_prob=prior,
            estimated_prob=estimated_prob,
            raw_final_prob=raw_final_prob,
            edge_gross=edge_gross,
            edge_net=edge_net,
            regime_min_edge=regime_min,
            days_to_resolution=days,
            confidence=confidence,
            direction=direction,
            passed_gross=passed_gross,
            passed_net=passed_net,
            news_sentiment=news_sentiment,
            news_budget_skipped=news_budget_skipped,
            guardrail_applied=news_guardrail_applied,
            guardrail_changed_decision=guardrail_changed_trade_decision,
            feat_fg_lo=feat_fg_lo,
            feat_mom_lo=feat_mom_lo,
            feat_news_lo=feat_news_lo,
            feat_mfld_lo=feat_mfld_lo,
            feat_btc_dom_lo=feat_btc_dom_lo,
        )

        if not can_trade:
            # Increment the appropriate edge-net counter
            if edge_net <= 0:
                self._skip_edge_net_nonpositive += 1
            else:
                self._skip_edge_net_below_regime += 1
            skip_parts: list[str] = []
            if not passed_gross:
                skip_parts.append(f"edge_gross={edge_gross:.3f}<{regime_min:.2f}(regime)")
            elif not passed_net:
                skip_parts.append(
                    f"edge_net={edge_net:.3f}<{regime_min:.2f}(regime) "
                    f"[gross={edge_gross:.3f} pass]"
                )
            if confidence < MIN_CONFIDENCE:
                skip_parts.append(f"conf={confidence:.2f}<{MIN_CONFIDENCE}")
            log.info(
                "SKIP_EDGE_NET      %-50s | cat=%-12s | family=%-28s | "
                "prior=%.3f | est=%.3f | gross=%+.3f | net=%+.3f | "
                "regime=%.2f | days=%d | conf=%.2f | %s | signals=%s | %s",
                market.question[:50], category, family,
                prior, estimated_prob, edge_gross, edge_net,
                regime_min, days, confidence, feat_str,
                ", ".join(sources[1:]) or "none",
                " | ".join(skip_parts),
            )
            return None

        # When GNews participated, expose raw vs final and the guardrail verdict
        # (Task 4 of the guardrail spec); otherwise keep the legacy format.
        if news_log_adj != 0.0:
            prob_part = (
                f"Prior=poly({prior:.3f}) → raw={raw_final_prob:.3f} "
                f"→ final={estimated_prob:.3f} | "
                f"GNews sent={news_sentiment:+.2f} | "
                f"guardrail={'applied' if news_guardrail_applied else 'none'} | "
                f"changed_decision={str(guardrail_changed_trade_decision).lower()} | "
                f"max_shift={MAX_NEWS_ONLY_PROB_SHIFT:.2f} | "
            )
        else:
            prob_part = f"Prior=poly({prior:.3f}) → estimate={estimated_prob:.3f} | "
        reasoning = (
            prob_part +
            f"Poly price={market.yes_price:.3f} | "
            f"edge_gross={edge_gross:+.3f} | edge_net={edge_net:+.3f} | "
            f"regime_min={regime_min:.2f} | days={days} | "
            f"family={family} | "
            f"Direction={direction} | "
            f"{feat_str} | "
            f"Signals: {', '.join(sources[1:])}"
        )

        log.info(
            "TRADE              %-50s | cat=%-12s | family=%-28s | "
            "prior=%.3f | est=%.3f | gross=%+.3f | net=%+.3f | "
            "regime=%.2f | days=%d | conf=%.2f | dir=%-8s | %s | signals=%s",
            market.question[:50], category, family,
            prior, estimated_prob, edge_gross, edge_net,
            regime_min, days, confidence, direction, feat_str,
            ", ".join(sources[1:]) or "none",
        )

        self._signal_count += 1
        if manifold_used:
            self._manifold_on_trade += 1
        return TradingSignal(
            market_id=market.id,
            question=market.question,
            polymarket_price=market.yes_price,
            estimated_prob=estimated_prob,
            edge=edge_gross,             # backward compat — same as edge_gross
            confidence=confidence,
            direction=direction,
            reasoning=reasoning,
            sources=sources,
            # Phase 1 new fields
            edge_gross=edge_gross,
            edge_net=edge_net,
            prior_prob=prior,
            final_prob=estimated_prob,
            mid_price=mid_price,
            spread_estimate=SPREAD_ESTIMATE,
            # Phase 2 new fields
            family_key=family,
            # Phase 4 new fields
            regime_min_edge=regime_min,
            # Phase 6 new fields — all in log-odds space
            feat_fg_lo=feat_fg_lo,
            feat_mom_lo=feat_mom_lo,
            feat_news_lo=feat_news_lo,
            feat_mfld_lo=feat_mfld_lo,
            feat_btc_dom_lo=feat_btc_dom_lo,
            # Manifold match audit — propagated through Order → Trade → DB.
            # mfld_audit_id is the hook main.py uses to flip the audit row's
            # used_in_trade=TRUE; suppress it when observational so the trail
            # truthfully shows Manifold drove no trades. The mfld_* fields below
            # stay as observational record (feat_mfld_lo is already 0.0).
            mfld_audit_id=(audit_id if MANIFOLD_SIGNAL_ENABLED else None),
            mfld_market_id=manifold_result.market_id if manifold_result else None,
            mfld_market_title=manifold_result.market_title if manifold_result else None,
            mfld_market_url=manifold_result.market_url if manifold_result else None,
            mfld_prob_raw=manifold_result.prob_raw if manifold_result else None,
            mfld_prob_final=manifold_result.prob_final if manifold_result else None,
            mfld_inverted=manifold_result.inverted if manifold_result else False,
            mfld_match_score=manifold_result.match_score if manifold_result else None,
            mfld_match_reason=manifold_result.match_reason if manifold_result else None,
            mfld_match_status=manifold_result.status if manifold_result else None,
        )


def _sigmoid(x: float) -> float:
    return 1 / (1 + math.exp(-x))
