feat(replay): R1 replay core — clock injection + replay of archived cycles

Re-executes BayesianStrategy.evaluate() over the R0 archive and stores results in replay_runs/replay_decisions, tagged with git sha + a hash of the strategy constants (same hash vs archive = determinism check, different hash = counterfactual run). - bayesian.py: optional as_of param on evaluate()/_days_to_resolution() (clock injection; default None = wall clock, prod behavior unchanged — the only touch to frozen code, purely additive) - bot/replay.py: replay engine + CLI (python -m bot.replay --from --to); ReplayNews feeds archived sentiment back (GNews never called, per-cycle budget bypassed — archived sentiment already encodes it); manifold/db not wired (observational-only in prod); recorded-vs-replayed compare at 1e-9 tolerance - schema.sql: replay_runs + replay_decisions (+ indexes), idempotent - db.py: 6 replay accessors/writers - tests: 19 new round-trip fidelity tests (104 total green) Validated against a real prod cycle (2026-07-02T14:03:15Z, 46 markets, 4 skip paths incl. the Georgia confidence record): 46/46 matched, max float delta 0.0. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-02 14:05:25 +00:00
parent eb4f67414a
commit 0ac48ba7f8
5 changed files with 919 additions and 4 deletions
@@ -0,0 +1,367 @@
+"""
+Tests for the Replay R1 replay core (bot/replay.py) and the as_of clock
+injection in BayesianStrategy.evaluate().
+
+The central contract is round-trip fidelity: a decision recorded by R0 and
+replayed through replay_cycle() with the same strategy constants must match
+field-for-field (matched=True, mismatch_field=None).  Each round-trip test
+produces the "archive" by running the real evaluate() with FakeNews, then
+replays the drained record as if it had been read back from the signals table.
+"""
+import asyncio
+from datetime import datetime, timedelta, timezone
+
+import pytest
+
+import bot.strategy.bayesian as bayesian
+from bot.data.polymarket import Market, market_family_key
+from bot.strategy.bayesian import BayesianStrategy, _days_to_resolution
+from bot.replay import (
+    ReplayNews,
+    build_ext,
+    build_market,
+    replay_cycle,
+    strategy_config_hash,
+)
+
+from tests.test_news_guardrail import FakeNews, _sentiment_for
+
+
+def _end_date(days_ahead: int = 20) -> str:
+    dt = datetime.now(timezone.utc) + timedelta(days=days_ahead)
+    return dt.strftime("%Y-%m-%dT00:00:00Z")
+
+
+def _make_market(
+    yes_price: float,
+    question: str = "Will John Smith win the election?",
+    category: str = "politics",
+    market_id: str = "mkt-replay-1",
+    end_date: str = None,
+) -> Market:
+    return Market(
+        id=market_id,
+        condition_id="cond-replay-1",
+        question=question,
+        yes_token_id="yes-tok",
+        no_token_id="no-tok",
+        yes_price=yes_price,
+        no_price=1.0 - yes_price,
+        volume_24h=50_000.0,
+        end_date=end_date if end_date is not None else _end_date(),
+        active=True,
+        category=category,
+    )
+
+
+def _snapshot(valid: bool = True) -> dict:
+    """An ext_snapshots row as read back from the DB."""
+    return {
+        "btc_price": 100_000.0,
+        "btc_change_24h": 0.0,
+        "eth_price": 4_000.0,
+        "eth_change_24h": 0.0,
+        "btc_dominance": 50.0,
+        "fear_greed_index": 50,
+        "fear_greed_label": "neutral",
+        "total_market_cap_change": 0.0,
+        "valid": valid,
+    }
+
+
+def _market_row(market: Market) -> dict:
+    """A markets-table row for the given Market."""
+    return {
+        "id": market.id,
+        "condition_id": market.condition_id,
+        "question": market.question,
+        "category": market.category,
+        "end_date": market.end_date,
+    }
+
+
+def _record_with_live_evaluate(
+    market: Market,
+    news=None,
+    families: set = frozenset(),
+) -> dict:
+    """Run the real evaluate() and return the R0 record it produced —
+    the same dict save_signal_records() would have archived."""
+    strategy = BayesianStrategy(news=news, manifold=None, db=None)
+    asyncio.run(strategy.evaluate(market, build_ext(_snapshot()), set(families)))
+    return strategy.drain_cycle_records()[0]
+
+
+def _replay_one(record: dict, market: Market, snapshot: dict = None) -> dict:
+    cycle_ts = datetime.now(timezone.utc)
+    decisions = asyncio.run(replay_cycle(
+        cycle_ts,
+        snapshot or _snapshot(),
+        [record],
+        {market.id: _market_row(market)},
+    ))
+    assert len(decisions) == 1
+    return decisions[0]
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Clock injection
+# ─────────────────────────────────────────────────────────────────────────────
+
+def test_days_to_resolution_uses_injected_clock():
+    end = "2026-08-01T00:00:00Z"
+    as_of = datetime(2026, 7, 2, 12, 0, tzinfo=timezone.utc)
+    assert _days_to_resolution(end, as_of) == 29
+    assert _days_to_resolution(end, as_of - timedelta(days=60)) == 89
+
+
+def test_default_clock_is_wall_clock():
+    end = _end_date(days_ahead=40)
+    assert _days_to_resolution(end) == _days_to_resolution(
+        end, datetime.now(timezone.utc)
+    )
+
+
+def test_as_of_changes_regime_threshold():
+    """Same politics market: <30 d out → regime 0.08; replayed from 60 d
+    earlier → regime 0.12.  The clock, not the wall time, must decide."""
+    market = _make_market(0.470)
+    sentiment = _sentiment_for(0.470, 0.601)
+
+    def _regime(as_of):
+        strategy = BayesianStrategy(news=FakeNews(sentiment), manifold=None, db=None)
+        asyncio.run(strategy.evaluate(
+            market, build_ext(_snapshot()), set(), as_of=as_of,
+        ))
+        return strategy.drain_cycle_records()[0]["regime_min_edge"]
+
+    now = datetime.now(timezone.utc)
+    assert _regime(now) == pytest.approx(0.08)
+    assert _regime(now - timedelta(days=60)) == pytest.approx(0.12)
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Round-trip fidelity: record with live evaluate(), replay, expect match
+# ─────────────────────────────────────────────────────────────────────────────
+
+def test_roundtrip_confidence_skip():
+    """Georgia signature: edge passes, confidence blocks — full-field match."""
+    sentiment = _sentiment_for(0.470, 0.601)
+    market = _make_market(0.470)
+    record = _record_with_live_evaluate(market, news=FakeNews(sentiment))
+    assert record["skip_reason"] == "confidence"
+
+    decision = _replay_one(record, market)
+    assert decision["matched"] is True
+    assert decision["mismatch_field"] is None
+    assert decision["skip_reason"] == "confidence"
+    assert decision["estimated_prob"] == pytest.approx(record["estimated_prob"])
+    assert decision["edge_net"] == pytest.approx(record["edge_net"])
+    assert decision["confidence"] == pytest.approx(record["confidence"])
+    assert decision["direction"] == record["direction"]
+    assert decision["would_trade"] is False
+
+
+def test_roundtrip_edge_net_skip():
+    market = _make_market(0.50)
+    record = _record_with_live_evaluate(market)
+    assert record["skip_reason"] == "edge_net"
+
+    decision = _replay_one(record, market)
+    assert decision["matched"] is True
+    assert decision["would_trade"] is False
+
+
+def test_roundtrip_guardrail_clamp():
+    """Clamped posterior must reproduce exactly (raw != final in archive)."""
+    market = _make_market(0.845)
+    record = _record_with_live_evaluate(
+        market, news=FakeNews(_sentiment_for(0.845, 0.431))
+    )
+    assert record["guardrail_applied"] is True
+
+    decision = _replay_one(record, market)
+    assert decision["matched"] is True
+    assert decision["raw_final_prob"] == pytest.approx(record["raw_final_prob"])
+    assert decision["estimated_prob"] == pytest.approx(record["estimated_prob"])
+
+
+def test_roundtrip_prior_extreme():
+    market = _make_market(0.03)
+    record = _record_with_live_evaluate(market)
+    assert record["skip_reason"] == "prior_extreme"
+
+    decision = _replay_one(record, market)
+    assert decision["matched"] is True
+    assert decision["skip_reason"] == "prior_extreme"
+
+
+def test_roundtrip_family_skip():
+    """Family-skipped rows replay with their own family injected as occupied."""
+    market = _make_market(0.50)
+    record = _record_with_live_evaluate(
+        market, families={market_family_key(market)}
+    )
+    assert record["skip_reason"] == "family"
+
+    decision = _replay_one(record, market)
+    assert decision["matched"] is True
+    assert decision["skip_reason"] == "family"
+
+
+def test_roundtrip_unsupported():
+    market = _make_market(0.50, question="Will it rain tomorrow?", category="")
+    record = _record_with_live_evaluate(market)
+    assert record["skip_reason"] == "unsupported"
+
+    decision = _replay_one(record, market)
+    assert decision["matched"] is True
+
+
+def test_roundtrip_no_signals():
+    """ext.valid=False archived → replay rebuilds the invalid snapshot."""
+    market = _make_market(0.50)
+    strategy = BayesianStrategy(news=None, manifold=None, db=None)
+    asyncio.run(strategy.evaluate(market, build_ext(_snapshot(valid=False)), set()))
+    record = strategy.drain_cycle_records()[0]
+    assert record["skip_reason"] == "no_signals"
+
+    decision = _replay_one(record, market, snapshot=_snapshot(valid=False))
+    assert decision["matched"] is True
+
+
+def test_roundtrip_trade_path(monkeypatch):
+    """skip_reason=None (tradeable) round-trips with would_trade=True.
+    Politics can't clear MIN_CONFIDENCE=0.55 (the known ceiling), so the
+    gate is lowered for this test only — both record and replay see the
+    same constant, which is exactly the config_hash contract."""
+    monkeypatch.setattr(bayesian, "MIN_CONFIDENCE", 0.45)
+    sentiment = _sentiment_for(0.470, 0.601)
+    market = _make_market(0.470)
+    record = _record_with_live_evaluate(market, news=FakeNews(sentiment))
+    assert record["skip_reason"] is None
+
+    decision = _replay_one(record, market)
+    assert decision["matched"] is True
+    assert decision["skip_reason"] is None
+    assert decision["would_trade"] is True
+    assert decision["direction"] == "BUY_YES"
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Replay-specific semantics
+# ─────────────────────────────────────────────────────────────────────────────
+
+def test_budget_skipped_row_replays_without_news():
+    """A budget-skipped archive row (sentiment 0.0) must replay to the same
+    no-news decision — and never consume a replay-side budget."""
+    market = _make_market(0.50)
+    strategy = BayesianStrategy(news=FakeNews(0.9), manifold=None, db=None)
+    strategy._news_queries_this_cycle = bayesian.MAX_NEWS_QUERIES_PER_CYCLE
+    asyncio.run(strategy.evaluate(market, build_ext(_snapshot()), set()))
+    record = strategy.drain_cycle_records()[0]
+    assert record["news_budget_skipped"] is True
+    assert record["news_sentiment"] == 0.0
+
+    decision = _replay_one(record, market)
+    assert decision["matched"] is True
+    assert decision["estimated_prob"] == pytest.approx(record["estimated_prob"])
+
+
+def test_reentry_guard_row_is_recalibrated_not_compared():
+    """record_skip() rows carry no decision fields; the replay re-evaluates
+    them (calibration data) but marks them non-comparable."""
+    market = _make_market(0.50)
+    strategy = BayesianStrategy(news=None, manifold=None, db=None)
+    strategy.record_skip(market, "reentry_guard")
+    record = strategy.drain_cycle_records()[0]
+
+    decision = _replay_one(record, market)
+    assert decision["matched"] is None
+    assert decision["recorded_skip_reason"] == "reentry_guard"
+    # Re-evaluated on its merits: a full decision despite the recorded skip
+    assert decision["estimated_prob"] is not None
+    assert decision["skip_reason"] == "edge_net"
+
+
+def test_missing_market_row_flagged_not_crashed():
+    market = _make_market(0.50)
+    record = _record_with_live_evaluate(market)
+
+    decisions = asyncio.run(replay_cycle(
+        datetime.now(timezone.utc), _snapshot(), [record], {},
+    ))
+    assert decisions[0]["matched"] is False
+    assert decisions[0]["mismatch_field"] == "market_missing"
+
+
+def test_mismatch_detected_when_config_differs(monkeypatch):
+    """Counterfactual sanity: replaying under a different guardrail band
+    must produce matched=False with the diverging field named."""
+    market = _make_market(0.845)
+    record = _record_with_live_evaluate(
+        market, news=FakeNews(_sentiment_for(0.845, 0.431))
+    )
+    assert record["guardrail_applied"] is True
+
+    monkeypatch.setattr(bayesian, "MAX_NEWS_ONLY_PROB_SHIFT", 0.10)
+    decision = _replay_one(record, market)
+    assert decision["matched"] is False
+    # Tighter clamp (prior 0.845 ± 0.10 → est 0.745): edge_net drops from
+    # 0.21 to 0.06 < regime 0.08, so the skip flips confidence → edge_net
+    # and skip_reason is the first field _compare() sees diverge.
+    assert decision["mismatch_field"] == "skip_reason"
+    assert decision["skip_reason"] == "edge_net"
+
+
+def test_multi_row_cycle_preserves_order_and_isolation():
+    """Rows replay independently within a cycle: a family skip and a full
+    evaluation with different sentiments don't bleed into each other."""
+    m1 = _make_market(0.470, market_id="m1")
+    m2 = _make_market(
+        0.50, market_id="m2",
+        question="Will Jane Doe win the Georgia Senate race?",
+    )
+    r1 = _record_with_live_evaluate(m1, news=FakeNews(_sentiment_for(0.470, 0.601)))
+    r2 = _record_with_live_evaluate(m2)  # no news → edge_net skip
+
+    decisions = asyncio.run(replay_cycle(
+        datetime.now(timezone.utc),
+        _snapshot(),
+        [r1, r2],
+        {"m1": _market_row(m1), "m2": _market_row(m2)},
+    ))
+    assert [d["market_id"] for d in decisions] == ["m1", "m2"]
+    assert all(d["matched"] is True for d in decisions)
+    assert decisions[0]["skip_reason"] == "confidence"
+    assert decisions[1]["skip_reason"] == "edge_net"
+
+
+# ─────────────────────────────────────────────────────────────────────────────
+# Run tagging
+# ─────────────────────────────────────────────────────────────────────────────
+
+def test_config_hash_stable_and_sensitive(monkeypatch):
+    h1 = strategy_config_hash()
+    assert strategy_config_hash() == h1
+    monkeypatch.setattr(bayesian, "MAX_NEWS_ONLY_PROB_SHIFT", 0.10)
+    assert strategy_config_hash() != h1
+
+
+def test_replay_news_returns_current_sentiment():
+    news = ReplayNews()
+    assert asyncio.run(news.get_sentiment("q")) == 0.0
+    news.sentiment = -0.42
+    assert asyncio.run(news.get_sentiment("q")) == -0.42
+
+
+def test_build_market_reconstruction():
+    market = _make_market(0.37)
+    record = _record_with_live_evaluate(market)
+    rebuilt = build_market(_market_row(market), record)
+    assert rebuilt.id == market.id
+    assert rebuilt.yes_price == pytest.approx(0.37)
+    assert rebuilt.volume_24h == pytest.approx(market.volume_24h)
+    assert rebuilt.end_date == market.end_date
+    assert rebuilt.category == "politics"
+    assert market_family_key(rebuilt) == market_family_key(market)