chemavxandClaude Fable 5
0ac48ba7f8
Re-executes BayesianStrategy.evaluate() over the R0 archive and stores results in replay_runs/replay_decisions, tagged with git sha + a hash of the strategy constants (same hash vs archive = determinism check, different hash = counterfactual run). - bayesian.py: optional as_of param on evaluate()/_days_to_resolution() (clock injection; default None = wall clock, prod behavior unchanged — the only touch to frozen code, purely additive) - bot/replay.py: replay engine + CLI (python -m bot.replay --from --to); ReplayNews feeds archived sentiment back (GNews never called, per-cycle budget bypassed — archived sentiment already encodes it); manifold/db not wired (observational-only in prod); recorded-vs-replayed compare at 1e-9 tolerance - schema.sql: replay_runs + replay_decisions (+ indexes), idempotent - db.py: 6 replay accessors/writers - tests: 19 new round-trip fidelity tests (104 total green) Validated against a real prod cycle (2026-07-02T14:03:15Z, 46 markets, 4 skip paths incl. the Georgia confidence record): 46/46 matched, max float delta 0.0. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
368 lines
15 KiB
Python
368 lines
15 KiB
Python
"""
|
|
Tests for the Replay R1 replay core (bot/replay.py) and the as_of clock
|
|
injection in BayesianStrategy.evaluate().
|
|
|
|
The central contract is round-trip fidelity: a decision recorded by R0 and
|
|
replayed through replay_cycle() with the same strategy constants must match
|
|
field-for-field (matched=True, mismatch_field=None). Each round-trip test
|
|
produces the "archive" by running the real evaluate() with FakeNews, then
|
|
replays the drained record as if it had been read back from the signals table.
|
|
"""
|
|
import asyncio
|
|
from datetime import datetime, timedelta, timezone
|
|
|
|
import pytest
|
|
|
|
import bot.strategy.bayesian as bayesian
|
|
from bot.data.polymarket import Market, market_family_key
|
|
from bot.strategy.bayesian import BayesianStrategy, _days_to_resolution
|
|
from bot.replay import (
|
|
ReplayNews,
|
|
build_ext,
|
|
build_market,
|
|
replay_cycle,
|
|
strategy_config_hash,
|
|
)
|
|
|
|
from tests.test_news_guardrail import FakeNews, _sentiment_for
|
|
|
|
|
|
def _end_date(days_ahead: int = 20) -> str:
|
|
dt = datetime.now(timezone.utc) + timedelta(days=days_ahead)
|
|
return dt.strftime("%Y-%m-%dT00:00:00Z")
|
|
|
|
|
|
def _make_market(
|
|
yes_price: float,
|
|
question: str = "Will John Smith win the election?",
|
|
category: str = "politics",
|
|
market_id: str = "mkt-replay-1",
|
|
end_date: str = None,
|
|
) -> Market:
|
|
return Market(
|
|
id=market_id,
|
|
condition_id="cond-replay-1",
|
|
question=question,
|
|
yes_token_id="yes-tok",
|
|
no_token_id="no-tok",
|
|
yes_price=yes_price,
|
|
no_price=1.0 - yes_price,
|
|
volume_24h=50_000.0,
|
|
end_date=end_date if end_date is not None else _end_date(),
|
|
active=True,
|
|
category=category,
|
|
)
|
|
|
|
|
|
def _snapshot(valid: bool = True) -> dict:
|
|
"""An ext_snapshots row as read back from the DB."""
|
|
return {
|
|
"btc_price": 100_000.0,
|
|
"btc_change_24h": 0.0,
|
|
"eth_price": 4_000.0,
|
|
"eth_change_24h": 0.0,
|
|
"btc_dominance": 50.0,
|
|
"fear_greed_index": 50,
|
|
"fear_greed_label": "neutral",
|
|
"total_market_cap_change": 0.0,
|
|
"valid": valid,
|
|
}
|
|
|
|
|
|
def _market_row(market: Market) -> dict:
|
|
"""A markets-table row for the given Market."""
|
|
return {
|
|
"id": market.id,
|
|
"condition_id": market.condition_id,
|
|
"question": market.question,
|
|
"category": market.category,
|
|
"end_date": market.end_date,
|
|
}
|
|
|
|
|
|
def _record_with_live_evaluate(
|
|
market: Market,
|
|
news=None,
|
|
families: set = frozenset(),
|
|
) -> dict:
|
|
"""Run the real evaluate() and return the R0 record it produced —
|
|
the same dict save_signal_records() would have archived."""
|
|
strategy = BayesianStrategy(news=news, manifold=None, db=None)
|
|
asyncio.run(strategy.evaluate(market, build_ext(_snapshot()), set(families)))
|
|
return strategy.drain_cycle_records()[0]
|
|
|
|
|
|
def _replay_one(record: dict, market: Market, snapshot: dict = None) -> dict:
|
|
cycle_ts = datetime.now(timezone.utc)
|
|
decisions = asyncio.run(replay_cycle(
|
|
cycle_ts,
|
|
snapshot or _snapshot(),
|
|
[record],
|
|
{market.id: _market_row(market)},
|
|
))
|
|
assert len(decisions) == 1
|
|
return decisions[0]
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Clock injection
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
def test_days_to_resolution_uses_injected_clock():
|
|
end = "2026-08-01T00:00:00Z"
|
|
as_of = datetime(2026, 7, 2, 12, 0, tzinfo=timezone.utc)
|
|
assert _days_to_resolution(end, as_of) == 29
|
|
assert _days_to_resolution(end, as_of - timedelta(days=60)) == 89
|
|
|
|
|
|
def test_default_clock_is_wall_clock():
|
|
end = _end_date(days_ahead=40)
|
|
assert _days_to_resolution(end) == _days_to_resolution(
|
|
end, datetime.now(timezone.utc)
|
|
)
|
|
|
|
|
|
def test_as_of_changes_regime_threshold():
|
|
"""Same politics market: <30 d out → regime 0.08; replayed from 60 d
|
|
earlier → regime 0.12. The clock, not the wall time, must decide."""
|
|
market = _make_market(0.470)
|
|
sentiment = _sentiment_for(0.470, 0.601)
|
|
|
|
def _regime(as_of):
|
|
strategy = BayesianStrategy(news=FakeNews(sentiment), manifold=None, db=None)
|
|
asyncio.run(strategy.evaluate(
|
|
market, build_ext(_snapshot()), set(), as_of=as_of,
|
|
))
|
|
return strategy.drain_cycle_records()[0]["regime_min_edge"]
|
|
|
|
now = datetime.now(timezone.utc)
|
|
assert _regime(now) == pytest.approx(0.08)
|
|
assert _regime(now - timedelta(days=60)) == pytest.approx(0.12)
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Round-trip fidelity: record with live evaluate(), replay, expect match
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
def test_roundtrip_confidence_skip():
|
|
"""Georgia signature: edge passes, confidence blocks — full-field match."""
|
|
sentiment = _sentiment_for(0.470, 0.601)
|
|
market = _make_market(0.470)
|
|
record = _record_with_live_evaluate(market, news=FakeNews(sentiment))
|
|
assert record["skip_reason"] == "confidence"
|
|
|
|
decision = _replay_one(record, market)
|
|
assert decision["matched"] is True
|
|
assert decision["mismatch_field"] is None
|
|
assert decision["skip_reason"] == "confidence"
|
|
assert decision["estimated_prob"] == pytest.approx(record["estimated_prob"])
|
|
assert decision["edge_net"] == pytest.approx(record["edge_net"])
|
|
assert decision["confidence"] == pytest.approx(record["confidence"])
|
|
assert decision["direction"] == record["direction"]
|
|
assert decision["would_trade"] is False
|
|
|
|
|
|
def test_roundtrip_edge_net_skip():
|
|
market = _make_market(0.50)
|
|
record = _record_with_live_evaluate(market)
|
|
assert record["skip_reason"] == "edge_net"
|
|
|
|
decision = _replay_one(record, market)
|
|
assert decision["matched"] is True
|
|
assert decision["would_trade"] is False
|
|
|
|
|
|
def test_roundtrip_guardrail_clamp():
|
|
"""Clamped posterior must reproduce exactly (raw != final in archive)."""
|
|
market = _make_market(0.845)
|
|
record = _record_with_live_evaluate(
|
|
market, news=FakeNews(_sentiment_for(0.845, 0.431))
|
|
)
|
|
assert record["guardrail_applied"] is True
|
|
|
|
decision = _replay_one(record, market)
|
|
assert decision["matched"] is True
|
|
assert decision["raw_final_prob"] == pytest.approx(record["raw_final_prob"])
|
|
assert decision["estimated_prob"] == pytest.approx(record["estimated_prob"])
|
|
|
|
|
|
def test_roundtrip_prior_extreme():
|
|
market = _make_market(0.03)
|
|
record = _record_with_live_evaluate(market)
|
|
assert record["skip_reason"] == "prior_extreme"
|
|
|
|
decision = _replay_one(record, market)
|
|
assert decision["matched"] is True
|
|
assert decision["skip_reason"] == "prior_extreme"
|
|
|
|
|
|
def test_roundtrip_family_skip():
|
|
"""Family-skipped rows replay with their own family injected as occupied."""
|
|
market = _make_market(0.50)
|
|
record = _record_with_live_evaluate(
|
|
market, families={market_family_key(market)}
|
|
)
|
|
assert record["skip_reason"] == "family"
|
|
|
|
decision = _replay_one(record, market)
|
|
assert decision["matched"] is True
|
|
assert decision["skip_reason"] == "family"
|
|
|
|
|
|
def test_roundtrip_unsupported():
|
|
market = _make_market(0.50, question="Will it rain tomorrow?", category="")
|
|
record = _record_with_live_evaluate(market)
|
|
assert record["skip_reason"] == "unsupported"
|
|
|
|
decision = _replay_one(record, market)
|
|
assert decision["matched"] is True
|
|
|
|
|
|
def test_roundtrip_no_signals():
|
|
"""ext.valid=False archived → replay rebuilds the invalid snapshot."""
|
|
market = _make_market(0.50)
|
|
strategy = BayesianStrategy(news=None, manifold=None, db=None)
|
|
asyncio.run(strategy.evaluate(market, build_ext(_snapshot(valid=False)), set()))
|
|
record = strategy.drain_cycle_records()[0]
|
|
assert record["skip_reason"] == "no_signals"
|
|
|
|
decision = _replay_one(record, market, snapshot=_snapshot(valid=False))
|
|
assert decision["matched"] is True
|
|
|
|
|
|
def test_roundtrip_trade_path(monkeypatch):
|
|
"""skip_reason=None (tradeable) round-trips with would_trade=True.
|
|
Politics can't clear MIN_CONFIDENCE=0.55 (the known ceiling), so the
|
|
gate is lowered for this test only — both record and replay see the
|
|
same constant, which is exactly the config_hash contract."""
|
|
monkeypatch.setattr(bayesian, "MIN_CONFIDENCE", 0.45)
|
|
sentiment = _sentiment_for(0.470, 0.601)
|
|
market = _make_market(0.470)
|
|
record = _record_with_live_evaluate(market, news=FakeNews(sentiment))
|
|
assert record["skip_reason"] is None
|
|
|
|
decision = _replay_one(record, market)
|
|
assert decision["matched"] is True
|
|
assert decision["skip_reason"] is None
|
|
assert decision["would_trade"] is True
|
|
assert decision["direction"] == "BUY_YES"
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Replay-specific semantics
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
def test_budget_skipped_row_replays_without_news():
|
|
"""A budget-skipped archive row (sentiment 0.0) must replay to the same
|
|
no-news decision — and never consume a replay-side budget."""
|
|
market = _make_market(0.50)
|
|
strategy = BayesianStrategy(news=FakeNews(0.9), manifold=None, db=None)
|
|
strategy._news_queries_this_cycle = bayesian.MAX_NEWS_QUERIES_PER_CYCLE
|
|
asyncio.run(strategy.evaluate(market, build_ext(_snapshot()), set()))
|
|
record = strategy.drain_cycle_records()[0]
|
|
assert record["news_budget_skipped"] is True
|
|
assert record["news_sentiment"] == 0.0
|
|
|
|
decision = _replay_one(record, market)
|
|
assert decision["matched"] is True
|
|
assert decision["estimated_prob"] == pytest.approx(record["estimated_prob"])
|
|
|
|
|
|
def test_reentry_guard_row_is_recalibrated_not_compared():
|
|
"""record_skip() rows carry no decision fields; the replay re-evaluates
|
|
them (calibration data) but marks them non-comparable."""
|
|
market = _make_market(0.50)
|
|
strategy = BayesianStrategy(news=None, manifold=None, db=None)
|
|
strategy.record_skip(market, "reentry_guard")
|
|
record = strategy.drain_cycle_records()[0]
|
|
|
|
decision = _replay_one(record, market)
|
|
assert decision["matched"] is None
|
|
assert decision["recorded_skip_reason"] == "reentry_guard"
|
|
# Re-evaluated on its merits: a full decision despite the recorded skip
|
|
assert decision["estimated_prob"] is not None
|
|
assert decision["skip_reason"] == "edge_net"
|
|
|
|
|
|
def test_missing_market_row_flagged_not_crashed():
|
|
market = _make_market(0.50)
|
|
record = _record_with_live_evaluate(market)
|
|
|
|
decisions = asyncio.run(replay_cycle(
|
|
datetime.now(timezone.utc), _snapshot(), [record], {},
|
|
))
|
|
assert decisions[0]["matched"] is False
|
|
assert decisions[0]["mismatch_field"] == "market_missing"
|
|
|
|
|
|
def test_mismatch_detected_when_config_differs(monkeypatch):
|
|
"""Counterfactual sanity: replaying under a different guardrail band
|
|
must produce matched=False with the diverging field named."""
|
|
market = _make_market(0.845)
|
|
record = _record_with_live_evaluate(
|
|
market, news=FakeNews(_sentiment_for(0.845, 0.431))
|
|
)
|
|
assert record["guardrail_applied"] is True
|
|
|
|
monkeypatch.setattr(bayesian, "MAX_NEWS_ONLY_PROB_SHIFT", 0.10)
|
|
decision = _replay_one(record, market)
|
|
assert decision["matched"] is False
|
|
# Tighter clamp (prior 0.845 ± 0.10 → est 0.745): edge_net drops from
|
|
# 0.21 to 0.06 < regime 0.08, so the skip flips confidence → edge_net
|
|
# and skip_reason is the first field _compare() sees diverge.
|
|
assert decision["mismatch_field"] == "skip_reason"
|
|
assert decision["skip_reason"] == "edge_net"
|
|
|
|
|
|
def test_multi_row_cycle_preserves_order_and_isolation():
|
|
"""Rows replay independently within a cycle: a family skip and a full
|
|
evaluation with different sentiments don't bleed into each other."""
|
|
m1 = _make_market(0.470, market_id="m1")
|
|
m2 = _make_market(
|
|
0.50, market_id="m2",
|
|
question="Will Jane Doe win the Georgia Senate race?",
|
|
)
|
|
r1 = _record_with_live_evaluate(m1, news=FakeNews(_sentiment_for(0.470, 0.601)))
|
|
r2 = _record_with_live_evaluate(m2) # no news → edge_net skip
|
|
|
|
decisions = asyncio.run(replay_cycle(
|
|
datetime.now(timezone.utc),
|
|
_snapshot(),
|
|
[r1, r2],
|
|
{"m1": _market_row(m1), "m2": _market_row(m2)},
|
|
))
|
|
assert [d["market_id"] for d in decisions] == ["m1", "m2"]
|
|
assert all(d["matched"] is True for d in decisions)
|
|
assert decisions[0]["skip_reason"] == "confidence"
|
|
assert decisions[1]["skip_reason"] == "edge_net"
|
|
|
|
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
# Run tagging
|
|
# ─────────────────────────────────────────────────────────────────────────────
|
|
|
|
def test_config_hash_stable_and_sensitive(monkeypatch):
|
|
h1 = strategy_config_hash()
|
|
assert strategy_config_hash() == h1
|
|
monkeypatch.setattr(bayesian, "MAX_NEWS_ONLY_PROB_SHIFT", 0.10)
|
|
assert strategy_config_hash() != h1
|
|
|
|
|
|
def test_replay_news_returns_current_sentiment():
|
|
news = ReplayNews()
|
|
assert asyncio.run(news.get_sentiment("q")) == 0.0
|
|
news.sentiment = -0.42
|
|
assert asyncio.run(news.get_sentiment("q")) == -0.42
|
|
|
|
|
|
def test_build_market_reconstruction():
|
|
market = _make_market(0.37)
|
|
record = _record_with_live_evaluate(market)
|
|
rebuilt = build_market(_market_row(market), record)
|
|
assert rebuilt.id == market.id
|
|
assert rebuilt.yes_price == pytest.approx(0.37)
|
|
assert rebuilt.volume_24h == pytest.approx(market.volume_24h)
|
|
assert rebuilt.end_date == market.end_date
|
|
assert rebuilt.category == "politics"
|
|
assert market_family_key(rebuilt) == market_family_key(market)
|