"""Replay R2 tests — outcome fetching and calibration scoring.""" import asyncio import math import pytest from bot.data.polymarket import MarketResolution from bot.outcomes import ( LOGLOSS_EPS, compute_calibration, fetch_outcomes, print_report, ) from datetime import datetime, timezone class FakePoly: """get_market_resolution stand-in driven by a dict of canned responses.""" def __init__(self, responses: dict): self.responses = responses self.calls: list[str] = [] async def get_market_resolution(self, market_id: str): self.calls.append(market_id) return self.responses.get(market_id) RESOLVED_AT = datetime(2026, 7, 1, 12, 0, tzinfo=timezone.utc) def _row(market_id="m1", category="politics", est=0.6, prior=0.5, outcome=1.0): return { "market_id": market_id, "category": category, "estimated_prob": est, "prior_prob": prior, "outcome": outcome, } # ── fetch_outcomes ─────────────────────────────────────────────────────────── def test_fetch_keeps_only_definitive_resolutions(): poly = FakePoly({ "yes": MarketResolution(resolved=True, resolution=1.0, resolved_at=RESOLVED_AT), "no": MarketResolution(resolved=True, resolution=0.0, resolved_at=None), "open": MarketResolution(resolved=False), "disputed": MarketResolution(resolved=False), "apierror": None, # get_market_resolution returns None on HTTP errors }) out = asyncio.run( fetch_outcomes(poly, ["yes", "no", "open", "disputed", "apierror"]) ) assert poly.calls == ["yes", "no", "open", "disputed", "apierror"] assert out == [ {"market_id": "yes", "outcome": 1.0, "resolved_at": RESOLVED_AT}, {"market_id": "no", "outcome": 0.0, "resolved_at": None}, ] def test_fetch_empty_list_is_noop(): poly = FakePoly({}) assert asyncio.run(fetch_outcomes(poly, [])) == [] assert poly.calls == [] # ── compute_calibration ────────────────────────────────────────────────────── def test_no_rows_returns_none(): assert compute_calibration([]) is None def test_single_row_known_values(): m = compute_calibration([_row(est=0.8, prior=0.6, outcome=1.0)]) assert m["n_evaluations"] == 1 assert m["n_markets"] == 1 assert m["brier_model"] == pytest.approx((0.8 - 1.0) ** 2) assert m["brier_prior"] == pytest.approx((0.6 - 1.0) ** 2) assert m["logloss_model"] == pytest.approx(-math.log(0.8)) assert m["logloss_prior"] == pytest.approx(-math.log(0.6)) # one market: macro == micro assert m["brier_model_macro"] == pytest.approx(m["brier_model"]) assert m["brier_prior_macro"] == pytest.approx(m["brier_prior"]) def test_logloss_no_outcome_branch(): m = compute_calibration([_row(est=0.2, prior=0.7, outcome=0.0)]) assert m["logloss_model"] == pytest.approx(-math.log(0.8)) assert m["logloss_prior"] == pytest.approx(-math.log(0.3)) def test_logloss_clipping_keeps_hard_miss_finite(): # A hard 1.0 estimate on a NO outcome must not produce inf. m = compute_calibration([_row(est=1.0, prior=0.5, outcome=0.0)]) assert math.isfinite(m["logloss_model"]) assert m["logloss_model"] == pytest.approx(-math.log(LOGLOSS_EPS)) def test_micro_weights_evaluations_macro_weights_markets(): # Market a: 3 evaluations, model error 0.1; market b: 1 evaluation, error 0.5. rows = [ _row(market_id="a", est=0.9, prior=0.8, outcome=1.0), _row(market_id="a", est=0.9, prior=0.8, outcome=1.0), _row(market_id="a", est=0.9, prior=0.8, outcome=1.0), _row(market_id="b", est=0.5, prior=0.6, outcome=1.0), ] m = compute_calibration(rows) assert m["n_evaluations"] == 4 assert m["n_markets"] == 2 # micro: (3*0.01 + 0.25) / 4 ; macro: (0.01 + 0.25) / 2 assert m["brier_model"] == pytest.approx((3 * 0.01 + 0.25) / 4) assert m["brier_model_macro"] == pytest.approx((0.01 + 0.25) / 2) assert m["brier_prior"] == pytest.approx((3 * 0.04 + 0.16) / 4) assert m["brier_prior_macro"] == pytest.approx((0.04 + 0.16) / 2) def test_model_beating_market_gives_negative_delta(): # est closer to the outcome than the price on every row rows = [ _row(market_id="a", est=0.8, prior=0.6, outcome=1.0), _row(market_id="b", est=0.3, prior=0.45, outcome=0.0), ] m = compute_calibration(rows) assert m["brier_model"] < m["brier_prior"] assert m["logloss_model"] < m["logloss_prior"] def test_per_category_grouping_and_unknown(): rows = [ _row(market_id="a", category="politics", est=0.8, prior=0.6, outcome=1.0), _row(market_id="b", category="politics", est=0.7, prior=0.6, outcome=1.0), _row(market_id="c", category=None, est=0.4, prior=0.5, outcome=0.0), ] m = compute_calibration(rows) assert set(m["per_category"]) == {"politics", "unknown"} pol = m["per_category"]["politics"] assert pol["n"] == 2 and pol["markets"] == 2 assert pol["brier_model"] == pytest.approx((0.04 + 0.09) / 2) unk = m["per_category"]["unknown"] assert unk["n"] == 1 and unk["markets"] == 1 assert unk["brier_model"] == pytest.approx(0.16) def test_repeated_market_counts_once_in_markets(): rows = [ _row(market_id="a", est=0.8, prior=0.6, outcome=1.0), _row(market_id="a", est=0.7, prior=0.55, outcome=1.0), ] m = compute_calibration(rows) assert m["n_markets"] == 1 assert m["per_category"]["politics"]["markets"] == 1 # ── print_report ───────────────────────────────────────────────────────────── def test_report_handles_no_metrics(capsys): print_report(None, "R0 archive") assert "no scorable rows yet" in capsys.readouterr().out def test_report_prints_all_metric_lines(capsys): m = compute_calibration([ _row(market_id="a", est=0.8, prior=0.6, outcome=1.0), _row(market_id="b", category=None, est=0.4, prior=0.5, outcome=0.0), ]) print_report(m, "R0 archive") out = capsys.readouterr().out assert "2 evaluations, 2 markets" in out for label in ("Brier micro", "Brier macro", "logloss micro", "politics", "unknown"): assert label in out