chemavxandClaude Fable 5
124b6d8558
Scores every archived estimate against reality — the sample multiplier the phase plan calls for: Brier/log-loss of estimated_prob benchmarked against the market price (prior_prob) on the same rows, over ALL evaluations with a resolved outcome, not just executed trades. - schema.sql: market_outcomes (one row per resolved market; outcome = final YES price 1.0/0.0, UMA-final only) - bot/outcomes.py: CLI (python -m bot.outcomes) with two phases — fetch resolutions for archived markets via the existing get_market_resolution() (open/disputed/ambiguous markets simply retry next invocation; no data-loss urgency, Gamma reports past resolutions at any time), then compute calibration: Brier micro (per evaluation) / macro (per market — the honest sample size given ~1 eval/min autocorrelation), log-loss with 1e-9 clipping, per-category breakdown. --run-id scores a replay run's re-estimates instead of the archive (counterfactual calibration). - db.py: 4 accessors (pending markets, outcome upsert, coverage, calibration rows for archive or run) - tests: 12 new (116 total green); compute_calibration is a pure function driven by literals No prod behavior change: the bot never imports bot.outcomes; the only shared surface is the idempotent schema migration (dry-run BEGIN/ROLLBACK clean against prod). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
175 lines
6.5 KiB
Python
175 lines
6.5 KiB
Python
"""Replay R2 tests — outcome fetching and calibration scoring."""
|
|
import asyncio
|
|
import math
|
|
|
|
import pytest
|
|
|
|
from bot.data.polymarket import MarketResolution
|
|
from bot.outcomes import (
|
|
LOGLOSS_EPS,
|
|
compute_calibration,
|
|
fetch_outcomes,
|
|
print_report,
|
|
)
|
|
from datetime import datetime, timezone
|
|
|
|
|
|
class FakePoly:
|
|
"""get_market_resolution stand-in driven by a dict of canned responses."""
|
|
|
|
def __init__(self, responses: dict):
|
|
self.responses = responses
|
|
self.calls: list[str] = []
|
|
|
|
async def get_market_resolution(self, market_id: str):
|
|
self.calls.append(market_id)
|
|
return self.responses.get(market_id)
|
|
|
|
|
|
RESOLVED_AT = datetime(2026, 7, 1, 12, 0, tzinfo=timezone.utc)
|
|
|
|
|
|
def _row(market_id="m1", category="politics", est=0.6, prior=0.5, outcome=1.0):
|
|
return {
|
|
"market_id": market_id,
|
|
"category": category,
|
|
"estimated_prob": est,
|
|
"prior_prob": prior,
|
|
"outcome": outcome,
|
|
}
|
|
|
|
|
|
# ── fetch_outcomes ───────────────────────────────────────────────────────────
|
|
|
|
def test_fetch_keeps_only_definitive_resolutions():
|
|
poly = FakePoly({
|
|
"yes": MarketResolution(resolved=True, resolution=1.0,
|
|
resolved_at=RESOLVED_AT),
|
|
"no": MarketResolution(resolved=True, resolution=0.0,
|
|
resolved_at=None),
|
|
"open": MarketResolution(resolved=False),
|
|
"disputed": MarketResolution(resolved=False),
|
|
"apierror": None, # get_market_resolution returns None on HTTP errors
|
|
})
|
|
out = asyncio.run(
|
|
fetch_outcomes(poly, ["yes", "no", "open", "disputed", "apierror"])
|
|
)
|
|
assert poly.calls == ["yes", "no", "open", "disputed", "apierror"]
|
|
assert out == [
|
|
{"market_id": "yes", "outcome": 1.0, "resolved_at": RESOLVED_AT},
|
|
{"market_id": "no", "outcome": 0.0, "resolved_at": None},
|
|
]
|
|
|
|
|
|
def test_fetch_empty_list_is_noop():
|
|
poly = FakePoly({})
|
|
assert asyncio.run(fetch_outcomes(poly, [])) == []
|
|
assert poly.calls == []
|
|
|
|
|
|
# ── compute_calibration ──────────────────────────────────────────────────────
|
|
|
|
def test_no_rows_returns_none():
|
|
assert compute_calibration([]) is None
|
|
|
|
|
|
def test_single_row_known_values():
|
|
m = compute_calibration([_row(est=0.8, prior=0.6, outcome=1.0)])
|
|
assert m["n_evaluations"] == 1
|
|
assert m["n_markets"] == 1
|
|
assert m["brier_model"] == pytest.approx((0.8 - 1.0) ** 2)
|
|
assert m["brier_prior"] == pytest.approx((0.6 - 1.0) ** 2)
|
|
assert m["logloss_model"] == pytest.approx(-math.log(0.8))
|
|
assert m["logloss_prior"] == pytest.approx(-math.log(0.6))
|
|
# one market: macro == micro
|
|
assert m["brier_model_macro"] == pytest.approx(m["brier_model"])
|
|
assert m["brier_prior_macro"] == pytest.approx(m["brier_prior"])
|
|
|
|
|
|
def test_logloss_no_outcome_branch():
|
|
m = compute_calibration([_row(est=0.2, prior=0.7, outcome=0.0)])
|
|
assert m["logloss_model"] == pytest.approx(-math.log(0.8))
|
|
assert m["logloss_prior"] == pytest.approx(-math.log(0.3))
|
|
|
|
|
|
def test_logloss_clipping_keeps_hard_miss_finite():
|
|
# A hard 1.0 estimate on a NO outcome must not produce inf.
|
|
m = compute_calibration([_row(est=1.0, prior=0.5, outcome=0.0)])
|
|
assert math.isfinite(m["logloss_model"])
|
|
assert m["logloss_model"] == pytest.approx(-math.log(LOGLOSS_EPS))
|
|
|
|
|
|
def test_micro_weights_evaluations_macro_weights_markets():
|
|
# Market a: 3 evaluations, model error 0.1; market b: 1 evaluation, error 0.5.
|
|
rows = [
|
|
_row(market_id="a", est=0.9, prior=0.8, outcome=1.0),
|
|
_row(market_id="a", est=0.9, prior=0.8, outcome=1.0),
|
|
_row(market_id="a", est=0.9, prior=0.8, outcome=1.0),
|
|
_row(market_id="b", est=0.5, prior=0.6, outcome=1.0),
|
|
]
|
|
m = compute_calibration(rows)
|
|
assert m["n_evaluations"] == 4
|
|
assert m["n_markets"] == 2
|
|
# micro: (3*0.01 + 0.25) / 4 ; macro: (0.01 + 0.25) / 2
|
|
assert m["brier_model"] == pytest.approx((3 * 0.01 + 0.25) / 4)
|
|
assert m["brier_model_macro"] == pytest.approx((0.01 + 0.25) / 2)
|
|
assert m["brier_prior"] == pytest.approx((3 * 0.04 + 0.16) / 4)
|
|
assert m["brier_prior_macro"] == pytest.approx((0.04 + 0.16) / 2)
|
|
|
|
|
|
def test_model_beating_market_gives_negative_delta():
|
|
# est closer to the outcome than the price on every row
|
|
rows = [
|
|
_row(market_id="a", est=0.8, prior=0.6, outcome=1.0),
|
|
_row(market_id="b", est=0.3, prior=0.45, outcome=0.0),
|
|
]
|
|
m = compute_calibration(rows)
|
|
assert m["brier_model"] < m["brier_prior"]
|
|
assert m["logloss_model"] < m["logloss_prior"]
|
|
|
|
|
|
def test_per_category_grouping_and_unknown():
|
|
rows = [
|
|
_row(market_id="a", category="politics", est=0.8, prior=0.6, outcome=1.0),
|
|
_row(market_id="b", category="politics", est=0.7, prior=0.6, outcome=1.0),
|
|
_row(market_id="c", category=None, est=0.4, prior=0.5, outcome=0.0),
|
|
]
|
|
m = compute_calibration(rows)
|
|
assert set(m["per_category"]) == {"politics", "unknown"}
|
|
pol = m["per_category"]["politics"]
|
|
assert pol["n"] == 2 and pol["markets"] == 2
|
|
assert pol["brier_model"] == pytest.approx((0.04 + 0.09) / 2)
|
|
unk = m["per_category"]["unknown"]
|
|
assert unk["n"] == 1 and unk["markets"] == 1
|
|
assert unk["brier_model"] == pytest.approx(0.16)
|
|
|
|
|
|
def test_repeated_market_counts_once_in_markets():
|
|
rows = [
|
|
_row(market_id="a", est=0.8, prior=0.6, outcome=1.0),
|
|
_row(market_id="a", est=0.7, prior=0.55, outcome=1.0),
|
|
]
|
|
m = compute_calibration(rows)
|
|
assert m["n_markets"] == 1
|
|
assert m["per_category"]["politics"]["markets"] == 1
|
|
|
|
|
|
# ── print_report ─────────────────────────────────────────────────────────────
|
|
|
|
def test_report_handles_no_metrics(capsys):
|
|
print_report(None, "R0 archive")
|
|
assert "no scorable rows yet" in capsys.readouterr().out
|
|
|
|
|
|
def test_report_prints_all_metric_lines(capsys):
|
|
m = compute_calibration([
|
|
_row(market_id="a", est=0.8, prior=0.6, outcome=1.0),
|
|
_row(market_id="b", category=None, est=0.4, prior=0.5, outcome=0.0),
|
|
])
|
|
print_report(m, "R0 archive")
|
|
out = capsys.readouterr().out
|
|
assert "2 evaluations, 2 markets" in out
|
|
for label in ("Brier micro", "Brier macro", "logloss micro",
|
|
"politics", "unknown"):
|
|
assert label in out
|