feat(replay): R2 outcomes + calibration metrics
Scores every archived estimate against reality — the sample multiplier the phase plan calls for: Brier/log-loss of estimated_prob benchmarked against the market price (prior_prob) on the same rows, over ALL evaluations with a resolved outcome, not just executed trades. - schema.sql: market_outcomes (one row per resolved market; outcome = final YES price 1.0/0.0, UMA-final only) - bot/outcomes.py: CLI (python -m bot.outcomes) with two phases — fetch resolutions for archived markets via the existing get_market_resolution() (open/disputed/ambiguous markets simply retry next invocation; no data-loss urgency, Gamma reports past resolutions at any time), then compute calibration: Brier micro (per evaluation) / macro (per market — the honest sample size given ~1 eval/min autocorrelation), log-loss with 1e-9 clipping, per-category breakdown. --run-id scores a replay run's re-estimates instead of the archive (counterfactual calibration). - db.py: 4 accessors (pending markets, outcome upsert, coverage, calibration rows for archive or run) - tests: 12 new (116 total green); compute_calibration is a pure function driven by literals No prod behavior change: the bot never imports bot.outcomes; the only shared surface is the idempotent schema migration (dry-run BEGIN/ROLLBACK clean against prod). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
chemavx
co-authored by
Claude Fable 5
Claude Fable 5
parent
6c544e46e2
commit
124b6d8558
@@ -826,6 +826,75 @@ class Database:
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17)
|
||||
""", rows)
|
||||
|
||||
# ── Replay R2: outcomes + calibration metrics ────────────────────────────
|
||||
|
||||
async def get_unresolved_archived_market_ids(self) -> list[str]:
|
||||
"""Archived markets (present in signals) with no stored outcome yet."""
|
||||
async with self._pool.acquire() as conn:
|
||||
rows = await conn.fetch("""
|
||||
SELECT DISTINCT s.market_id FROM signals s
|
||||
LEFT JOIN market_outcomes o ON o.market_id = s.market_id
|
||||
WHERE o.market_id IS NULL
|
||||
ORDER BY s.market_id
|
||||
""")
|
||||
return [r["market_id"] for r in rows]
|
||||
|
||||
async def upsert_market_outcome(
|
||||
self, market_id: str, outcome: float, resolved_at
|
||||
) -> None:
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.execute("""
|
||||
INSERT INTO market_outcomes (market_id, outcome, resolved_at)
|
||||
VALUES ($1, $2, $3)
|
||||
ON CONFLICT (market_id) DO UPDATE
|
||||
SET outcome = EXCLUDED.outcome,
|
||||
resolved_at = EXCLUDED.resolved_at,
|
||||
fetched_at = NOW()
|
||||
""", market_id, outcome, resolved_at)
|
||||
|
||||
async def get_outcome_coverage(self) -> dict:
|
||||
"""How much of the archive is scorable: resolved vs archived markets."""
|
||||
async with self._pool.acquire() as conn:
|
||||
row = await conn.fetchrow("""
|
||||
SELECT
|
||||
(SELECT COUNT(DISTINCT market_id) FROM signals) AS archived,
|
||||
(SELECT COUNT(*) FROM market_outcomes
|
||||
WHERE market_id IN (SELECT DISTINCT market_id FROM signals)
|
||||
) AS resolved
|
||||
""")
|
||||
return dict(row)
|
||||
|
||||
async def get_calibration_rows(self, run_id: Optional[str] = None) -> list[dict]:
|
||||
"""Every archived evaluation with a full estimate AND a known outcome.
|
||||
|
||||
run_id None scores the R0 archive (signals); a run_id scores that
|
||||
replay run's re-estimates instead (counterfactual calibration).
|
||||
Rows without estimated_prob (skipped before estimation: prior_extreme,
|
||||
unsupported, family, no_signals) carry no model prediction to score.
|
||||
"""
|
||||
async with self._pool.acquire() as conn:
|
||||
if run_id is None:
|
||||
rows = await conn.fetch("""
|
||||
SELECT s.market_id, s.category,
|
||||
s.estimated_prob, s.prior_prob, o.outcome
|
||||
FROM signals s
|
||||
JOIN market_outcomes o ON o.market_id = s.market_id
|
||||
WHERE s.estimated_prob IS NOT NULL
|
||||
AND s.prior_prob IS NOT NULL
|
||||
""")
|
||||
else:
|
||||
rows = await conn.fetch("""
|
||||
SELECT d.market_id, m.category,
|
||||
d.estimated_prob, d.prior_prob, o.outcome
|
||||
FROM replay_decisions d
|
||||
JOIN market_outcomes o ON o.market_id = d.market_id
|
||||
LEFT JOIN markets m ON m.id = d.market_id
|
||||
WHERE d.run_id = $1
|
||||
AND d.estimated_prob IS NOT NULL
|
||||
AND d.prior_prob IS NOT NULL
|
||||
""", run_id)
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
async def mark_manifold_audit_used(self, audit_id: str) -> None:
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
|
||||
@@ -431,3 +431,24 @@ CREATE TABLE IF NOT EXISTS replay_decisions (
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_replay_decisions_run ON replay_decisions(run_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_replay_decisions_mkt ON replay_decisions(market_id);
|
||||
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- Replay R2: outcomes + calibration metrics
|
||||
--
|
||||
-- One row per resolved market, fetched from the Gamma API via
|
||||
-- get_market_resolution() (UMA-final only: a market closed but still in
|
||||
-- proposal/dispute is not stored). outcome is the final YES price:
|
||||
-- 1.0 = YES won, 0.0 = NO won.
|
||||
--
|
||||
-- Joining signals (or replay_decisions) to market_outcomes scores every
|
||||
-- archived estimate against reality — Brier / log-loss of estimated_prob
|
||||
-- benchmarked against the market price (prior_prob) on the same rows,
|
||||
-- answering "does the model add value over the market?" across ALL
|
||||
-- evaluations, not just executed trades.
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
CREATE TABLE IF NOT EXISTS market_outcomes (
|
||||
market_id TEXT PRIMARY KEY,
|
||||
outcome DOUBLE PRECISION NOT NULL,
|
||||
resolved_at TIMESTAMPTZ,
|
||||
fetched_at TIMESTAMPTZ DEFAULT NOW()
|
||||
);
|
||||
|
||||
Reference in New Issue
Block a user