From 43d9577fb25e2d8bf096007971d2668ffbfff357 Mon Sep 17 00:00:00 2001 From: chemavx Date: Fri, 12 Jun 2026 07:12:55 +0000 Subject: [PATCH] feat(metrics): real Sharpe ratio from daily PnL curve with minimum-sample gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit sharpe_ratio was hardcoded to 0.0 in MetricsTracker and exposed as 'or 0' in /api/summary. With only 1 resolved trade (~40 flat days plus one +299 jump) any computed Sharpe is statistically meaningless, so: - bot/metrics/sharpe.py: annualized Sharpe (sqrt(365)) from daily total_pnl closes, normalized by bankroll; sharpe_with_gate() returns None + status until >=30 days observed AND >=10 resolved trades. - Database.get_daily_pnl_closes(): last metrics_daily snapshot per UTC day, oldest first — the return series input. - MetricsTracker: stores the real (gated) Sharpe in the snapshot, NULL below the gate; log line now includes sharpe. - /api/summary: live Sharpe + sharpe_status/days_observed/min_* fields explaining why it is null; resolved_count now live from COUNT(*). - promotion_ready: requires resolved>=10, days>=30, and non-null win_rate/calibration/sharpe plus existing thresholds — a single lucky resolved trade can no longer promote. - Dashboard Sharpe card shows the insufficient-sample explanation when null instead of a bare em dash. Tests: 13 new in tests/test_sharpe_gate.py (formula, gate, API contract, tracker snapshot); verified failing pre-fix. Suite: 62 passed. Co-Authored-By: Claude Fable 5 --- api/main.py | 66 +++++-- bot/data/db.py | 18 ++ bot/metrics/sharpe.py | 79 +++++++++ bot/metrics/tracker.py | 17 +- dashboard/src/App.jsx | 8 +- tests/test_api_summary_consistency.py | 4 + tests/test_sharpe_gate.py | 242 ++++++++++++++++++++++++++ 7 files changed, 412 insertions(+), 22 deletions(-) create mode 100644 bot/metrics/sharpe.py create mode 100644 tests/test_sharpe_gate.py diff --git a/api/main.py b/api/main.py index 5ac5f12..6f8e82e 100644 --- a/api/main.py +++ b/api/main.py @@ -12,6 +12,11 @@ from fastapi.middleware.cors import CORSMiddleware from bot.data.db import Database from bot.executor.paper import cash_available +from bot.metrics.sharpe import ( + MIN_DAYS_OBSERVED, + MIN_RESOLVED_TRADES, + sharpe_with_gate, +) # Phase 6 format (Phase 6+): values already in log-odds space. # "fg_lo=+0.1200 mom_lo=+0.0000 news_lo=+0.0000 mfld_lo=-0.7483 btc_dom_lo=+0.0000" @@ -280,23 +285,40 @@ async def get_summary(): PnL and performance metrics come from the latest metrics_daily snapshot, which is written by the bot every cycle via MetricsTracker.update_daily_summary(). After Fix 3, that snapshot is also DB-computed — not dependent on pod restarts. + sharpe_ratio is the exception: it is recomputed live here from the daily + PnL-close series (same sharpe_with_gate the tracker uses), so the + explanation fields (sharpe_status, days_observed) always match the value. """ - latest_metrics, counts, position_data, inverted, legacy_count = await asyncio.gather( - db.get_metrics_history(days=1), - db.compute_metrics_from_db(), - db.get_open_position_data(), - db.get_recently_closed_inverted(hours=24), - db.get_legacy_incomplete_count(), + latest_metrics, counts, position_data, inverted, legacy_count, daily_closes = ( + await asyncio.gather( + db.get_metrics_history(days=1), + db.compute_metrics_from_db(), + db.get_open_position_data(), + db.get_recently_closed_inverted(hours=24), + db.get_legacy_incomplete_count(), + db.get_daily_pnl_closes(), + ) ) latest = latest_metrics[0] if latest_metrics else {} paper_bankroll = float(os.getenv("PAPER_BANKROLL", "10000")) total_trades = int(counts["total_trades"] or 0) + resolved_count = int(counts.get("resolved_count") or 0) # Same source PaperExecutor.initialize() uses to reconstruct cash: # total_net_cost_open = SUM(net_cost) over open trades, uncapped. _, total_net_cost_open = position_data total_deployed = total_net_cost_open + # Sharpe: computed live from the daily PnL curve (same function the + # tracker uses for the snapshot). None + status while the minimum-sample + # gate (>=30 days observed, >=10 resolved trades) is not met — a Sharpe + # over 1 resolved trade is statistically meaningless. + days_observed = len(daily_closes) + sharpe, sharpe_status = sharpe_with_gate(daily_closes, paper_bankroll, resolved_count) + + win_rate = latest.get("win_rate") + calibration = latest.get("calibration_score") + return { # ── Portfolio state (live from DB) ────────────────────────────────── "paper_mode": os.getenv("PAPER_MODE", "true") == "true", @@ -319,25 +341,35 @@ async def get_summary(): "realized_pnl": latest.get("realized_pnl") or 0, "total_pnl": latest.get("total_pnl") or 0, - # ── Performance metrics (from latest metrics_daily snapshot) ───────── + # ── Performance metrics ────────────────────────────────────────────── # win_rate: fraction of resolved closed trades where close_pnl > 0. # null if fewer than 5 resolved trades. Source: closed+resolved trades. - # sharpe_ratio: 0.0 — requires daily-return time series (not yet tracked). + # sharpe_ratio: annualized Sharpe of the daily total_pnl curve, computed + # live from metrics_daily. null while the minimum-sample gate fails + # (sharpe_status explains why). Source: bot/metrics/sharpe.py. # calibration_score: 1 − Brier score on resolved trades (higher = better). # null if fewer than 10 resolved trades. Source: closed+resolved trades. - "win_rate": latest.get("win_rate"), # null if < 5 resolved - "sharpe_ratio": latest.get("sharpe_ratio") or 0, # 0.0 until tracked - "calibration_score": latest.get("calibration_score"), # null if < 10 resolved + "win_rate": win_rate, # null if < 5 resolved + "sharpe_ratio": sharpe, # null if gate fails + "sharpe_status": sharpe_status, # ok | insufficient_sample | zero_variance + "days_observed": days_observed, + "min_days_required": MIN_DAYS_OBSERVED, + "min_resolved_required": MIN_RESOLVED_TRADES, + "calibration_score": calibration, # null if < 10 resolved - # ── Counters from snapshot ─────────────────────────────────────────── - "resolved_count": latest.get("resolved_count") or 0, + # ── Counters (live from DB) ────────────────────────────────────────── + "resolved_count": resolved_count, # ── Promotion gate ─────────────────────────────────────────────────── - # All thresholds must pass; null metrics count as not-ready. + # Never promote on a tiny sample: requires the resolved/days minimums + # AND non-null metrics AND all thresholds. A single lucky resolved + # trade must not flip this to true. "promotion_ready": ( - (latest.get("sharpe_ratio") or 0) >= 0.5 - and (latest.get("win_rate") or 0) >= 0.52 - and (latest.get("calibration_score") or 0) >= 0.7 + resolved_count >= MIN_RESOLVED_TRADES + and days_observed >= MIN_DAYS_OBSERVED + and win_rate is not None and win_rate >= 0.52 + and calibration is not None and calibration >= 0.7 + and sharpe is not None and sharpe >= 0.5 and total_trades >= 50 ), } diff --git a/bot/data/db.py b/bot/data/db.py index 4a7f21a..b622634 100644 --- a/bot/data/db.py +++ b/bot/data/db.py @@ -348,6 +348,24 @@ class Database: ) return [dict(r) for r in rows] + async def get_daily_pnl_closes(self) -> list[float]: + """Return the closing total_pnl of every observed UTC day, oldest first. + + One value per calendar day with at least one metrics_daily snapshot + (the day's last snapshot, same collapse rule as get_metrics_history). + This is the input series for the Sharpe ratio: len() = days observed, + consecutive deltas = daily PnL changes. + """ + async with self._pool.acquire() as conn: + rows = await conn.fetch( + """ + SELECT DISTINCT ON (timestamp::date) total_pnl + FROM metrics_daily + ORDER BY timestamp::date ASC, timestamp DESC + """ + ) + return [float(r["total_pnl"] or 0) for r in rows] + async def backfill_feature_columns(self) -> int: """Back-populate feat_*_lo for trades created before Phase 6. diff --git a/bot/metrics/sharpe.py b/bot/metrics/sharpe.py new file mode 100644 index 0000000..8b55339 --- /dev/null +++ b/bot/metrics/sharpe.py @@ -0,0 +1,79 @@ +""" +Sharpe ratio from the paper portfolio's daily PnL curve, with a minimum-sample gate. + +The input series is the closing total_pnl of each observed UTC day +(Database.get_daily_pnl_closes). Daily returns are PnL deltas normalized by +the paper bankroll: + + r_t = (pnl_t − pnl_{t−1}) / bankroll + +Sharpe = mean(r) / sample_std(r) × √365, annualized — prediction markets +resolve every calendar day, so 365 is used instead of 252 trading days. +Risk-free rate is taken as 0. + +Gate: with a tiny sample (e.g. 1 resolved trade over a flat curve plus one ++299 jump) any Sharpe value is statistically meaningless — artificially huge +or tiny depending on where the jump lands. So no numeric Sharpe is exposed +until BOTH minimums are met: + + days observed >= MIN_DAYS_OBSERVED (30) + resolved trades >= MIN_RESOLVED_TRADES (10) + +Below either minimum the value is None with status "insufficient_sample". +A perfectly flat curve (zero variance) also yields None ("zero_variance"): +Sharpe is undefined there, not infinite. +""" +from statistics import mean, stdev +from typing import Optional + +MIN_DAYS_OBSERVED = 30 +MIN_RESOLVED_TRADES = 10 +ANNUALIZATION_DAYS = 365 + +SHARPE_OK = "ok" +SHARPE_INSUFFICIENT = "insufficient_sample" +SHARPE_ZERO_VARIANCE = "zero_variance" + + +def daily_returns(daily_pnl_closes: list[float], bankroll: float) -> list[float]: + """Bankroll-normalized day-over-day returns from a daily PnL-close series.""" + return [ + (curr - prev) / bankroll + for prev, curr in zip(daily_pnl_closes, daily_pnl_closes[1:]) + ] + + +def compute_sharpe(daily_pnl_closes: list[float], bankroll: float) -> Optional[float]: + """Annualized Sharpe of the daily PnL curve, or None if undefined. + + None when there are fewer than 2 returns (need 3+ daily closes) or the + return series has zero variance. No sample-size gate here — see + sharpe_with_gate() for the exposed value. + """ + returns = daily_returns(daily_pnl_closes, bankroll) + if len(returns) < 2: + return None + sd = stdev(returns) + if sd == 0: + return None + return mean(returns) / sd * ANNUALIZATION_DAYS ** 0.5 + + +def sharpe_with_gate( + daily_pnl_closes: list[float], + bankroll: float, + resolved_count: int, +) -> tuple[Optional[float], str]: + """Return (sharpe, status) applying the minimum-sample gate. + + status: "ok" — sharpe is a meaningful float + "insufficient_sample" — sample below minimums, sharpe is None + "zero_variance" — sample OK but flat curve, sharpe is None + """ + days_observed = len(daily_pnl_closes) + if days_observed < MIN_DAYS_OBSERVED or resolved_count < MIN_RESOLVED_TRADES: + return None, SHARPE_INSUFFICIENT + sharpe = compute_sharpe(daily_pnl_closes, bankroll) + if sharpe is None: + return None, SHARPE_ZERO_VARIANCE + return sharpe, SHARPE_OK diff --git a/bot/metrics/tracker.py b/bot/metrics/tracker.py index 881a486..4e9c81e 100644 --- a/bot/metrics/tracker.py +++ b/bot/metrics/tracker.py @@ -15,12 +15,16 @@ win_rate Fraction of resolved closed trades with close_pnl > 0. NULL if fewer than 5 resolved trades. calibration_score 1 − AVG((final_prob − resolution)²) on resolved trades. Brier score (higher = better calibration). NULL if < 10 resolved. -sharpe_ratio 0.0 — requires a daily-return time series, not yet tracked. +sharpe_ratio Annualized Sharpe of the daily total_pnl curve (see + bot/metrics/sharpe.py). NULL until the sample gate passes: + >= 30 days observed AND >= 10 resolved trades. """ import logging +import os from datetime import datetime, UTC from bot.data.db import Database +from bot.metrics.sharpe import sharpe_with_gate log = logging.getLogger(__name__) @@ -61,6 +65,12 @@ class MetricsTracker: avg_edge = total_pnl / total_deployed if total_deployed > 0 else 0.0 + # Sharpe: real value from the daily PnL curve, NULL while the sample + # gate (>=30 days observed, >=10 resolved) is not met. + bankroll = float(os.getenv("PAPER_BANKROLL", "10000")) + daily_closes = await self._db.get_daily_pnl_closes() + sharpe, sharpe_status = sharpe_with_gate(daily_closes, bankroll, resolved) + metrics = { "timestamp": datetime.now(UTC), "total_trades": int(raw["total_trades"]), @@ -74,7 +84,7 @@ class MetricsTracker: "total_pnl": total_pnl, "win_rate": win_rate, "avg_edge": avg_edge, - "sharpe_ratio": 0.0, # requires daily-return series (not yet tracked) + "sharpe_ratio": sharpe, # NULL until sample gate passes "calibration_score": calibration, "paper_mode": True, } @@ -83,9 +93,10 @@ class MetricsTracker: log.info( "Daily metrics | trades=%d (open=%d closed=%d resolved=%d) | " "unrealized=$%.2f realized=$%.2f total=$%.2f | " - "win_rate=%s calibration=%s", + "win_rate=%s calibration=%s sharpe=%s", metrics["total_trades"], open_count, closed_count, resolved, unrealized, realized, total_pnl, f"{win_rate:.1%}" if win_rate is not None else "n/a (<5)", f"{calibration:.3f}" if calibration is not None else "n/a (<10)", + f"{sharpe:.2f}" if sharpe is not None else f"n/a ({sharpe_status})", ) diff --git a/dashboard/src/App.jsx b/dashboard/src/App.jsx index a6ce284..141bdc9 100644 --- a/dashboard/src/App.jsx +++ b/dashboard/src/App.jsx @@ -200,8 +200,12 @@ export default function App() { = 0.5 ? 'var(--green)' : 'var(--amber)'} /> tuple[dict, PaperExecutor]: monkeypatch.setattr(api_main, "db", db) diff --git a/tests/test_sharpe_gate.py b/tests/test_sharpe_gate.py new file mode 100644 index 0000000..00456ff --- /dev/null +++ b/tests/test_sharpe_gate.py @@ -0,0 +1,242 @@ +""" +Tests for the real Sharpe ratio with minimum-sample gate. + +Regression: sharpe_ratio was hardcoded to 0.0 in MetricsTracker and exposed +as `latest.get("sharpe_ratio") or 0` in /api/summary, and promotion_ready +could in principle flip on a statistically meaningless sample (e.g. 1 +resolved trade over ~40 days of flat PnL plus a single +299 jump). + +Fix: bot/metrics/sharpe.py computes an annualized Sharpe from the daily +total_pnl close series, gated to None ("insufficient_sample") below 30 days +observed / 10 resolved trades. /api/summary exposes the value plus an +explanation (sharpe_status, days_observed, min_* fields), and +promotion_ready additionally requires the sample minimums and non-null +metrics. +""" +import asyncio +from statistics import mean, stdev + +import pytest + +import api.main as api_main +from bot.metrics.sharpe import ( + MIN_DAYS_OBSERVED, + MIN_RESOLVED_TRADES, + SHARPE_INSUFFICIENT, + SHARPE_OK, + SHARPE_ZERO_VARIANCE, + compute_sharpe, + daily_returns, + sharpe_with_gate, +) +from bot.metrics.tracker import MetricsTracker + + +BANKROLL = 10_000.0 + + +def _closes_from_deltas(deltas: list[float], start: float = 0.0) -> list[float]: + closes = [start] + for d in deltas: + closes.append(closes[-1] + d) + return closes + + +# ── Pure computation ───────────────────────────────────────────────────────── + +def test_daily_returns_are_bankroll_normalized_deltas(): + closes = [0.0, 100.0, 50.0, 50.0] + assert daily_returns(closes, BANKROLL) == pytest.approx([0.01, -0.005, 0.0]) + + +def test_compute_sharpe_matches_manual_formula(): + deltas = [10.0, 14.0, 8.0, 12.0, 6.0, 13.0, 9.0] + closes = _closes_from_deltas(deltas) + rets = [d / BANKROLL for d in deltas] + expected = mean(rets) / stdev(rets) * 365 ** 0.5 + assert compute_sharpe(closes, BANKROLL) == pytest.approx(expected) + assert compute_sharpe(closes, BANKROLL) > 0 + + +def test_compute_sharpe_undefined_cases_return_none(): + assert compute_sharpe([], BANKROLL) is None + assert compute_sharpe([0.0], BANKROLL) is None + assert compute_sharpe([0.0, 50.0], BANKROLL) is None # only 1 return + assert compute_sharpe([0.0] * 40, BANKROLL) is None # zero variance + + +# ── Minimum-sample gate ─────────────────────────────────────────────────────── + +def test_gate_blocks_current_situation_one_resolved_trade(): + """~40 flat days plus a single +299 jump, 1 resolved trade → no Sharpe.""" + closes = [0.0] * 35 + [299.06] * 5 + sharpe, status = sharpe_with_gate(closes, BANKROLL, resolved_count=1) + assert sharpe is None + assert status == SHARPE_INSUFFICIENT + # The raw (ungated) value would exist and be wildly misleading: + assert compute_sharpe(closes, BANKROLL) is not None + + +def test_gate_blocks_too_few_days_even_with_enough_resolved(): + closes = _closes_from_deltas([10.0, -5.0] * 10) # 21 days < 30 + sharpe, status = sharpe_with_gate(closes, BANKROLL, resolved_count=15) + assert sharpe is None + assert status == SHARPE_INSUFFICIENT + + +def test_gate_passes_with_sufficient_sample(): + deltas = [10.0, 14.0, 8.0, 12.0, 6.0] * 8 # 40 returns → 41 days + closes = _closes_from_deltas(deltas) + sharpe, status = sharpe_with_gate(closes, BANKROLL, resolved_count=MIN_RESOLVED_TRADES) + assert status == SHARPE_OK + assert sharpe == pytest.approx(compute_sharpe(closes, BANKROLL)) + + +def test_gate_flat_curve_with_sufficient_sample_is_zero_variance(): + sharpe, status = sharpe_with_gate([0.0] * 40, BANKROLL, resolved_count=12) + assert sharpe is None + assert status == SHARPE_ZERO_VARIANCE + + +# ── /api/summary ───────────────────────────────────────────────────────────── + +class FakeDB: + def __init__(self, daily_closes, resolved_count, total_trades=60, + win_rate=0.6, calibration=0.8): + self._closes = daily_closes + self._resolved = resolved_count + self._total = total_trades + self._win_rate = win_rate + self._calibration = calibration + + async def get_metrics_history(self, days=1): + return [{ + "win_rate": self._win_rate, + "calibration_score": self._calibration, + "unrealized_pnl_est": 0.0, + "realized_pnl": 299.06, + "total_pnl": 299.06, + }] + + async def compute_metrics_from_db(self): + return { + "total_trades": self._total, + "open_count": self._total - self._resolved, + "closed_count": self._resolved, + "resolved_count": self._resolved, + } + + async def get_open_position_data(self): + return {}, 0.0 + + async def get_recently_closed_inverted(self, hours=24): + return set() + + async def get_legacy_incomplete_count(self): + return 0 + + async def get_daily_pnl_closes(self): + return list(self._closes) + + +def _summary(db, monkeypatch) -> dict: + monkeypatch.setattr(api_main, "db", db) + monkeypatch.delenv("PAPER_BANKROLL", raising=False) + return asyncio.run(api_main.get_summary()) + + +def test_api_insufficient_sample_returns_null_with_explanation(monkeypatch): + """Current prod situation: 1 resolved, ~40 days → null Sharpe, not ready.""" + db = FakeDB(daily_closes=[0.0] * 35 + [299.06] * 5, resolved_count=1) + s = _summary(db, monkeypatch) + assert s["sharpe_ratio"] is None + assert s["sharpe_status"] == SHARPE_INSUFFICIENT + assert s["resolved_count"] == 1 + assert s["min_resolved_required"] == MIN_RESOLVED_TRADES == 10 + assert s["days_observed"] == 40 + assert s["min_days_required"] == MIN_DAYS_OBSERVED == 30 + # One lucky resolved trade must never promote, even with perfect + # win_rate/calibration and 50+ trades. + assert s["promotion_ready"] is False + + +def test_api_sharpe_appears_with_sufficient_sample(monkeypatch): + deltas = [10.0, 14.0, 8.0, 12.0, 6.0] * 8 + db = FakeDB(daily_closes=_closes_from_deltas(deltas), resolved_count=12) + s = _summary(db, monkeypatch) + assert s["sharpe_status"] == SHARPE_OK + assert s["sharpe_ratio"] == pytest.approx( + compute_sharpe(_closes_from_deltas(deltas), BANKROLL) + ) + assert s["sharpe_ratio"] >= 0.5 + assert s["promotion_ready"] is True + + +def test_api_not_ready_when_sharpe_below_threshold(monkeypatch): + # Zero-drift curve: mean return ~0 → Sharpe ≈ 0 < 0.5 + deltas = [50.0, -50.0] * 20 + db = FakeDB(daily_closes=_closes_from_deltas(deltas), resolved_count=12) + s = _summary(db, monkeypatch) + assert s["sharpe_status"] == SHARPE_OK + assert s["sharpe_ratio"] < 0.5 + assert s["promotion_ready"] is False + + +def test_api_not_ready_when_metrics_null(monkeypatch): + db = FakeDB( + daily_closes=_closes_from_deltas([10.0, 14.0, 8.0, 12.0, 6.0] * 8), + resolved_count=12, + win_rate=None, + calibration=None, + ) + s = _summary(db, monkeypatch) + assert s["sharpe_status"] == SHARPE_OK + assert s["promotion_ready"] is False + + +# ── MetricsTracker: no hardcoded 0.0 in the snapshot ───────────────────────── + +class FakeTrackerDB: + def __init__(self, daily_closes, resolved_count): + self._closes = daily_closes + self._resolved = resolved_count + self.saved = None + + async def compute_metrics_from_db(self): + return { + "total_trades": 60, + "open_count": 40, + "closed_count": 20, + "resolved_count": self._resolved, + "wins_realized": self._resolved, + "unrealized_pnl_est": 0.0, + "realized_pnl": 100.0, + "total_deployed": 1000.0, + "total_fees": 20.0, + "calibration_score": 0.8, + } + + async def get_daily_pnl_closes(self): + return list(self._closes) + + async def save_daily_metrics(self, metrics): + self.saved = metrics + + +def test_tracker_stores_null_sharpe_below_gate(monkeypatch): + monkeypatch.delenv("PAPER_BANKROLL", raising=False) + db = FakeTrackerDB(daily_closes=[0.0] * 35 + [299.06] * 5, resolved_count=1) + asyncio.run(MetricsTracker(db).update_daily_summary()) + assert db.saved is not None + assert db.saved["sharpe_ratio"] is None + + +def test_tracker_stores_real_sharpe_above_gate(monkeypatch): + monkeypatch.delenv("PAPER_BANKROLL", raising=False) + closes = _closes_from_deltas([10.0, 14.0, 8.0, 12.0, 6.0] * 8) + db = FakeTrackerDB(daily_closes=closes, resolved_count=12) + asyncio.run(MetricsTracker(db).update_daily_summary()) + assert db.saved["sharpe_ratio"] == pytest.approx( + compute_sharpe(closes, BANKROLL) + ) + assert db.saved["sharpe_ratio"] != 0.0