feat(metrics): real Sharpe ratio from daily PnL curve with minimum-sample gate
CI/CD / build-and-push (push) Successful in 10s

sharpe_ratio was hardcoded to 0.0 in MetricsTracker and exposed as
'or 0' in /api/summary. With only 1 resolved trade (~40 flat days plus
one +299 jump) any computed Sharpe is statistically meaningless, so:

- bot/metrics/sharpe.py: annualized Sharpe (sqrt(365)) from daily
  total_pnl closes, normalized by bankroll; sharpe_with_gate() returns
  None + status until >=30 days observed AND >=10 resolved trades.
- Database.get_daily_pnl_closes(): last metrics_daily snapshot per UTC
  day, oldest first — the return series input.
- MetricsTracker: stores the real (gated) Sharpe in the snapshot, NULL
  below the gate; log line now includes sharpe.
- /api/summary: live Sharpe + sharpe_status/days_observed/min_* fields
  explaining why it is null; resolved_count now live from COUNT(*).
- promotion_ready: requires resolved>=10, days>=30, and non-null
  win_rate/calibration/sharpe plus existing thresholds — a single lucky
  resolved trade can no longer promote.
- Dashboard Sharpe card shows the insufficient-sample explanation when
  null instead of a bare em dash.

Tests: 13 new in tests/test_sharpe_gate.py (formula, gate, API contract,
tracker snapshot); verified failing pre-fix. Suite: 62 passed.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
chemavx
2026-06-12 07:12:55 +00:00
co-authored by Claude Fable 5
parent 1797b79f7b
commit 43d9577fb2
7 changed files with 412 additions and 22 deletions
+44 -12
View File
@@ -12,6 +12,11 @@ from fastapi.middleware.cors import CORSMiddleware
from bot.data.db import Database
from bot.executor.paper import cash_available
from bot.metrics.sharpe import (
MIN_DAYS_OBSERVED,
MIN_RESOLVED_TRADES,
sharpe_with_gate,
)
# Phase 6 format (Phase 6+): values already in log-odds space.
# "fg_lo=+0.1200 mom_lo=+0.0000 news_lo=+0.0000 mfld_lo=-0.7483 btc_dom_lo=+0.0000"
@@ -280,23 +285,40 @@ async def get_summary():
PnL and performance metrics come from the latest metrics_daily snapshot,
which is written by the bot every cycle via MetricsTracker.update_daily_summary().
After Fix 3, that snapshot is also DB-computed — not dependent on pod restarts.
sharpe_ratio is the exception: it is recomputed live here from the daily
PnL-close series (same sharpe_with_gate the tracker uses), so the
explanation fields (sharpe_status, days_observed) always match the value.
"""
latest_metrics, counts, position_data, inverted, legacy_count = await asyncio.gather(
latest_metrics, counts, position_data, inverted, legacy_count, daily_closes = (
await asyncio.gather(
db.get_metrics_history(days=1),
db.compute_metrics_from_db(),
db.get_open_position_data(),
db.get_recently_closed_inverted(hours=24),
db.get_legacy_incomplete_count(),
db.get_daily_pnl_closes(),
)
)
latest = latest_metrics[0] if latest_metrics else {}
paper_bankroll = float(os.getenv("PAPER_BANKROLL", "10000"))
total_trades = int(counts["total_trades"] or 0)
resolved_count = int(counts.get("resolved_count") or 0)
# Same source PaperExecutor.initialize() uses to reconstruct cash:
# total_net_cost_open = SUM(net_cost) over open trades, uncapped.
_, total_net_cost_open = position_data
total_deployed = total_net_cost_open
# Sharpe: computed live from the daily PnL curve (same function the
# tracker uses for the snapshot). None + status while the minimum-sample
# gate (>=30 days observed, >=10 resolved trades) is not met — a Sharpe
# over 1 resolved trade is statistically meaningless.
days_observed = len(daily_closes)
sharpe, sharpe_status = sharpe_with_gate(daily_closes, paper_bankroll, resolved_count)
win_rate = latest.get("win_rate")
calibration = latest.get("calibration_score")
return {
# ── Portfolio state (live from DB) ──────────────────────────────────
"paper_mode": os.getenv("PAPER_MODE", "true") == "true",
@@ -319,25 +341,35 @@ async def get_summary():
"realized_pnl": latest.get("realized_pnl") or 0,
"total_pnl": latest.get("total_pnl") or 0,
# ── Performance metrics (from latest metrics_daily snapshot) ─────────
# ── Performance metrics ──────────────────────────────────────────────
# win_rate: fraction of resolved closed trades where close_pnl > 0.
# null if fewer than 5 resolved trades. Source: closed+resolved trades.
# sharpe_ratio: 0.0 — requires daily-return time series (not yet tracked).
# sharpe_ratio: annualized Sharpe of the daily total_pnl curve, computed
# live from metrics_daily. null while the minimum-sample gate fails
# (sharpe_status explains why). Source: bot/metrics/sharpe.py.
# calibration_score: 1 Brier score on resolved trades (higher = better).
# null if fewer than 10 resolved trades. Source: closed+resolved trades.
"win_rate": latest.get("win_rate"), # null if < 5 resolved
"sharpe_ratio": latest.get("sharpe_ratio") or 0, # 0.0 until tracked
"calibration_score": latest.get("calibration_score"), # null if < 10 resolved
"win_rate": win_rate, # null if < 5 resolved
"sharpe_ratio": sharpe, # null if gate fails
"sharpe_status": sharpe_status, # ok | insufficient_sample | zero_variance
"days_observed": days_observed,
"min_days_required": MIN_DAYS_OBSERVED,
"min_resolved_required": MIN_RESOLVED_TRADES,
"calibration_score": calibration, # null if < 10 resolved
# ── Counters from snapshot ───────────────────────────────────────────
"resolved_count": latest.get("resolved_count") or 0,
# ── Counters (live from DB) ──────────────────────────────────────────
"resolved_count": resolved_count,
# ── Promotion gate ───────────────────────────────────────────────────
# All thresholds must pass; null metrics count as not-ready.
# Never promote on a tiny sample: requires the resolved/days minimums
# AND non-null metrics AND all thresholds. A single lucky resolved
# trade must not flip this to true.
"promotion_ready": (
(latest.get("sharpe_ratio") or 0) >= 0.5
and (latest.get("win_rate") or 0) >= 0.52
and (latest.get("calibration_score") or 0) >= 0.7
resolved_count >= MIN_RESOLVED_TRADES
and days_observed >= MIN_DAYS_OBSERVED
and win_rate is not None and win_rate >= 0.52
and calibration is not None and calibration >= 0.7
and sharpe is not None and sharpe >= 0.5
and total_trades >= 50
),
}
+18
View File
@@ -348,6 +348,24 @@ class Database:
)
return [dict(r) for r in rows]
async def get_daily_pnl_closes(self) -> list[float]:
"""Return the closing total_pnl of every observed UTC day, oldest first.
One value per calendar day with at least one metrics_daily snapshot
(the day's last snapshot, same collapse rule as get_metrics_history).
This is the input series for the Sharpe ratio: len() = days observed,
consecutive deltas = daily PnL changes.
"""
async with self._pool.acquire() as conn:
rows = await conn.fetch(
"""
SELECT DISTINCT ON (timestamp::date) total_pnl
FROM metrics_daily
ORDER BY timestamp::date ASC, timestamp DESC
"""
)
return [float(r["total_pnl"] or 0) for r in rows]
async def backfill_feature_columns(self) -> int:
"""Back-populate feat_*_lo for trades created before Phase 6.
+79
View File
@@ -0,0 +1,79 @@
"""
Sharpe ratio from the paper portfolio's daily PnL curve, with a minimum-sample gate.
The input series is the closing total_pnl of each observed UTC day
(Database.get_daily_pnl_closes). Daily returns are PnL deltas normalized by
the paper bankroll:
r_t = (pnl_t pnl_{t1}) / bankroll
Sharpe = mean(r) / sample_std(r) × 365, annualized prediction markets
resolve every calendar day, so 365 is used instead of 252 trading days.
Risk-free rate is taken as 0.
Gate: with a tiny sample (e.g. 1 resolved trade over a flat curve plus one
+299 jump) any Sharpe value is statistically meaningless artificially huge
or tiny depending on where the jump lands. So no numeric Sharpe is exposed
until BOTH minimums are met:
days observed >= MIN_DAYS_OBSERVED (30)
resolved trades >= MIN_RESOLVED_TRADES (10)
Below either minimum the value is None with status "insufficient_sample".
A perfectly flat curve (zero variance) also yields None ("zero_variance"):
Sharpe is undefined there, not infinite.
"""
from statistics import mean, stdev
from typing import Optional
MIN_DAYS_OBSERVED = 30
MIN_RESOLVED_TRADES = 10
ANNUALIZATION_DAYS = 365
SHARPE_OK = "ok"
SHARPE_INSUFFICIENT = "insufficient_sample"
SHARPE_ZERO_VARIANCE = "zero_variance"
def daily_returns(daily_pnl_closes: list[float], bankroll: float) -> list[float]:
"""Bankroll-normalized day-over-day returns from a daily PnL-close series."""
return [
(curr - prev) / bankroll
for prev, curr in zip(daily_pnl_closes, daily_pnl_closes[1:])
]
def compute_sharpe(daily_pnl_closes: list[float], bankroll: float) -> Optional[float]:
"""Annualized Sharpe of the daily PnL curve, or None if undefined.
None when there are fewer than 2 returns (need 3+ daily closes) or the
return series has zero variance. No sample-size gate here see
sharpe_with_gate() for the exposed value.
"""
returns = daily_returns(daily_pnl_closes, bankroll)
if len(returns) < 2:
return None
sd = stdev(returns)
if sd == 0:
return None
return mean(returns) / sd * ANNUALIZATION_DAYS ** 0.5
def sharpe_with_gate(
daily_pnl_closes: list[float],
bankroll: float,
resolved_count: int,
) -> tuple[Optional[float], str]:
"""Return (sharpe, status) applying the minimum-sample gate.
status: "ok" sharpe is a meaningful float
"insufficient_sample" sample below minimums, sharpe is None
"zero_variance" sample OK but flat curve, sharpe is None
"""
days_observed = len(daily_pnl_closes)
if days_observed < MIN_DAYS_OBSERVED or resolved_count < MIN_RESOLVED_TRADES:
return None, SHARPE_INSUFFICIENT
sharpe = compute_sharpe(daily_pnl_closes, bankroll)
if sharpe is None:
return None, SHARPE_ZERO_VARIANCE
return sharpe, SHARPE_OK
+14 -3
View File
@@ -15,12 +15,16 @@ win_rate Fraction of resolved closed trades with close_pnl > 0.
NULL if fewer than 5 resolved trades.
calibration_score 1 AVG((final_prob resolution)²) on resolved trades.
Brier score (higher = better calibration). NULL if < 10 resolved.
sharpe_ratio 0.0 requires a daily-return time series, not yet tracked.
sharpe_ratio Annualized Sharpe of the daily total_pnl curve (see
bot/metrics/sharpe.py). NULL until the sample gate passes:
>= 30 days observed AND >= 10 resolved trades.
"""
import logging
import os
from datetime import datetime, UTC
from bot.data.db import Database
from bot.metrics.sharpe import sharpe_with_gate
log = logging.getLogger(__name__)
@@ -61,6 +65,12 @@ class MetricsTracker:
avg_edge = total_pnl / total_deployed if total_deployed > 0 else 0.0
# Sharpe: real value from the daily PnL curve, NULL while the sample
# gate (>=30 days observed, >=10 resolved) is not met.
bankroll = float(os.getenv("PAPER_BANKROLL", "10000"))
daily_closes = await self._db.get_daily_pnl_closes()
sharpe, sharpe_status = sharpe_with_gate(daily_closes, bankroll, resolved)
metrics = {
"timestamp": datetime.now(UTC),
"total_trades": int(raw["total_trades"]),
@@ -74,7 +84,7 @@ class MetricsTracker:
"total_pnl": total_pnl,
"win_rate": win_rate,
"avg_edge": avg_edge,
"sharpe_ratio": 0.0, # requires daily-return series (not yet tracked)
"sharpe_ratio": sharpe, # NULL until sample gate passes
"calibration_score": calibration,
"paper_mode": True,
}
@@ -83,9 +93,10 @@ class MetricsTracker:
log.info(
"Daily metrics | trades=%d (open=%d closed=%d resolved=%d) | "
"unrealized=$%.2f realized=$%.2f total=$%.2f | "
"win_rate=%s calibration=%s",
"win_rate=%s calibration=%s sharpe=%s",
metrics["total_trades"], open_count, closed_count, resolved,
unrealized, realized, total_pnl,
f"{win_rate:.1%}" if win_rate is not None else "n/a (<5)",
f"{calibration:.3f}" if calibration is not None else "n/a (<10)",
f"{sharpe:.2f}" if sharpe is not None else f"n/a ({sharpe_status})",
)
+6 -2
View File
@@ -200,8 +200,12 @@ export default function App() {
<MetricCard
title="Sharpe"
value={fmt(summary.sharpe_ratio)}
subtitle="Objetivo ≥ 0.5"
progress={Math.min(1, summary.sharpe_ratio / 2)}
subtitle={
summary.sharpe_ratio == null
? `Muestra insuficiente: ${summary.resolved_count}/${summary.min_resolved_required} resueltos, ${summary.days_observed}/${summary.min_days_required} días`
: 'Objetivo ≥ 0.5'
}
progress={summary.sharpe_ratio == null ? 0 : Math.min(1, summary.sharpe_ratio / 2)}
progressColor={summary.sharpe_ratio >= 0.5 ? 'var(--green)' : 'var(--amber)'}
/>
<MetricCard
+4
View File
@@ -44,6 +44,7 @@ class FakeDB:
"total_trades": self._total,
"open_count": self._open,
"closed_count": self._total - self._open,
"resolved_count": 0,
}
async def get_recently_closed_inverted(self, hours=24):
@@ -52,6 +53,9 @@ class FakeDB:
async def get_legacy_incomplete_count(self):
return 0
async def get_daily_pnl_closes(self):
return []
def _run(db: FakeDB, monkeypatch) -> tuple[dict, PaperExecutor]:
monkeypatch.setattr(api_main, "db", db)
+242
View File
@@ -0,0 +1,242 @@
"""
Tests for the real Sharpe ratio with minimum-sample gate.
Regression: sharpe_ratio was hardcoded to 0.0 in MetricsTracker and exposed
as `latest.get("sharpe_ratio") or 0` in /api/summary, and promotion_ready
could in principle flip on a statistically meaningless sample (e.g. 1
resolved trade over ~40 days of flat PnL plus a single +299 jump).
Fix: bot/metrics/sharpe.py computes an annualized Sharpe from the daily
total_pnl close series, gated to None ("insufficient_sample") below 30 days
observed / 10 resolved trades. /api/summary exposes the value plus an
explanation (sharpe_status, days_observed, min_* fields), and
promotion_ready additionally requires the sample minimums and non-null
metrics.
"""
import asyncio
from statistics import mean, stdev
import pytest
import api.main as api_main
from bot.metrics.sharpe import (
MIN_DAYS_OBSERVED,
MIN_RESOLVED_TRADES,
SHARPE_INSUFFICIENT,
SHARPE_OK,
SHARPE_ZERO_VARIANCE,
compute_sharpe,
daily_returns,
sharpe_with_gate,
)
from bot.metrics.tracker import MetricsTracker
BANKROLL = 10_000.0
def _closes_from_deltas(deltas: list[float], start: float = 0.0) -> list[float]:
closes = [start]
for d in deltas:
closes.append(closes[-1] + d)
return closes
# ── Pure computation ─────────────────────────────────────────────────────────
def test_daily_returns_are_bankroll_normalized_deltas():
closes = [0.0, 100.0, 50.0, 50.0]
assert daily_returns(closes, BANKROLL) == pytest.approx([0.01, -0.005, 0.0])
def test_compute_sharpe_matches_manual_formula():
deltas = [10.0, 14.0, 8.0, 12.0, 6.0, 13.0, 9.0]
closes = _closes_from_deltas(deltas)
rets = [d / BANKROLL for d in deltas]
expected = mean(rets) / stdev(rets) * 365 ** 0.5
assert compute_sharpe(closes, BANKROLL) == pytest.approx(expected)
assert compute_sharpe(closes, BANKROLL) > 0
def test_compute_sharpe_undefined_cases_return_none():
assert compute_sharpe([], BANKROLL) is None
assert compute_sharpe([0.0], BANKROLL) is None
assert compute_sharpe([0.0, 50.0], BANKROLL) is None # only 1 return
assert compute_sharpe([0.0] * 40, BANKROLL) is None # zero variance
# ── Minimum-sample gate ───────────────────────────────────────────────────────
def test_gate_blocks_current_situation_one_resolved_trade():
"""~40 flat days plus a single +299 jump, 1 resolved trade → no Sharpe."""
closes = [0.0] * 35 + [299.06] * 5
sharpe, status = sharpe_with_gate(closes, BANKROLL, resolved_count=1)
assert sharpe is None
assert status == SHARPE_INSUFFICIENT
# The raw (ungated) value would exist and be wildly misleading:
assert compute_sharpe(closes, BANKROLL) is not None
def test_gate_blocks_too_few_days_even_with_enough_resolved():
closes = _closes_from_deltas([10.0, -5.0] * 10) # 21 days < 30
sharpe, status = sharpe_with_gate(closes, BANKROLL, resolved_count=15)
assert sharpe is None
assert status == SHARPE_INSUFFICIENT
def test_gate_passes_with_sufficient_sample():
deltas = [10.0, 14.0, 8.0, 12.0, 6.0] * 8 # 40 returns → 41 days
closes = _closes_from_deltas(deltas)
sharpe, status = sharpe_with_gate(closes, BANKROLL, resolved_count=MIN_RESOLVED_TRADES)
assert status == SHARPE_OK
assert sharpe == pytest.approx(compute_sharpe(closes, BANKROLL))
def test_gate_flat_curve_with_sufficient_sample_is_zero_variance():
sharpe, status = sharpe_with_gate([0.0] * 40, BANKROLL, resolved_count=12)
assert sharpe is None
assert status == SHARPE_ZERO_VARIANCE
# ── /api/summary ─────────────────────────────────────────────────────────────
class FakeDB:
def __init__(self, daily_closes, resolved_count, total_trades=60,
win_rate=0.6, calibration=0.8):
self._closes = daily_closes
self._resolved = resolved_count
self._total = total_trades
self._win_rate = win_rate
self._calibration = calibration
async def get_metrics_history(self, days=1):
return [{
"win_rate": self._win_rate,
"calibration_score": self._calibration,
"unrealized_pnl_est": 0.0,
"realized_pnl": 299.06,
"total_pnl": 299.06,
}]
async def compute_metrics_from_db(self):
return {
"total_trades": self._total,
"open_count": self._total - self._resolved,
"closed_count": self._resolved,
"resolved_count": self._resolved,
}
async def get_open_position_data(self):
return {}, 0.0
async def get_recently_closed_inverted(self, hours=24):
return set()
async def get_legacy_incomplete_count(self):
return 0
async def get_daily_pnl_closes(self):
return list(self._closes)
def _summary(db, monkeypatch) -> dict:
monkeypatch.setattr(api_main, "db", db)
monkeypatch.delenv("PAPER_BANKROLL", raising=False)
return asyncio.run(api_main.get_summary())
def test_api_insufficient_sample_returns_null_with_explanation(monkeypatch):
"""Current prod situation: 1 resolved, ~40 days → null Sharpe, not ready."""
db = FakeDB(daily_closes=[0.0] * 35 + [299.06] * 5, resolved_count=1)
s = _summary(db, monkeypatch)
assert s["sharpe_ratio"] is None
assert s["sharpe_status"] == SHARPE_INSUFFICIENT
assert s["resolved_count"] == 1
assert s["min_resolved_required"] == MIN_RESOLVED_TRADES == 10
assert s["days_observed"] == 40
assert s["min_days_required"] == MIN_DAYS_OBSERVED == 30
# One lucky resolved trade must never promote, even with perfect
# win_rate/calibration and 50+ trades.
assert s["promotion_ready"] is False
def test_api_sharpe_appears_with_sufficient_sample(monkeypatch):
deltas = [10.0, 14.0, 8.0, 12.0, 6.0] * 8
db = FakeDB(daily_closes=_closes_from_deltas(deltas), resolved_count=12)
s = _summary(db, monkeypatch)
assert s["sharpe_status"] == SHARPE_OK
assert s["sharpe_ratio"] == pytest.approx(
compute_sharpe(_closes_from_deltas(deltas), BANKROLL)
)
assert s["sharpe_ratio"] >= 0.5
assert s["promotion_ready"] is True
def test_api_not_ready_when_sharpe_below_threshold(monkeypatch):
# Zero-drift curve: mean return ~0 → Sharpe ≈ 0 < 0.5
deltas = [50.0, -50.0] * 20
db = FakeDB(daily_closes=_closes_from_deltas(deltas), resolved_count=12)
s = _summary(db, monkeypatch)
assert s["sharpe_status"] == SHARPE_OK
assert s["sharpe_ratio"] < 0.5
assert s["promotion_ready"] is False
def test_api_not_ready_when_metrics_null(monkeypatch):
db = FakeDB(
daily_closes=_closes_from_deltas([10.0, 14.0, 8.0, 12.0, 6.0] * 8),
resolved_count=12,
win_rate=None,
calibration=None,
)
s = _summary(db, monkeypatch)
assert s["sharpe_status"] == SHARPE_OK
assert s["promotion_ready"] is False
# ── MetricsTracker: no hardcoded 0.0 in the snapshot ─────────────────────────
class FakeTrackerDB:
def __init__(self, daily_closes, resolved_count):
self._closes = daily_closes
self._resolved = resolved_count
self.saved = None
async def compute_metrics_from_db(self):
return {
"total_trades": 60,
"open_count": 40,
"closed_count": 20,
"resolved_count": self._resolved,
"wins_realized": self._resolved,
"unrealized_pnl_est": 0.0,
"realized_pnl": 100.0,
"total_deployed": 1000.0,
"total_fees": 20.0,
"calibration_score": 0.8,
}
async def get_daily_pnl_closes(self):
return list(self._closes)
async def save_daily_metrics(self, metrics):
self.saved = metrics
def test_tracker_stores_null_sharpe_below_gate(monkeypatch):
monkeypatch.delenv("PAPER_BANKROLL", raising=False)
db = FakeTrackerDB(daily_closes=[0.0] * 35 + [299.06] * 5, resolved_count=1)
asyncio.run(MetricsTracker(db).update_daily_summary())
assert db.saved is not None
assert db.saved["sharpe_ratio"] is None
def test_tracker_stores_real_sharpe_above_gate(monkeypatch):
monkeypatch.delenv("PAPER_BANKROLL", raising=False)
closes = _closes_from_deltas([10.0, 14.0, 8.0, 12.0, 6.0] * 8)
db = FakeTrackerDB(daily_closes=closes, resolved_count=12)
asyncio.run(MetricsTracker(db).update_daily_summary())
assert db.saved["sharpe_ratio"] == pytest.approx(
compute_sharpe(closes, BANKROLL)
)
assert db.saved["sharpe_ratio"] != 0.0