feat(metrics): real Sharpe ratio from daily PnL curve with minimum-sample gate
CI/CD / build-and-push (push) Successful in 10s
CI/CD / build-and-push (push) Successful in 10s
sharpe_ratio was hardcoded to 0.0 in MetricsTracker and exposed as 'or 0' in /api/summary. With only 1 resolved trade (~40 flat days plus one +299 jump) any computed Sharpe is statistically meaningless, so: - bot/metrics/sharpe.py: annualized Sharpe (sqrt(365)) from daily total_pnl closes, normalized by bankroll; sharpe_with_gate() returns None + status until >=30 days observed AND >=10 resolved trades. - Database.get_daily_pnl_closes(): last metrics_daily snapshot per UTC day, oldest first — the return series input. - MetricsTracker: stores the real (gated) Sharpe in the snapshot, NULL below the gate; log line now includes sharpe. - /api/summary: live Sharpe + sharpe_status/days_observed/min_* fields explaining why it is null; resolved_count now live from COUNT(*). - promotion_ready: requires resolved>=10, days>=30, and non-null win_rate/calibration/sharpe plus existing thresholds — a single lucky resolved trade can no longer promote. - Dashboard Sharpe card shows the insufficient-sample explanation when null instead of a bare em dash. Tests: 13 new in tests/test_sharpe_gate.py (formula, gate, API contract, tracker snapshot); verified failing pre-fix. Suite: 62 passed. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
1797b79f7b
commit
43d9577fb2
@@ -348,6 +348,24 @@ class Database:
|
||||
)
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
async def get_daily_pnl_closes(self) -> list[float]:
|
||||
"""Return the closing total_pnl of every observed UTC day, oldest first.
|
||||
|
||||
One value per calendar day with at least one metrics_daily snapshot
|
||||
(the day's last snapshot, same collapse rule as get_metrics_history).
|
||||
This is the input series for the Sharpe ratio: len() = days observed,
|
||||
consecutive deltas = daily PnL changes.
|
||||
"""
|
||||
async with self._pool.acquire() as conn:
|
||||
rows = await conn.fetch(
|
||||
"""
|
||||
SELECT DISTINCT ON (timestamp::date) total_pnl
|
||||
FROM metrics_daily
|
||||
ORDER BY timestamp::date ASC, timestamp DESC
|
||||
"""
|
||||
)
|
||||
return [float(r["total_pnl"] or 0) for r in rows]
|
||||
|
||||
async def backfill_feature_columns(self) -> int:
|
||||
"""Back-populate feat_*_lo for trades created before Phase 6.
|
||||
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
"""
|
||||
Sharpe ratio from the paper portfolio's daily PnL curve, with a minimum-sample gate.
|
||||
|
||||
The input series is the closing total_pnl of each observed UTC day
|
||||
(Database.get_daily_pnl_closes). Daily returns are PnL deltas normalized by
|
||||
the paper bankroll:
|
||||
|
||||
r_t = (pnl_t − pnl_{t−1}) / bankroll
|
||||
|
||||
Sharpe = mean(r) / sample_std(r) × √365, annualized — prediction markets
|
||||
resolve every calendar day, so 365 is used instead of 252 trading days.
|
||||
Risk-free rate is taken as 0.
|
||||
|
||||
Gate: with a tiny sample (e.g. 1 resolved trade over a flat curve plus one
|
||||
+299 jump) any Sharpe value is statistically meaningless — artificially huge
|
||||
or tiny depending on where the jump lands. So no numeric Sharpe is exposed
|
||||
until BOTH minimums are met:
|
||||
|
||||
days observed >= MIN_DAYS_OBSERVED (30)
|
||||
resolved trades >= MIN_RESOLVED_TRADES (10)
|
||||
|
||||
Below either minimum the value is None with status "insufficient_sample".
|
||||
A perfectly flat curve (zero variance) also yields None ("zero_variance"):
|
||||
Sharpe is undefined there, not infinite.
|
||||
"""
|
||||
from statistics import mean, stdev
|
||||
from typing import Optional
|
||||
|
||||
MIN_DAYS_OBSERVED = 30
|
||||
MIN_RESOLVED_TRADES = 10
|
||||
ANNUALIZATION_DAYS = 365
|
||||
|
||||
SHARPE_OK = "ok"
|
||||
SHARPE_INSUFFICIENT = "insufficient_sample"
|
||||
SHARPE_ZERO_VARIANCE = "zero_variance"
|
||||
|
||||
|
||||
def daily_returns(daily_pnl_closes: list[float], bankroll: float) -> list[float]:
|
||||
"""Bankroll-normalized day-over-day returns from a daily PnL-close series."""
|
||||
return [
|
||||
(curr - prev) / bankroll
|
||||
for prev, curr in zip(daily_pnl_closes, daily_pnl_closes[1:])
|
||||
]
|
||||
|
||||
|
||||
def compute_sharpe(daily_pnl_closes: list[float], bankroll: float) -> Optional[float]:
|
||||
"""Annualized Sharpe of the daily PnL curve, or None if undefined.
|
||||
|
||||
None when there are fewer than 2 returns (need 3+ daily closes) or the
|
||||
return series has zero variance. No sample-size gate here — see
|
||||
sharpe_with_gate() for the exposed value.
|
||||
"""
|
||||
returns = daily_returns(daily_pnl_closes, bankroll)
|
||||
if len(returns) < 2:
|
||||
return None
|
||||
sd = stdev(returns)
|
||||
if sd == 0:
|
||||
return None
|
||||
return mean(returns) / sd * ANNUALIZATION_DAYS ** 0.5
|
||||
|
||||
|
||||
def sharpe_with_gate(
|
||||
daily_pnl_closes: list[float],
|
||||
bankroll: float,
|
||||
resolved_count: int,
|
||||
) -> tuple[Optional[float], str]:
|
||||
"""Return (sharpe, status) applying the minimum-sample gate.
|
||||
|
||||
status: "ok" — sharpe is a meaningful float
|
||||
"insufficient_sample" — sample below minimums, sharpe is None
|
||||
"zero_variance" — sample OK but flat curve, sharpe is None
|
||||
"""
|
||||
days_observed = len(daily_pnl_closes)
|
||||
if days_observed < MIN_DAYS_OBSERVED or resolved_count < MIN_RESOLVED_TRADES:
|
||||
return None, SHARPE_INSUFFICIENT
|
||||
sharpe = compute_sharpe(daily_pnl_closes, bankroll)
|
||||
if sharpe is None:
|
||||
return None, SHARPE_ZERO_VARIANCE
|
||||
return sharpe, SHARPE_OK
|
||||
+14
-3
@@ -15,12 +15,16 @@ win_rate Fraction of resolved closed trades with close_pnl > 0.
|
||||
NULL if fewer than 5 resolved trades.
|
||||
calibration_score 1 − AVG((final_prob − resolution)²) on resolved trades.
|
||||
Brier score (higher = better calibration). NULL if < 10 resolved.
|
||||
sharpe_ratio 0.0 — requires a daily-return time series, not yet tracked.
|
||||
sharpe_ratio Annualized Sharpe of the daily total_pnl curve (see
|
||||
bot/metrics/sharpe.py). NULL until the sample gate passes:
|
||||
>= 30 days observed AND >= 10 resolved trades.
|
||||
"""
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime, UTC
|
||||
|
||||
from bot.data.db import Database
|
||||
from bot.metrics.sharpe import sharpe_with_gate
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -61,6 +65,12 @@ class MetricsTracker:
|
||||
|
||||
avg_edge = total_pnl / total_deployed if total_deployed > 0 else 0.0
|
||||
|
||||
# Sharpe: real value from the daily PnL curve, NULL while the sample
|
||||
# gate (>=30 days observed, >=10 resolved) is not met.
|
||||
bankroll = float(os.getenv("PAPER_BANKROLL", "10000"))
|
||||
daily_closes = await self._db.get_daily_pnl_closes()
|
||||
sharpe, sharpe_status = sharpe_with_gate(daily_closes, bankroll, resolved)
|
||||
|
||||
metrics = {
|
||||
"timestamp": datetime.now(UTC),
|
||||
"total_trades": int(raw["total_trades"]),
|
||||
@@ -74,7 +84,7 @@ class MetricsTracker:
|
||||
"total_pnl": total_pnl,
|
||||
"win_rate": win_rate,
|
||||
"avg_edge": avg_edge,
|
||||
"sharpe_ratio": 0.0, # requires daily-return series (not yet tracked)
|
||||
"sharpe_ratio": sharpe, # NULL until sample gate passes
|
||||
"calibration_score": calibration,
|
||||
"paper_mode": True,
|
||||
}
|
||||
@@ -83,9 +93,10 @@ class MetricsTracker:
|
||||
log.info(
|
||||
"Daily metrics | trades=%d (open=%d closed=%d resolved=%d) | "
|
||||
"unrealized=$%.2f realized=$%.2f total=$%.2f | "
|
||||
"win_rate=%s calibration=%s",
|
||||
"win_rate=%s calibration=%s sharpe=%s",
|
||||
metrics["total_trades"], open_count, closed_count, resolved,
|
||||
unrealized, realized, total_pnl,
|
||||
f"{win_rate:.1%}" if win_rate is not None else "n/a (<5)",
|
||||
f"{calibration:.3f}" if calibration is not None else "n/a (<10)",
|
||||
f"{sharpe:.2f}" if sharpe is not None else f"n/a ({sharpe_status})",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user