feat(metrics): real Sharpe ratio from daily PnL curve with minimum-sample gate
CI/CD / build-and-push (push) Successful in 10s
CI/CD / build-and-push (push) Successful in 10s
sharpe_ratio was hardcoded to 0.0 in MetricsTracker and exposed as 'or 0' in /api/summary. With only 1 resolved trade (~40 flat days plus one +299 jump) any computed Sharpe is statistically meaningless, so: - bot/metrics/sharpe.py: annualized Sharpe (sqrt(365)) from daily total_pnl closes, normalized by bankroll; sharpe_with_gate() returns None + status until >=30 days observed AND >=10 resolved trades. - Database.get_daily_pnl_closes(): last metrics_daily snapshot per UTC day, oldest first — the return series input. - MetricsTracker: stores the real (gated) Sharpe in the snapshot, NULL below the gate; log line now includes sharpe. - /api/summary: live Sharpe + sharpe_status/days_observed/min_* fields explaining why it is null; resolved_count now live from COUNT(*). - promotion_ready: requires resolved>=10, days>=30, and non-null win_rate/calibration/sharpe plus existing thresholds — a single lucky resolved trade can no longer promote. - Dashboard Sharpe card shows the insufficient-sample explanation when null instead of a bare em dash. Tests: 13 new in tests/test_sharpe_gate.py (formula, gate, API contract, tracker snapshot); verified failing pre-fix. Suite: 62 passed. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
1797b79f7b
commit
43d9577fb2
@@ -44,6 +44,7 @@ class FakeDB:
|
||||
"total_trades": self._total,
|
||||
"open_count": self._open,
|
||||
"closed_count": self._total - self._open,
|
||||
"resolved_count": 0,
|
||||
}
|
||||
|
||||
async def get_recently_closed_inverted(self, hours=24):
|
||||
@@ -52,6 +53,9 @@ class FakeDB:
|
||||
async def get_legacy_incomplete_count(self):
|
||||
return 0
|
||||
|
||||
async def get_daily_pnl_closes(self):
|
||||
return []
|
||||
|
||||
|
||||
def _run(db: FakeDB, monkeypatch) -> tuple[dict, PaperExecutor]:
|
||||
monkeypatch.setattr(api_main, "db", db)
|
||||
|
||||
@@ -0,0 +1,242 @@
|
||||
"""
|
||||
Tests for the real Sharpe ratio with minimum-sample gate.
|
||||
|
||||
Regression: sharpe_ratio was hardcoded to 0.0 in MetricsTracker and exposed
|
||||
as `latest.get("sharpe_ratio") or 0` in /api/summary, and promotion_ready
|
||||
could in principle flip on a statistically meaningless sample (e.g. 1
|
||||
resolved trade over ~40 days of flat PnL plus a single +299 jump).
|
||||
|
||||
Fix: bot/metrics/sharpe.py computes an annualized Sharpe from the daily
|
||||
total_pnl close series, gated to None ("insufficient_sample") below 30 days
|
||||
observed / 10 resolved trades. /api/summary exposes the value plus an
|
||||
explanation (sharpe_status, days_observed, min_* fields), and
|
||||
promotion_ready additionally requires the sample minimums and non-null
|
||||
metrics.
|
||||
"""
|
||||
import asyncio
|
||||
from statistics import mean, stdev
|
||||
|
||||
import pytest
|
||||
|
||||
import api.main as api_main
|
||||
from bot.metrics.sharpe import (
|
||||
MIN_DAYS_OBSERVED,
|
||||
MIN_RESOLVED_TRADES,
|
||||
SHARPE_INSUFFICIENT,
|
||||
SHARPE_OK,
|
||||
SHARPE_ZERO_VARIANCE,
|
||||
compute_sharpe,
|
||||
daily_returns,
|
||||
sharpe_with_gate,
|
||||
)
|
||||
from bot.metrics.tracker import MetricsTracker
|
||||
|
||||
|
||||
BANKROLL = 10_000.0
|
||||
|
||||
|
||||
def _closes_from_deltas(deltas: list[float], start: float = 0.0) -> list[float]:
|
||||
closes = [start]
|
||||
for d in deltas:
|
||||
closes.append(closes[-1] + d)
|
||||
return closes
|
||||
|
||||
|
||||
# ── Pure computation ─────────────────────────────────────────────────────────
|
||||
|
||||
def test_daily_returns_are_bankroll_normalized_deltas():
|
||||
closes = [0.0, 100.0, 50.0, 50.0]
|
||||
assert daily_returns(closes, BANKROLL) == pytest.approx([0.01, -0.005, 0.0])
|
||||
|
||||
|
||||
def test_compute_sharpe_matches_manual_formula():
|
||||
deltas = [10.0, 14.0, 8.0, 12.0, 6.0, 13.0, 9.0]
|
||||
closes = _closes_from_deltas(deltas)
|
||||
rets = [d / BANKROLL for d in deltas]
|
||||
expected = mean(rets) / stdev(rets) * 365 ** 0.5
|
||||
assert compute_sharpe(closes, BANKROLL) == pytest.approx(expected)
|
||||
assert compute_sharpe(closes, BANKROLL) > 0
|
||||
|
||||
|
||||
def test_compute_sharpe_undefined_cases_return_none():
|
||||
assert compute_sharpe([], BANKROLL) is None
|
||||
assert compute_sharpe([0.0], BANKROLL) is None
|
||||
assert compute_sharpe([0.0, 50.0], BANKROLL) is None # only 1 return
|
||||
assert compute_sharpe([0.0] * 40, BANKROLL) is None # zero variance
|
||||
|
||||
|
||||
# ── Minimum-sample gate ───────────────────────────────────────────────────────
|
||||
|
||||
def test_gate_blocks_current_situation_one_resolved_trade():
|
||||
"""~40 flat days plus a single +299 jump, 1 resolved trade → no Sharpe."""
|
||||
closes = [0.0] * 35 + [299.06] * 5
|
||||
sharpe, status = sharpe_with_gate(closes, BANKROLL, resolved_count=1)
|
||||
assert sharpe is None
|
||||
assert status == SHARPE_INSUFFICIENT
|
||||
# The raw (ungated) value would exist and be wildly misleading:
|
||||
assert compute_sharpe(closes, BANKROLL) is not None
|
||||
|
||||
|
||||
def test_gate_blocks_too_few_days_even_with_enough_resolved():
|
||||
closes = _closes_from_deltas([10.0, -5.0] * 10) # 21 days < 30
|
||||
sharpe, status = sharpe_with_gate(closes, BANKROLL, resolved_count=15)
|
||||
assert sharpe is None
|
||||
assert status == SHARPE_INSUFFICIENT
|
||||
|
||||
|
||||
def test_gate_passes_with_sufficient_sample():
|
||||
deltas = [10.0, 14.0, 8.0, 12.0, 6.0] * 8 # 40 returns → 41 days
|
||||
closes = _closes_from_deltas(deltas)
|
||||
sharpe, status = sharpe_with_gate(closes, BANKROLL, resolved_count=MIN_RESOLVED_TRADES)
|
||||
assert status == SHARPE_OK
|
||||
assert sharpe == pytest.approx(compute_sharpe(closes, BANKROLL))
|
||||
|
||||
|
||||
def test_gate_flat_curve_with_sufficient_sample_is_zero_variance():
|
||||
sharpe, status = sharpe_with_gate([0.0] * 40, BANKROLL, resolved_count=12)
|
||||
assert sharpe is None
|
||||
assert status == SHARPE_ZERO_VARIANCE
|
||||
|
||||
|
||||
# ── /api/summary ─────────────────────────────────────────────────────────────
|
||||
|
||||
class FakeDB:
|
||||
def __init__(self, daily_closes, resolved_count, total_trades=60,
|
||||
win_rate=0.6, calibration=0.8):
|
||||
self._closes = daily_closes
|
||||
self._resolved = resolved_count
|
||||
self._total = total_trades
|
||||
self._win_rate = win_rate
|
||||
self._calibration = calibration
|
||||
|
||||
async def get_metrics_history(self, days=1):
|
||||
return [{
|
||||
"win_rate": self._win_rate,
|
||||
"calibration_score": self._calibration,
|
||||
"unrealized_pnl_est": 0.0,
|
||||
"realized_pnl": 299.06,
|
||||
"total_pnl": 299.06,
|
||||
}]
|
||||
|
||||
async def compute_metrics_from_db(self):
|
||||
return {
|
||||
"total_trades": self._total,
|
||||
"open_count": self._total - self._resolved,
|
||||
"closed_count": self._resolved,
|
||||
"resolved_count": self._resolved,
|
||||
}
|
||||
|
||||
async def get_open_position_data(self):
|
||||
return {}, 0.0
|
||||
|
||||
async def get_recently_closed_inverted(self, hours=24):
|
||||
return set()
|
||||
|
||||
async def get_legacy_incomplete_count(self):
|
||||
return 0
|
||||
|
||||
async def get_daily_pnl_closes(self):
|
||||
return list(self._closes)
|
||||
|
||||
|
||||
def _summary(db, monkeypatch) -> dict:
|
||||
monkeypatch.setattr(api_main, "db", db)
|
||||
monkeypatch.delenv("PAPER_BANKROLL", raising=False)
|
||||
return asyncio.run(api_main.get_summary())
|
||||
|
||||
|
||||
def test_api_insufficient_sample_returns_null_with_explanation(monkeypatch):
|
||||
"""Current prod situation: 1 resolved, ~40 days → null Sharpe, not ready."""
|
||||
db = FakeDB(daily_closes=[0.0] * 35 + [299.06] * 5, resolved_count=1)
|
||||
s = _summary(db, monkeypatch)
|
||||
assert s["sharpe_ratio"] is None
|
||||
assert s["sharpe_status"] == SHARPE_INSUFFICIENT
|
||||
assert s["resolved_count"] == 1
|
||||
assert s["min_resolved_required"] == MIN_RESOLVED_TRADES == 10
|
||||
assert s["days_observed"] == 40
|
||||
assert s["min_days_required"] == MIN_DAYS_OBSERVED == 30
|
||||
# One lucky resolved trade must never promote, even with perfect
|
||||
# win_rate/calibration and 50+ trades.
|
||||
assert s["promotion_ready"] is False
|
||||
|
||||
|
||||
def test_api_sharpe_appears_with_sufficient_sample(monkeypatch):
|
||||
deltas = [10.0, 14.0, 8.0, 12.0, 6.0] * 8
|
||||
db = FakeDB(daily_closes=_closes_from_deltas(deltas), resolved_count=12)
|
||||
s = _summary(db, monkeypatch)
|
||||
assert s["sharpe_status"] == SHARPE_OK
|
||||
assert s["sharpe_ratio"] == pytest.approx(
|
||||
compute_sharpe(_closes_from_deltas(deltas), BANKROLL)
|
||||
)
|
||||
assert s["sharpe_ratio"] >= 0.5
|
||||
assert s["promotion_ready"] is True
|
||||
|
||||
|
||||
def test_api_not_ready_when_sharpe_below_threshold(monkeypatch):
|
||||
# Zero-drift curve: mean return ~0 → Sharpe ≈ 0 < 0.5
|
||||
deltas = [50.0, -50.0] * 20
|
||||
db = FakeDB(daily_closes=_closes_from_deltas(deltas), resolved_count=12)
|
||||
s = _summary(db, monkeypatch)
|
||||
assert s["sharpe_status"] == SHARPE_OK
|
||||
assert s["sharpe_ratio"] < 0.5
|
||||
assert s["promotion_ready"] is False
|
||||
|
||||
|
||||
def test_api_not_ready_when_metrics_null(monkeypatch):
|
||||
db = FakeDB(
|
||||
daily_closes=_closes_from_deltas([10.0, 14.0, 8.0, 12.0, 6.0] * 8),
|
||||
resolved_count=12,
|
||||
win_rate=None,
|
||||
calibration=None,
|
||||
)
|
||||
s = _summary(db, monkeypatch)
|
||||
assert s["sharpe_status"] == SHARPE_OK
|
||||
assert s["promotion_ready"] is False
|
||||
|
||||
|
||||
# ── MetricsTracker: no hardcoded 0.0 in the snapshot ─────────────────────────
|
||||
|
||||
class FakeTrackerDB:
|
||||
def __init__(self, daily_closes, resolved_count):
|
||||
self._closes = daily_closes
|
||||
self._resolved = resolved_count
|
||||
self.saved = None
|
||||
|
||||
async def compute_metrics_from_db(self):
|
||||
return {
|
||||
"total_trades": 60,
|
||||
"open_count": 40,
|
||||
"closed_count": 20,
|
||||
"resolved_count": self._resolved,
|
||||
"wins_realized": self._resolved,
|
||||
"unrealized_pnl_est": 0.0,
|
||||
"realized_pnl": 100.0,
|
||||
"total_deployed": 1000.0,
|
||||
"total_fees": 20.0,
|
||||
"calibration_score": 0.8,
|
||||
}
|
||||
|
||||
async def get_daily_pnl_closes(self):
|
||||
return list(self._closes)
|
||||
|
||||
async def save_daily_metrics(self, metrics):
|
||||
self.saved = metrics
|
||||
|
||||
|
||||
def test_tracker_stores_null_sharpe_below_gate(monkeypatch):
|
||||
monkeypatch.delenv("PAPER_BANKROLL", raising=False)
|
||||
db = FakeTrackerDB(daily_closes=[0.0] * 35 + [299.06] * 5, resolved_count=1)
|
||||
asyncio.run(MetricsTracker(db).update_daily_summary())
|
||||
assert db.saved is not None
|
||||
assert db.saved["sharpe_ratio"] is None
|
||||
|
||||
|
||||
def test_tracker_stores_real_sharpe_above_gate(monkeypatch):
|
||||
monkeypatch.delenv("PAPER_BANKROLL", raising=False)
|
||||
closes = _closes_from_deltas([10.0, 14.0, 8.0, 12.0, 6.0] * 8)
|
||||
db = FakeTrackerDB(daily_closes=closes, resolved_count=12)
|
||||
asyncio.run(MetricsTracker(db).update_daily_summary())
|
||||
assert db.saved["sharpe_ratio"] == pytest.approx(
|
||||
compute_sharpe(closes, BANKROLL)
|
||||
)
|
||||
assert db.saved["sharpe_ratio"] != 0.0
|
||||
Reference in New Issue
Block a user