chemavxandClaude Fable 5
43d9577fb2
CI/CD / build-and-push (push) Successful in 10s
sharpe_ratio was hardcoded to 0.0 in MetricsTracker and exposed as 'or 0' in /api/summary. With only 1 resolved trade (~40 flat days plus one +299 jump) any computed Sharpe is statistically meaningless, so: - bot/metrics/sharpe.py: annualized Sharpe (sqrt(365)) from daily total_pnl closes, normalized by bankroll; sharpe_with_gate() returns None + status until >=30 days observed AND >=10 resolved trades. - Database.get_daily_pnl_closes(): last metrics_daily snapshot per UTC day, oldest first — the return series input. - MetricsTracker: stores the real (gated) Sharpe in the snapshot, NULL below the gate; log line now includes sharpe. - /api/summary: live Sharpe + sharpe_status/days_observed/min_* fields explaining why it is null; resolved_count now live from COUNT(*). - promotion_ready: requires resolved>=10, days>=30, and non-null win_rate/calibration/sharpe plus existing thresholds — a single lucky resolved trade can no longer promote. - Dashboard Sharpe card shows the insufficient-sample explanation when null instead of a bare em dash. Tests: 13 new in tests/test_sharpe_gate.py (formula, gate, API contract, tracker snapshot); verified failing pre-fix. Suite: 62 passed. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
376 lines
17 KiB
Python
376 lines
17 KiB
Python
"""
|
||
FastAPI Backend — serves metrics and trade data to the React dashboard.
|
||
"""
|
||
import asyncio
|
||
from contextlib import asynccontextmanager
|
||
from datetime import datetime, timezone
|
||
import os
|
||
import re
|
||
|
||
from fastapi import FastAPI
|
||
from fastapi.middleware.cors import CORSMiddleware
|
||
|
||
from bot.data.db import Database
|
||
from bot.executor.paper import cash_available
|
||
from bot.metrics.sharpe import (
|
||
MIN_DAYS_OBSERVED,
|
||
MIN_RESOLVED_TRADES,
|
||
sharpe_with_gate,
|
||
)
|
||
|
||
# Phase 6 format (Phase 6+): values already in log-odds space.
|
||
# "fg_lo=+0.1200 mom_lo=+0.0000 news_lo=+0.0000 mfld_lo=-0.7483 btc_dom_lo=+0.0000"
|
||
_FEAT_RE_LO = re.compile(
|
||
r"fg_lo=([+-]?[\d.]+).*?mom_lo=([+-]?[\d.]+).*?"
|
||
r"news_lo=([+-]?[\d.]+).*?mfld_lo=([+-]?[\d.]+).*?btc_dom_lo=([+-]?[\d.]+)"
|
||
)
|
||
|
||
# Pre-Phase-6 format: raw probability-delta values (fg/mom need ×2 for log-odds).
|
||
# "fg=+0.0600 mom=+0.0000 news=+0.0000 mfld=-0.7483"
|
||
_FEAT_RE_RAW = re.compile(
|
||
r"fg=([+-]?[\d.]+).*?mom=([+-]?[\d.]+).*?news=([+-]?[\d.]+).*?mfld=([+-]?[\d.]+)"
|
||
)
|
||
|
||
|
||
def _dominant_feature(sc: dict | None) -> str | None:
|
||
"""Return the name of the signal_components key with the largest abs log-odds value.
|
||
|
||
Returns None if signal_components is absent or all features are below threshold.
|
||
Threshold 0.0001 matches the "triggered" definition used in /api/metrics/features.
|
||
"""
|
||
if not sc:
|
||
return None
|
||
candidates = {
|
||
k: abs(v)
|
||
for k, v in sc.items()
|
||
if k != "unit" and v is not None and abs(v) > 0.0001
|
||
}
|
||
if not candidates:
|
||
return None
|
||
return max(candidates, key=candidates.__getitem__)
|
||
|
||
|
||
def _enrich_trade(trade: dict) -> dict:
|
||
"""Add days_open, signal_components (log-odds), and dominant_feature to a trade."""
|
||
ts = trade.get("timestamp")
|
||
if ts is not None:
|
||
now = datetime.now(timezone.utc)
|
||
if getattr(ts, "tzinfo", None) is None:
|
||
ts = ts.replace(tzinfo=timezone.utc)
|
||
trade["days_open"] = round((now - ts).total_seconds() / 86400, 1)
|
||
else:
|
||
trade["days_open"] = None
|
||
|
||
# Prefer DB columns (Phase 6+) — exact, no parsing required.
|
||
if trade.get("feat_fg_lo") is not None:
|
||
sc = {
|
||
"unit": "log_odds",
|
||
"fg": trade["feat_fg_lo"],
|
||
"mom": trade["feat_mom_lo"],
|
||
"news": trade["feat_news_lo"],
|
||
"mfld": trade["feat_mfld_lo"],
|
||
"btc_dom": trade.get("feat_btc_dom_lo"),
|
||
}
|
||
else:
|
||
# Fallback: parse reasoning string (trades before Phase 6 DB columns exist).
|
||
reasoning = trade.get("reasoning") or ""
|
||
m_lo = _FEAT_RE_LO.search(reasoning)
|
||
m_raw = _FEAT_RE_RAW.search(reasoning)
|
||
if m_lo:
|
||
sc = {
|
||
"unit": "log_odds",
|
||
"fg": float(m_lo.group(1)),
|
||
"mom": float(m_lo.group(2)),
|
||
"news": float(m_lo.group(3)),
|
||
"mfld": float(m_lo.group(4)),
|
||
"btc_dom": float(m_lo.group(5)),
|
||
}
|
||
elif m_raw:
|
||
# Pre-Phase-6: fg/mom are raw probability-deltas → multiply ×2.
|
||
sc = {
|
||
"unit": "log_odds",
|
||
"fg": float(m_raw.group(1)) * 2,
|
||
"mom": float(m_raw.group(2)) * 2,
|
||
"news": float(m_raw.group(3)),
|
||
"mfld": float(m_raw.group(4)),
|
||
"btc_dom": None,
|
||
}
|
||
else:
|
||
sc = None
|
||
|
||
trade["signal_components"] = sc
|
||
trade["dominant_feature"] = _dominant_feature(sc)
|
||
return trade
|
||
|
||
db = Database()
|
||
|
||
|
||
@asynccontextmanager
|
||
async def lifespan(app: FastAPI):
|
||
await db.connect()
|
||
yield
|
||
await db.disconnect()
|
||
|
||
|
||
app = FastAPI(title="Polymarket Bot API", lifespan=lifespan)
|
||
|
||
app.add_middleware(
|
||
CORSMiddleware,
|
||
allow_origins=["*"],
|
||
allow_methods=["GET"],
|
||
allow_headers=["*"],
|
||
)
|
||
|
||
|
||
@app.get("/health")
|
||
async def health():
|
||
return {"status": "ok", "paper_mode": os.getenv("PAPER_MODE", "true")}
|
||
|
||
|
||
@app.get("/api/metrics")
|
||
async def get_metrics():
|
||
history = await db.get_metrics_history(days=42)
|
||
if not history:
|
||
return {"history": [], "latest": None}
|
||
return {"history": history, "latest": history[0]}
|
||
|
||
|
||
@app.get("/api/trades")
|
||
async def get_trades(limit: int = 50, status: str = "open"):
|
||
"""
|
||
status: "open" (default) | "closed" | "all"
|
||
Open trades include days_open and signal_components {fg, mom, news, mfld}.
|
||
"""
|
||
if status not in ("open", "closed", "all"):
|
||
status = "open"
|
||
filter_status = None if status == "all" else status
|
||
trades = await db.get_recent_trades(limit=limit, status=filter_status)
|
||
if filter_status == "open":
|
||
trades = [_enrich_trade(t) for t in trades]
|
||
return {"trades": trades, "count": len(trades), "status_filter": status}
|
||
|
||
|
||
@app.get("/api/metrics/features")
|
||
async def get_feature_metrics():
|
||
"""Per-signal-feature performance breakdown — all values in log-odds space.
|
||
|
||
Each feature key contains:
|
||
unit "log_odds" (common unit for all features)
|
||
materiality_threshold |lo| threshold for "material" classification
|
||
triggered_count trades where |feat_lo| > 0.0001 (signal fired)
|
||
material_count trades where |feat_lo| >= threshold (moved the model)
|
||
avg_contribution_lo mean signed contribution (triggered trades)
|
||
avg_abs_contribution_lo mean absolute contribution (triggered trades)
|
||
avg_edge_net_when_material mean edge_net for material trades
|
||
unrealized_pnl_est estimated open-position PnL (triggered trades)
|
||
realized_pnl sum close_pnl for resolved triggered trades
|
||
resolved_count closed triggered trades with known outcome
|
||
win_rate null if resolved_count < 5
|
||
net_positive_count triggered trades where feature pushed BUY direction
|
||
net_negative_count triggered trades where feature pushed SELL direction
|
||
|
||
NULL values in resolved_count / win_rate are expected early in the paper run.
|
||
"""
|
||
features = await db.compute_feature_metrics_from_db()
|
||
return {"features": features}
|
||
|
||
|
||
@app.get("/api/trades/legacy")
|
||
async def get_legacy_trades():
|
||
"""Trades with NULL edge_net — pre-Phase-1 data, excluded from PnL estimates.
|
||
|
||
These trades have no signal quality information (edge_net, final_prob)
|
||
and are excluded from unrealized_pnl_est in /api/summary.
|
||
They may also be missing feat_*_lo columns if the reasoning string
|
||
predates the Phase 6 format.
|
||
"""
|
||
trades = await db.get_legacy_incomplete_trades()
|
||
return {"trades": trades, "count": len(trades)}
|
||
|
||
|
||
@app.get("/api/metrics/attribution")
|
||
async def get_attribution():
|
||
"""Alpha attribution by dominant signal feature.
|
||
|
||
Groups Phase 6 trades by their dominant feature — the feat_*_lo with the
|
||
largest absolute log-odds value — to reveal which signal is actually
|
||
generating opportunities.
|
||
|
||
Each key in "attribution" contains:
|
||
trade_count trades where this feature was dominant
|
||
avg_edge_net mean net edge for those trades
|
||
unrealized_pnl_est open-position PnL estimate (edge_net × net_cost − fee)
|
||
realized_pnl sum close_pnl for resolved trades in this group
|
||
resolved_count closed trades with known outcome
|
||
win_rate null if resolved_count < 5
|
||
|
||
"none" appears for Phase 6 trades where all feat_*_lo values are < 0.0001
|
||
(signals fired but below trigger threshold).
|
||
|
||
Only Phase 6 trades (feat_fg_lo IS NOT NULL) are included.
|
||
Pre-Phase-6 trades appear in /api/trades/legacy instead.
|
||
"""
|
||
attribution = await db.compute_attribution_from_db()
|
||
total = sum(v["trade_count"] for v in attribution.values())
|
||
return {"attribution": attribution, "total_attributed_trades": total}
|
||
|
||
|
||
@app.get("/api/metrics/manifold-matches")
|
||
async def get_manifold_matches():
|
||
"""Manifold match audit — version-split summary and recent match attempts.
|
||
|
||
summary.current_version — stats for the active matcher (MANIFOLD_MATCHER_VERSION):
|
||
version — the matcher version string
|
||
total_accepted — matches accepted (score >= 0.40, inversion unambiguous)
|
||
total_rejected — matches rejected (low score or ambiguous inversion)
|
||
total_no_results — no Manifold market found or API error
|
||
avg_match_score — mean Jaccard score for accepted matches
|
||
used_in_trade — accepted matches that were actually executed
|
||
summary.all_time — accepted/rejected/no_results across every matcher version.
|
||
summary.legacy.accepted_without_outcome_type — pre-outcome-guard accepted
|
||
records that the current matcher would reject (not counted in current_version).
|
||
summary.trades_dominated_by_mfld — non-excluded accepted-match trades where
|
||
feat_mfld_lo is the largest signal (consistent with attribution/features,
|
||
which also exclude excluded_from_metrics trades).
|
||
summary.unique_markets — distinct-market coverage (per-market, not per-attempt):
|
||
evaluated — DISTINCT poly_market_id in manifold_match_audit (all versions)
|
||
accepted — DISTINCT poly_market_id accepted by the current matcher
|
||
coverage_rate — accepted / evaluated (null when evaluated=0)
|
||
|
||
recent_matches: last 50 rows from manifold_match_audit, newest first, each
|
||
tagged with matcher_version.
|
||
used_in_trade=True only when status='accepted' AND a trade was actually executed.
|
||
"""
|
||
data = await db.get_manifold_matches(limit=50)
|
||
for match in data["recent_matches"]:
|
||
ts = match.get("timestamp")
|
||
if ts is not None and hasattr(ts, "isoformat"):
|
||
match["timestamp"] = ts.isoformat()
|
||
return data
|
||
|
||
|
||
@app.get("/api/metrics/manifold-coverage")
|
||
async def get_manifold_coverage():
|
||
"""Manifold coverage by semantic market category, counted by UNIQUE market.
|
||
|
||
Unlike the raw audit counters (which count per-attempt and are inflated by the
|
||
trading loop re-evaluating the same markets), this measures real coverage:
|
||
how many DISTINCT markets in each category the matcher found an accepted
|
||
Manifold counterpart for. Base table is manifold_match_audit filtered to the
|
||
current matcher (v3_outcome_guard); category is inferred from trade family_key
|
||
when available, otherwise from the question text.
|
||
|
||
coverage_by_category — one entry per category, ordered by unique_evaluated DESC:
|
||
category — gubernatorial | mayoral | senate | primary-republican |
|
||
primary-democrat | big-tech | geopolitics | other
|
||
unique_evaluated — distinct markets audited in this category
|
||
unique_accepted — distinct markets with at least one accepted match
|
||
unique_rejected — distinct markets with at least one rejected match
|
||
unique_no_results — distinct markets with at least one no_results outcome
|
||
coverage_rate — unique_accepted / unique_evaluated (null if evaluated=0)
|
||
summary — total_unique_evaluated, total_unique_accepted, overall_coverage_rate
|
||
(null if nothing evaluated), categories_with_coverage (categories with
|
||
unique_accepted > 0).
|
||
"""
|
||
return await db.get_manifold_coverage_by_category()
|
||
|
||
|
||
@app.get("/api/summary")
|
||
async def get_summary():
|
||
"""Dashboard summary card data.
|
||
|
||
All portfolio counts (total_trades, open_trades_count, total_deployed,
|
||
cash_available) are computed live from the DB on every request.
|
||
|
||
PnL and performance metrics come from the latest metrics_daily snapshot,
|
||
which is written by the bot every cycle via MetricsTracker.update_daily_summary().
|
||
After Fix 3, that snapshot is also DB-computed — not dependent on pod restarts.
|
||
sharpe_ratio is the exception: it is recomputed live here from the daily
|
||
PnL-close series (same sharpe_with_gate the tracker uses), so the
|
||
explanation fields (sharpe_status, days_observed) always match the value.
|
||
"""
|
||
latest_metrics, counts, position_data, inverted, legacy_count, daily_closes = (
|
||
await asyncio.gather(
|
||
db.get_metrics_history(days=1),
|
||
db.compute_metrics_from_db(),
|
||
db.get_open_position_data(),
|
||
db.get_recently_closed_inverted(hours=24),
|
||
db.get_legacy_incomplete_count(),
|
||
db.get_daily_pnl_closes(),
|
||
)
|
||
)
|
||
|
||
latest = latest_metrics[0] if latest_metrics else {}
|
||
paper_bankroll = float(os.getenv("PAPER_BANKROLL", "10000"))
|
||
total_trades = int(counts["total_trades"] or 0)
|
||
resolved_count = int(counts.get("resolved_count") or 0)
|
||
# Same source PaperExecutor.initialize() uses to reconstruct cash:
|
||
# total_net_cost_open = SUM(net_cost) over open trades, uncapped.
|
||
_, total_net_cost_open = position_data
|
||
total_deployed = total_net_cost_open
|
||
|
||
# Sharpe: computed live from the daily PnL curve (same function the
|
||
# tracker uses for the snapshot). None + status while the minimum-sample
|
||
# gate (>=30 days observed, >=10 resolved trades) is not met — a Sharpe
|
||
# over 1 resolved trade is statistically meaningless.
|
||
days_observed = len(daily_closes)
|
||
sharpe, sharpe_status = sharpe_with_gate(daily_closes, paper_bankroll, resolved_count)
|
||
|
||
win_rate = latest.get("win_rate")
|
||
calibration = latest.get("calibration_score")
|
||
|
||
return {
|
||
# ── Portfolio state (live from DB) ──────────────────────────────────
|
||
"paper_mode": os.getenv("PAPER_MODE", "true") == "true",
|
||
"paper_bankroll": paper_bankroll,
|
||
"total_trades": total_trades, # COUNT(*), uncapped
|
||
"open_trades_count": int(counts["open_count"] or 0), # COUNT(*), uncapped
|
||
"closed_trades_count": int(counts["closed_count"] or 0), # COUNT(*), uncapped
|
||
"total_deployed": total_deployed, # exact, from DB
|
||
"cash_available": cash_available(paper_bankroll, total_net_cost_open),
|
||
"legacy_incomplete_count": legacy_count, # exact, from DB
|
||
"reentry_guard_blocks_24h": len(inverted), # exact, from DB
|
||
|
||
# ── P&L (from latest metrics_daily snapshot) ────────────────────────
|
||
# unrealized_pnl_est: open positions, edge_net × net_cost − fee.
|
||
# Estimated — uses model signal, not live price. Source: open trades.
|
||
# realized_pnl: closed positions with known resolution.
|
||
# Exact — payout − net_cost per trade (net of fee), matches logs/Telegram.
|
||
# total_pnl: sum of both.
|
||
"unrealized_pnl_est": latest.get("unrealized_pnl_est") or 0,
|
||
"realized_pnl": latest.get("realized_pnl") or 0,
|
||
"total_pnl": latest.get("total_pnl") or 0,
|
||
|
||
# ── Performance metrics ──────────────────────────────────────────────
|
||
# win_rate: fraction of resolved closed trades where close_pnl > 0.
|
||
# null if fewer than 5 resolved trades. Source: closed+resolved trades.
|
||
# sharpe_ratio: annualized Sharpe of the daily total_pnl curve, computed
|
||
# live from metrics_daily. null while the minimum-sample gate fails
|
||
# (sharpe_status explains why). Source: bot/metrics/sharpe.py.
|
||
# calibration_score: 1 − Brier score on resolved trades (higher = better).
|
||
# null if fewer than 10 resolved trades. Source: closed+resolved trades.
|
||
"win_rate": win_rate, # null if < 5 resolved
|
||
"sharpe_ratio": sharpe, # null if gate fails
|
||
"sharpe_status": sharpe_status, # ok | insufficient_sample | zero_variance
|
||
"days_observed": days_observed,
|
||
"min_days_required": MIN_DAYS_OBSERVED,
|
||
"min_resolved_required": MIN_RESOLVED_TRADES,
|
||
"calibration_score": calibration, # null if < 10 resolved
|
||
|
||
# ── Counters (live from DB) ──────────────────────────────────────────
|
||
"resolved_count": resolved_count,
|
||
|
||
# ── Promotion gate ───────────────────────────────────────────────────
|
||
# Never promote on a tiny sample: requires the resolved/days minimums
|
||
# AND non-null metrics AND all thresholds. A single lucky resolved
|
||
# trade must not flip this to true.
|
||
"promotion_ready": (
|
||
resolved_count >= MIN_RESOLVED_TRADES
|
||
and days_observed >= MIN_DAYS_OBSERVED
|
||
and win_rate is not None and win_rate >= 0.52
|
||
and calibration is not None and calibration >= 0.7
|
||
and sharpe is not None and sharpe >= 0.5
|
||
and total_trades >= 50
|
||
),
|
||
}
|