-- Polymarket Bot Database Schema CREATE TABLE IF NOT EXISTS trades ( id TEXT PRIMARY KEY, market_id TEXT NOT NULL, question TEXT NOT NULL, direction TEXT NOT NULL, -- BUY_YES | BUY_NO size_usdc DOUBLE PRECISION, entry_price DOUBLE PRECISION, shares DOUBLE PRECISION, fee_usdc DOUBLE PRECISION, net_cost DOUBLE PRECISION, timestamp TIMESTAMPTZ NOT NULL, reasoning TEXT, paper BOOLEAN DEFAULT TRUE ); CREATE TABLE IF NOT EXISTS metrics_daily ( id SERIAL PRIMARY KEY, timestamp TIMESTAMPTZ NOT NULL, total_trades INTEGER, total_deployed DOUBLE PRECISION, total_fees DOUBLE PRECISION, total_pnl DOUBLE PRECISION, win_rate DOUBLE PRECISION, avg_edge DOUBLE PRECISION, sharpe_ratio DOUBLE PRECISION, calibration_score DOUBLE PRECISION, paper_mode BOOLEAN DEFAULT TRUE ); CREATE TABLE IF NOT EXISTS markets ( id TEXT PRIMARY KEY, condition_id TEXT, question TEXT NOT NULL, category TEXT, end_date TEXT, active BOOLEAN DEFAULT TRUE, last_seen TIMESTAMPTZ DEFAULT NOW() ); CREATE TABLE IF NOT EXISTS signals ( id SERIAL PRIMARY KEY, market_id TEXT NOT NULL, timestamp TIMESTAMPTZ NOT NULL, polymarket_price DOUBLE PRECISION, estimated_prob DOUBLE PRECISION, edge DOUBLE PRECISION, confidence DOUBLE PRECISION, direction TEXT, acted_on BOOLEAN DEFAULT FALSE ); CREATE INDEX IF NOT EXISTS idx_trades_timestamp ON trades(timestamp DESC); CREATE INDEX IF NOT EXISTS idx_trades_market ON trades(market_id); CREATE INDEX IF NOT EXISTS idx_metrics_timestamp ON metrics_daily(timestamp DESC); CREATE INDEX IF NOT EXISTS idx_signals_timestamp ON signals(timestamp DESC); -- ───────────────────────────────────────────────────────────────────────────── -- Phase 1 migrations: edge neto real -- -- spread_estimate and commission are HEURISTICS, not exact Polymarket exchange -- costs. spread_estimate ≈ estimated half-spread for medium-liquidity markets. -- commission = COMMISSION_RATE (0.02) * size_usdc — mirrors Polymarket taker fee. -- edge_net = edge_gross - spread_estimate - commission/size_usdc -- = edge_gross - 0.02 - 0.02 (always 0.04 deduction at current rates) -- -- These are stored per-trade so we can audit whether the model's cost assumptions -- were met in practice once markets resolve. -- ───────────────────────────────────────────────────────────────────────────── ALTER TABLE trades ADD COLUMN IF NOT EXISTS edge_gross DOUBLE PRECISION; ALTER TABLE trades ADD COLUMN IF NOT EXISTS edge_net DOUBLE PRECISION; ALTER TABLE trades ADD COLUMN IF NOT EXISTS prior_prob DOUBLE PRECISION; ALTER TABLE trades ADD COLUMN IF NOT EXISTS final_prob DOUBLE PRECISION; ALTER TABLE trades ADD COLUMN IF NOT EXISTS mid_price DOUBLE PRECISION; ALTER TABLE trades ADD COLUMN IF NOT EXISTS spread_estimate DOUBLE PRECISION; ALTER TABLE trades ADD COLUMN IF NOT EXISTS commission DOUBLE PRECISION; ALTER TABLE trades ADD COLUMN IF NOT EXISTS family_key TEXT; -- ───────────────────────────────────────────────────────────────────────────── -- Phase 2 / Phase 5 migrations: market families + observability -- -- Signals table extended so each evaluated market carries its audit trail: -- skip_reason — why the market was not traded ("edge_net", "family", -- "gnews_priority", "regime", "prior_extreme", etc.) -- passed_gross — True if edge_gross alone met regime_min_edge -- passed_net — True if edge_net met regime_min_edge (the actual gate) -- family_key — market family slug (e.g. "texas-republican-2026") -- regime_min_edge — threshold that applied to this market/category -- ───────────────────────────────────────────────────────────────────────────── ALTER TABLE signals ADD COLUMN IF NOT EXISTS edge_gross DOUBLE PRECISION; ALTER TABLE signals ADD COLUMN IF NOT EXISTS edge_net DOUBLE PRECISION; ALTER TABLE signals ADD COLUMN IF NOT EXISTS family_key TEXT; ALTER TABLE signals ADD COLUMN IF NOT EXISTS regime_min_edge DOUBLE PRECISION; ALTER TABLE signals ADD COLUMN IF NOT EXISTS skip_reason TEXT; ALTER TABLE signals ADD COLUMN IF NOT EXISTS passed_gross BOOLEAN; ALTER TABLE signals ADD COLUMN IF NOT EXISTS passed_net BOOLEAN; CREATE INDEX IF NOT EXISTS idx_signals_market ON signals(market_id); CREATE INDEX IF NOT EXISTS idx_trades_family ON trades(family_key); -- ───────────────────────────────────────────────────────────────────────────── -- Position lifecycle: legacy scan can close erroneous paper positions. -- closed_at IS NULL → position is open (all open-position queries filter this). -- closed_at NOT NULL → position closed; close_reason explains why. -- ───────────────────────────────────────────────────────────────────────────── ALTER TABLE trades ADD COLUMN IF NOT EXISTS closed_at TIMESTAMPTZ; ALTER TABLE trades ADD COLUMN IF NOT EXISTS close_reason TEXT; CREATE INDEX IF NOT EXISTS idx_trades_closed ON trades(closed_at) WHERE closed_at IS NOT NULL; -- ───────────────────────────────────────────────────────────────────────────── -- Fix 3: market resolution and realized P&L per trade -- -- resolution: 1.0 if YES resolved, 0.0 if NO resolved, NULL if not yet settled. -- close_pnl: realized P&L in USDC at close time — NET of fee (payout − net_cost), -- the same definition PaperExecutor.close_position() reports in logs/Telegram. -- BUY_YES: resolution * shares - net_cost -- BUY_NO: (1 - resolution) * shares - net_cost -- NULL if closed without a known resolution (legacy closes, inversion fixes). -- ───────────────────────────────────────────────────────────────────────────── ALTER TABLE trades ADD COLUMN IF NOT EXISTS close_pnl DOUBLE PRECISION; ALTER TABLE trades ADD COLUMN IF NOT EXISTS resolution DOUBLE PRECISION; -- ───────────────────────────────────────────────────────────────────────────── -- Phase 6: per-feature signal attribution — all values in log-odds space -- -- All four primary features share a common unit (log-odds contribution to -- the posterior estimate) so they can be compared directly: -- -- feat_fg_lo = _fg_contribution × 2 -- Fear & Greed direction-adjusted delta, ×2 to log-odds. -- Non-zero for every trade. Range ≈ ±0.12. -- Materiality threshold: |lo| ≥ 0.05. -- -- feat_mom_lo = _momentum_contribution × 2 -- Momentum delta (direction-adjusted), ×2 to log-odds. -- Zero when |btc_change_24h| ≤ 2 %. Range ≈ ±0.15. -- Materiality threshold: |lo| ≥ 0.05. -- -- feat_news_lo = news_log_adj (already in log-odds, no scaling) -- GNews sentiment × NEWS_LOGODDS_WEIGHT (1.5). -- Zero for non-politics or when GNews budget exhausted. -- Range ≈ ±1.5. Materiality threshold: |lo| ≥ 0.10. -- -- feat_mfld_lo = manifold_log_adj (already in log-odds, no scaling) -- Manifold divergence × MANIFOLD_LOGODDS_WEIGHT (0.6). -- Zero when Manifold returned no result. -- Range ≈ ±0.6. Materiality threshold: |lo| ≥ 0.10. -- -- feat_btc_dom_lo = _btc_dom_contribution × 2 -- BTC-dominance alt-pressure delta, ×2 to log-odds. -- Only fires for ETH / altcoin / general-crypto markets -- when btc_dominance > 55 % or < 45 %. -- Values: { −0.06, 0.0, +0.06 }. -- Materiality threshold: |lo| ≥ 0.05. -- -- NULL for pre-Phase-6 trades. Backfilled at startup via -- Database.backfill_feature_columns() using reasoning-string regex -- (fg_lo/mom_lo multiplied by 2 from raw; news_lo/mfld_lo taken directly; -- btc_dom_lo cannot be backfilled and remains NULL for legacy trades). -- ───────────────────────────────────────────────────────────────────────────── ALTER TABLE trades ADD COLUMN IF NOT EXISTS feat_fg_lo DOUBLE PRECISION; ALTER TABLE trades ADD COLUMN IF NOT EXISTS feat_mom_lo DOUBLE PRECISION; ALTER TABLE trades ADD COLUMN IF NOT EXISTS feat_news_lo DOUBLE PRECISION; ALTER TABLE trades ADD COLUMN IF NOT EXISTS feat_mfld_lo DOUBLE PRECISION; ALTER TABLE trades ADD COLUMN IF NOT EXISTS feat_btc_dom_lo DOUBLE PRECISION; CREATE INDEX IF NOT EXISTS idx_trades_feat_fg ON trades(feat_fg_lo) WHERE feat_fg_lo IS NOT NULL; CREATE INDEX IF NOT EXISTS idx_trades_feat_mfld ON trades(feat_mfld_lo) WHERE feat_mfld_lo IS NOT NULL; -- ───────────────────────────────────────────────────────────────────────────── -- Manifold match audit — per-trade columns in trades -- -- Persisted for every trade where Manifold was queried (status='accepted'). -- mfld_match_status: 'accepted' | 'rejected' | 'no_results' -- mfld_inverted: TRUE when prob_final = 1 - prob_raw (party complement match) -- ───────────────────────────────────────────────────────────────────────────── ALTER TABLE trades ADD COLUMN IF NOT EXISTS mfld_market_id TEXT; ALTER TABLE trades ADD COLUMN IF NOT EXISTS mfld_market_title TEXT; ALTER TABLE trades ADD COLUMN IF NOT EXISTS mfld_market_url TEXT; ALTER TABLE trades ADD COLUMN IF NOT EXISTS mfld_prob_raw DOUBLE PRECISION; ALTER TABLE trades ADD COLUMN IF NOT EXISTS mfld_prob_final DOUBLE PRECISION; ALTER TABLE trades ADD COLUMN IF NOT EXISTS mfld_inverted BOOLEAN; ALTER TABLE trades ADD COLUMN IF NOT EXISTS mfld_match_score DOUBLE PRECISION; ALTER TABLE trades ADD COLUMN IF NOT EXISTS mfld_match_reason TEXT; ALTER TABLE trades ADD COLUMN IF NOT EXISTS mfld_match_status TEXT; -- ───────────────────────────────────────────────────────────────────────────── -- Manifold match audit table — records every Manifold query attempt -- -- Populated for ALL queries: accepted, rejected, and no_results. -- used_in_trade=TRUE is set after executor confirms a trade was executed. -- poly_market_id: Market.id from the Polymarket Market dataclass (never NULL). -- ───────────────────────────────────────────────────────────────────────────── CREATE TABLE IF NOT EXISTS manifold_match_audit ( id TEXT PRIMARY KEY, timestamp TIMESTAMPTZ DEFAULT NOW(), poly_market_id TEXT NOT NULL, poly_question TEXT NOT NULL, search_query TEXT, mfld_market_id TEXT, mfld_market_title TEXT, mfld_market_url TEXT, prob_raw DOUBLE PRECISION, prob_final DOUBLE PRECISION, inverted BOOLEAN DEFAULT FALSE, match_score DOUBLE PRECISION, match_reason TEXT, match_status TEXT NOT NULL, used_in_trade BOOLEAN DEFAULT FALSE, poly_outcome_type TEXT, mfld_outcome_type TEXT ); CREATE INDEX IF NOT EXISTS idx_mfld_audit_timestamp ON manifold_match_audit(timestamp DESC); CREATE INDEX IF NOT EXISTS idx_mfld_audit_status ON manifold_match_audit(match_status); CREATE INDEX IF NOT EXISTS idx_mfld_audit_poly_mkt ON manifold_match_audit(poly_market_id); -- Backfill outcome-type columns on pre-existing tables (idempotent). ALTER TABLE manifold_match_audit ADD COLUMN IF NOT EXISTS poly_outcome_type TEXT; ALTER TABLE manifold_match_audit ADD COLUMN IF NOT EXISTS mfld_outcome_type TEXT; -- ───────────────────────────────────────────────────────────────────────────── -- Matcher versioning — separate current-matcher metrics from legacy records -- -- matcher_version tags each audit row with the matcher that produced it -- (MANIFOLD_MATCHER_VERSION in bot/data/manifold.py). This lets the metrics -- endpoint isolate current_version stats from pre-versioning records, whose -- accepted matches would now be rejected by the outcome-compatibility guard. -- -- Backfill is one-shot and idempotent (only touches NULL matcher_version rows): -- * rows with no outcome types → 'legacy_pre_outcome_guard' (pre outcome-guard; -- accepted without any outcome-type validation) -- * rows with an outcome type → 'v2_outcome_guard_no_version' (existed between -- the outcome-guard and this versioning; real version not persisted) -- We tag rather than infer the exact version that wasn't recorded. -- ───────────────────────────────────────────────────────────────────────────── ALTER TABLE manifold_match_audit ADD COLUMN IF NOT EXISTS matcher_version TEXT; UPDATE manifold_match_audit SET matcher_version = 'legacy_pre_outcome_guard' WHERE matcher_version IS NULL AND poly_outcome_type IS NULL AND mfld_outcome_type IS NULL; UPDATE manifold_match_audit SET matcher_version = 'v2_outcome_guard_no_version' WHERE matcher_version IS NULL AND (poly_outcome_type IS NOT NULL OR mfld_outcome_type IS NOT NULL); CREATE INDEX IF NOT EXISTS idx_mfld_audit_version ON manifold_match_audit(matcher_version); -- ───────────────────────────────────────────────────────────────────────────── -- Metric exclusion — administrative closure flag -- -- excluded_from_metrics: TRUE for trades closed for non-signal reasons -- (bad matcher, data error, admin close). These trades are excluded from -- win_rate, calibration_score, realized_pnl, and feature attribution. -- exclusion_reason: free-text label for the exclusion cause. -- e.g. 'invalid_manifold_match_legacy' -- ───────────────────────────────────────────────────────────────────────────── ALTER TABLE trades ADD COLUMN IF NOT EXISTS excluded_from_metrics BOOLEAN DEFAULT FALSE; ALTER TABLE trades ADD COLUMN IF NOT EXISTS exclusion_reason TEXT; CREATE INDEX IF NOT EXISTS idx_trades_excluded ON trades(excluded_from_metrics) WHERE excluded_from_metrics = TRUE; -- ───────────────────────────────────────────────────────────────────────────── -- Fix 3: extended metrics_daily columns for DB-computed metrics -- -- unrealized_pnl_est: SUM(edge_net * net_cost - fee) on open trades with edge_net. -- Estimated — uses model's own edge signal, not live market price. -- realized_pnl: SUM(close_pnl) on closed trades with a known resolution. -- Exact — derived from actual market outcome. -- open_count / closed_count / resolved_count: trade counts at snapshot time. -- ───────────────────────────────────────────────────────────────────────────── ALTER TABLE metrics_daily ADD COLUMN IF NOT EXISTS unrealized_pnl_est DOUBLE PRECISION; ALTER TABLE metrics_daily ADD COLUMN IF NOT EXISTS realized_pnl DOUBLE PRECISION; ALTER TABLE metrics_daily ADD COLUMN IF NOT EXISTS open_count INTEGER; ALTER TABLE metrics_daily ADD COLUMN IF NOT EXISTS closed_count INTEGER; ALTER TABLE metrics_daily ADD COLUMN IF NOT EXISTS resolved_count INTEGER; -- ───────────────────────────────────────────────────────────────────────────── -- Checkpoint alerts — one-shot and rate-limited Telegram observation alerts -- -- fired_at: timestamp of the first fire (immutable for one-shot checkpoints) -- last_fired_at: updated on every fire (used for rate-limiting repeatable alerts) -- ───────────────────────────────────────────────────────────────────────────── CREATE TABLE IF NOT EXISTS checkpoint_alerts ( checkpoint_name TEXT PRIMARY KEY, fired_at TIMESTAMPTZ NOT NULL, last_fired_at TIMESTAMPTZ ); -- ───────────────────────────────────────────────────────────────────────────── -- Manifold evaluation cooldown — per-market backoff for the Manifold matcher -- -- The trading loop re-evaluates the same ~stable set of politics/tech markets -- every cycle (~60s). Most resolve to a stable terminal verdict (no Manifold -- coverage, low-score, outcome mismatch, conditional market) that will not change -- on the next cycle. Re-querying them every minute floods manifold_match_audit -- with redundant rows and makes the metrics uninterpretable. -- -- This table records, per poly_market_id, when the market was last evaluated and -- the earliest time it should be evaluated again (retry_after). evaluate() in -- bot/strategy/bayesian.py consults it BEFORE calling the matcher and skips the -- call (and the audit write) entirely while now() < retry_after. -- -- last_status / cooldown_reason are stored for observability only. -- ───────────────────────────────────────────────────────────────────────────── CREATE TABLE IF NOT EXISTS manifold_eval_cooldown ( poly_market_id TEXT PRIMARY KEY, last_evaluated_at TIMESTAMPTZ NOT NULL, last_status TEXT NOT NULL, retry_after TIMESTAMPTZ NOT NULL, cooldown_reason TEXT ); CREATE INDEX IF NOT EXISTS idx_mfld_cooldown_retry ON manifold_eval_cooldown(retry_after); -- ───────────────────────────────────────────────────────────────────────────── -- Replay R0: snapshot recorder — the archive the replay engine reads from -- -- The signals table (Phase 2/5 schema) never had a writer; R0 makes it the -- per-(market, cycle) decision archive. One row per evaluated market per -- cycle, carrying both the INPUTS the strategy saw (external signals, news -- sentiment, per-feature log-odds) and the OUTPUTS it produced (probs, edges, -- gates, skip_reason). A replay run rebuilds Market/ExternalSignals from -- these rows plus ext_snapshots and re-executes evaluate() deterministically. -- -- cycle_ts groups all rows of one trading cycle and joins them to their -- ext_snapshots row (same timestamp; no FK to keep writes independent). -- days_to_resolution is persisted so replay does not depend on wall-clock. -- news_budget_skipped distinguishes "GNews had nothing" from "GNews was not -- asked this cycle" (5-query budget) — without it politics replay would treat -- budget starvation as absence of news. -- Retention: rows older than SIGNALS_RETENTION_DAYS (default 90) are pruned. -- ───────────────────────────────────────────────────────────────────────────── ALTER TABLE signals ADD COLUMN IF NOT EXISTS cycle_ts TIMESTAMPTZ; ALTER TABLE signals ADD COLUMN IF NOT EXISTS category TEXT; ALTER TABLE signals ADD COLUMN IF NOT EXISTS prior_prob DOUBLE PRECISION; ALTER TABLE signals ADD COLUMN IF NOT EXISTS raw_final_prob DOUBLE PRECISION; ALTER TABLE signals ADD COLUMN IF NOT EXISTS days_to_resolution INTEGER; ALTER TABLE signals ADD COLUMN IF NOT EXISTS volume_24h DOUBLE PRECISION; ALTER TABLE signals ADD COLUMN IF NOT EXISTS news_sentiment DOUBLE PRECISION; ALTER TABLE signals ADD COLUMN IF NOT EXISTS news_budget_skipped BOOLEAN; ALTER TABLE signals ADD COLUMN IF NOT EXISTS guardrail_applied BOOLEAN; ALTER TABLE signals ADD COLUMN IF NOT EXISTS guardrail_changed_decision BOOLEAN; ALTER TABLE signals ADD COLUMN IF NOT EXISTS feat_fg_lo DOUBLE PRECISION; ALTER TABLE signals ADD COLUMN IF NOT EXISTS feat_mom_lo DOUBLE PRECISION; ALTER TABLE signals ADD COLUMN IF NOT EXISTS feat_news_lo DOUBLE PRECISION; ALTER TABLE signals ADD COLUMN IF NOT EXISTS feat_mfld_lo DOUBLE PRECISION; ALTER TABLE signals ADD COLUMN IF NOT EXISTS feat_btc_dom_lo DOUBLE PRECISION; CREATE INDEX IF NOT EXISTS idx_signals_cycle ON signals(cycle_ts); -- One row per trading cycle: the ExternalSignals snapshot every market in -- that cycle was evaluated against. Written once per cycle before the -- evaluation loop; signals rows join on cycle_ts. CREATE TABLE IF NOT EXISTS ext_snapshots ( cycle_ts TIMESTAMPTZ PRIMARY KEY, btc_price DOUBLE PRECISION, btc_change_24h DOUBLE PRECISION, eth_price DOUBLE PRECISION, eth_change_24h DOUBLE PRECISION, btc_dominance DOUBLE PRECISION, fear_greed_index INTEGER, fear_greed_label TEXT, total_market_cap_change DOUBLE PRECISION, valid BOOLEAN ); -- ───────────────────────────────────────────────────────────────────────────── -- Replay R1: replay core — re-execute evaluate() over the R0 archive -- -- A replay run reads cycles from signals + ext_snapshots + markets, rebuilds -- the exact inputs (including archived news_sentiment — GNews is never called), -- re-runs BayesianStrategy.evaluate() with the archived cycle_ts as clock, and -- writes one replay_decisions row per (cycle, market). -- -- replay_runs tags every run with the code (git_sha) and strategy constants -- (config_hash) that produced it: two runs over the same window with different -- config_hash values are a counterfactual comparison; same config_hash against -- the recorded rows is a determinism check (mismatches should be 0, modulo -- day-boundary crossings between cycle_ts and the original wall-clock). -- -- matched: replayed decision equals the recorded one (skip_reason, probs, -- confidence, direction). NULL when not comparable — e.g. reentry_guard -- rows, recorded outside evaluate() with no decision fields to compare; -- the replay still re-evaluates them, which is extra calibration data. -- mismatch_field: first field that differed, for triage. -- ───────────────────────────────────────────────────────────────────────────── CREATE TABLE IF NOT EXISTS replay_runs ( run_id TEXT PRIMARY KEY, created_at TIMESTAMPTZ DEFAULT NOW(), git_sha TEXT, config_hash TEXT, config_json TEXT, from_ts TIMESTAMPTZ, to_ts TIMESTAMPTZ, cycles INTEGER, decisions INTEGER, matched INTEGER, mismatched INTEGER, note TEXT ); CREATE TABLE IF NOT EXISTS replay_decisions ( id SERIAL PRIMARY KEY, run_id TEXT NOT NULL, cycle_ts TIMESTAMPTZ NOT NULL, market_id TEXT NOT NULL, -- replayed outputs (same semantics as the signals columns) skip_reason TEXT, prior_prob DOUBLE PRECISION, estimated_prob DOUBLE PRECISION, raw_final_prob DOUBLE PRECISION, edge_gross DOUBLE PRECISION, edge_net DOUBLE PRECISION, regime_min_edge DOUBLE PRECISION, days_to_resolution INTEGER, confidence DOUBLE PRECISION, direction TEXT, would_trade BOOLEAN, -- fidelity vs the recorded signals row recorded_skip_reason TEXT, matched BOOLEAN, mismatch_field TEXT ); CREATE INDEX IF NOT EXISTS idx_replay_decisions_run ON replay_decisions(run_id); CREATE INDEX IF NOT EXISTS idx_replay_decisions_mkt ON replay_decisions(market_id); -- ───────────────────────────────────────────────────────────────────────────── -- Replay R2: outcomes + calibration metrics -- -- One row per resolved market, fetched from the Gamma API via -- get_market_resolution() (UMA-final only: a market closed but still in -- proposal/dispute is not stored). outcome is the final YES price: -- 1.0 = YES won, 0.0 = NO won. -- -- Joining signals (or replay_decisions) to market_outcomes scores every -- archived estimate against reality — Brier / log-loss of estimated_prob -- benchmarked against the market price (prior_prob) on the same rows, -- answering "does the model add value over the market?" across ALL -- evaluations, not just executed trades. -- ───────────────────────────────────────────────────────────────────────────── CREATE TABLE IF NOT EXISTS market_outcomes ( market_id TEXT PRIMARY KEY, outcome DOUBLE PRECISION NOT NULL, resolved_at TIMESTAMPTZ, fetched_at TIMESTAMPTZ DEFAULT NOW() );