feat(replay): R1 replay core — clock injection + replay of archived cycles
Re-executes BayesianStrategy.evaluate() over the R0 archive and stores results in replay_runs/replay_decisions, tagged with git sha + a hash of the strategy constants (same hash vs archive = determinism check, different hash = counterfactual run). - bayesian.py: optional as_of param on evaluate()/_days_to_resolution() (clock injection; default None = wall clock, prod behavior unchanged — the only touch to frozen code, purely additive) - bot/replay.py: replay engine + CLI (python -m bot.replay --from --to); ReplayNews feeds archived sentiment back (GNews never called, per-cycle budget bypassed — archived sentiment already encodes it); manifold/db not wired (observational-only in prod); recorded-vs-replayed compare at 1e-9 tolerance - schema.sql: replay_runs + replay_decisions (+ indexes), idempotent - db.py: 6 replay accessors/writers - tests: 19 new round-trip fidelity tests (104 total green) Validated against a real prod cycle (2026-07-02T14:03:15Z, 46 markets, 4 skip paths incl. the Georgia confidence record): 46/46 matched, max float delta 0.0. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
eb4f67414a
commit
0ac48ba7f8
@@ -747,6 +747,85 @@ class Database:
|
||||
except (ValueError, IndexError):
|
||||
return 0
|
||||
|
||||
# ── Replay R1: replay core ───────────────────────────────────────────────
|
||||
|
||||
async def get_replay_cycles(self, from_ts, to_ts) -> list:
|
||||
"""Return the cycle_ts values with archived decisions in [from_ts, to_ts)."""
|
||||
async with self._pool.acquire() as conn:
|
||||
rows = await conn.fetch("""
|
||||
SELECT DISTINCT cycle_ts FROM signals
|
||||
WHERE cycle_ts >= $1 AND cycle_ts < $2
|
||||
ORDER BY cycle_ts
|
||||
""", from_ts, to_ts)
|
||||
return [r["cycle_ts"] for r in rows]
|
||||
|
||||
async def get_ext_snapshot(self, cycle_ts) -> Optional[dict]:
|
||||
"""Return one cycle's ExternalSignals snapshot, or None if missing."""
|
||||
async with self._pool.acquire() as conn:
|
||||
row = await conn.fetchrow(
|
||||
"SELECT * FROM ext_snapshots WHERE cycle_ts = $1", cycle_ts
|
||||
)
|
||||
return dict(row) if row else None
|
||||
|
||||
async def get_cycle_signal_rows(self, cycle_ts) -> list[dict]:
|
||||
"""Return one cycle's archived decision rows in original evaluation
|
||||
order (id = insertion order = the order main.py evaluated them)."""
|
||||
async with self._pool.acquire() as conn:
|
||||
rows = await conn.fetch(
|
||||
"SELECT * FROM signals WHERE cycle_ts = $1 ORDER BY id", cycle_ts
|
||||
)
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
async def get_markets_by_ids(self, market_ids: list[str]) -> dict[str, dict]:
|
||||
"""Return market metadata rows keyed by id (for Market reconstruction)."""
|
||||
if not market_ids:
|
||||
return {}
|
||||
async with self._pool.acquire() as conn:
|
||||
rows = await conn.fetch(
|
||||
"SELECT * FROM markets WHERE id = ANY($1::text[])", market_ids
|
||||
)
|
||||
return {r["id"]: dict(r) for r in rows}
|
||||
|
||||
async def save_replay_run(self, run: dict) -> None:
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.execute("""
|
||||
INSERT INTO replay_runs (
|
||||
run_id, git_sha, config_hash, config_json,
|
||||
from_ts, to_ts, cycles, decisions, matched, mismatched, note
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)
|
||||
""",
|
||||
run["run_id"], run["git_sha"], run["config_hash"],
|
||||
run["config_json"], run["from_ts"], run["to_ts"],
|
||||
run["cycles"], run["decisions"], run["matched"],
|
||||
run["mismatched"], run["note"],
|
||||
)
|
||||
|
||||
async def save_replay_decisions(self, run_id: str, decisions: list[dict]) -> None:
|
||||
if not decisions:
|
||||
return
|
||||
rows = [
|
||||
(
|
||||
run_id, d["cycle_ts"], d["market_id"],
|
||||
d["skip_reason"], d["prior_prob"], d["estimated_prob"],
|
||||
d["raw_final_prob"], d["edge_gross"], d["edge_net"],
|
||||
d["regime_min_edge"], d["days_to_resolution"],
|
||||
d["confidence"], d["direction"], d["would_trade"],
|
||||
d["recorded_skip_reason"], d["matched"], d["mismatch_field"],
|
||||
)
|
||||
for d in decisions
|
||||
]
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.executemany("""
|
||||
INSERT INTO replay_decisions (
|
||||
run_id, cycle_ts, market_id,
|
||||
skip_reason, prior_prob, estimated_prob,
|
||||
raw_final_prob, edge_gross, edge_net,
|
||||
regime_min_edge, days_to_resolution,
|
||||
confidence, direction, would_trade,
|
||||
recorded_skip_reason, matched, mismatch_field
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17)
|
||||
""", rows)
|
||||
|
||||
async def mark_manifold_audit_used(self, audit_id: str) -> None:
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.execute(
|
||||
|
||||
@@ -370,3 +370,64 @@ CREATE TABLE IF NOT EXISTS ext_snapshots (
|
||||
total_market_cap_change DOUBLE PRECISION,
|
||||
valid BOOLEAN
|
||||
);
|
||||
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- Replay R1: replay core — re-execute evaluate() over the R0 archive
|
||||
--
|
||||
-- A replay run reads cycles from signals + ext_snapshots + markets, rebuilds
|
||||
-- the exact inputs (including archived news_sentiment — GNews is never called),
|
||||
-- re-runs BayesianStrategy.evaluate() with the archived cycle_ts as clock, and
|
||||
-- writes one replay_decisions row per (cycle, market).
|
||||
--
|
||||
-- replay_runs tags every run with the code (git_sha) and strategy constants
|
||||
-- (config_hash) that produced it: two runs over the same window with different
|
||||
-- config_hash values are a counterfactual comparison; same config_hash against
|
||||
-- the recorded rows is a determinism check (mismatches should be 0, modulo
|
||||
-- day-boundary crossings between cycle_ts and the original wall-clock).
|
||||
--
|
||||
-- matched: replayed decision equals the recorded one (skip_reason, probs,
|
||||
-- confidence, direction). NULL when not comparable — e.g. reentry_guard
|
||||
-- rows, recorded outside evaluate() with no decision fields to compare;
|
||||
-- the replay still re-evaluates them, which is extra calibration data.
|
||||
-- mismatch_field: first field that differed, for triage.
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
CREATE TABLE IF NOT EXISTS replay_runs (
|
||||
run_id TEXT PRIMARY KEY,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
git_sha TEXT,
|
||||
config_hash TEXT,
|
||||
config_json TEXT,
|
||||
from_ts TIMESTAMPTZ,
|
||||
to_ts TIMESTAMPTZ,
|
||||
cycles INTEGER,
|
||||
decisions INTEGER,
|
||||
matched INTEGER,
|
||||
mismatched INTEGER,
|
||||
note TEXT
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS replay_decisions (
|
||||
id SERIAL PRIMARY KEY,
|
||||
run_id TEXT NOT NULL,
|
||||
cycle_ts TIMESTAMPTZ NOT NULL,
|
||||
market_id TEXT NOT NULL,
|
||||
-- replayed outputs (same semantics as the signals columns)
|
||||
skip_reason TEXT,
|
||||
prior_prob DOUBLE PRECISION,
|
||||
estimated_prob DOUBLE PRECISION,
|
||||
raw_final_prob DOUBLE PRECISION,
|
||||
edge_gross DOUBLE PRECISION,
|
||||
edge_net DOUBLE PRECISION,
|
||||
regime_min_edge DOUBLE PRECISION,
|
||||
days_to_resolution INTEGER,
|
||||
confidence DOUBLE PRECISION,
|
||||
direction TEXT,
|
||||
would_trade BOOLEAN,
|
||||
-- fidelity vs the recorded signals row
|
||||
recorded_skip_reason TEXT,
|
||||
matched BOOLEAN,
|
||||
mismatch_field TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_replay_decisions_run ON replay_decisions(run_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_replay_decisions_mkt ON replay_decisions(market_id);
|
||||
|
||||
Reference in New Issue
Block a user