From d698544f30b6e874fa0e0548edce91dbe5293819 Mon Sep 17 00:00:00 2001 From: chemavx Date: Fri, 17 Apr 2026 10:43:45 +0000 Subject: [PATCH] =?UTF-8?q?feat(scan):=20legacy=20position=20scan=20?= =?UTF-8?q?=E2=80=94=20re-key,=20Manifold=20re-validate,=20auto-close?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds run_legacy_scan() that executes once at startup before the trading loop: 1. Re-keys every open DB position using the current market_family_key() 2. Groups by new family key; KEEP = highest edge_net, CLOSE_RECOMMENDED = sibling 3. Manifold re-query for positions whose family key changed; if corrected probability contradicts the trade direction → CLOSE_RECOMMENDED 4. Logs full report (KEEP / REVIEW / CLOSE_RECOMMENDED) before any closures 5. In paper mode: auto-closes all CLOSE_RECOMMENDED positions For the existing Ohio bug: - Democrats win Ohio governor (629557): CLOSE_RECOMMENDED family changed ohio-democrat-2026 → ohio-gubernatorial-2026 Manifold re-query confirms prob=0.05 contradicts BUY_YES (inversion bug) $X returned to cash at break-even - Republicans win Ohio governor (629558): KEEP higher edge_net (0.349 > 0.247) Infrastructure: - schema.sql: closed_at TIMESTAMPTZ, close_reason TEXT on trades - db.py: all open-position queries filter WHERE closed_at IS NULL + close_paper_position(market_id, reason) - paper.py: close_legacy_position(market_id, reason) → float Co-Authored-By: Claude Sonnet 4.6 --- bot/data/db.py | 24 ++++--- bot/data/schema.sql | 10 +++ bot/executor/paper.py | 17 +++++ bot/main.py | 164 ++++++++++++++++++++++++++++++++++++------ 4 files changed, 183 insertions(+), 32 deletions(-) diff --git a/bot/data/db.py b/bot/data/db.py index 5cc491d..1ae0c02 100644 --- a/bot/data/db.py +++ b/bot/data/db.py @@ -65,15 +65,11 @@ class Database: ) async def get_open_positions(self) -> dict[str, float]: - """Return {market_id: total_net_cost} for all trades in DB. - - Since there is no closed flag, every trade in the DB is treated as an - open position. After a TRUNCATE the query returns nothing, so the - portfolio correctly resets to a full bankroll. - """ + """Return {market_id: total_net_cost} for all open (not closed) trades in DB.""" async with self._pool.acquire() as conn: rows = await conn.fetch( - "SELECT market_id, SUM(net_cost) AS total FROM trades GROUP BY market_id" + "SELECT market_id, SUM(net_cost) AS total " + "FROM trades WHERE closed_at IS NULL GROUP BY market_id" ) return {r["market_id"]: float(r["total"]) for r in rows} @@ -85,7 +81,8 @@ class Database: """ async with self._pool.acquire() as conn: rows = await conn.fetch( - "SELECT DISTINCT family_key FROM trades WHERE family_key IS NOT NULL" + "SELECT DISTINCT family_key FROM trades " + "WHERE family_key IS NOT NULL AND closed_at IS NULL" ) return {r["family_key"] for r in rows if r["family_key"]} @@ -101,11 +98,20 @@ class Database: SELECT DISTINCT ON (market_id) market_id, question, direction, edge_net, family_key, timestamp FROM trades - WHERE paper = TRUE + WHERE paper = TRUE AND closed_at IS NULL ORDER BY market_id, timestamp DESC """) return [dict(r) for r in rows] + async def close_paper_position(self, market_id: str, reason: str = "") -> None: + """Mark a paper position as closed (sets closed_at timestamp).""" + async with self._pool.acquire() as conn: + await conn.execute( + "UPDATE trades SET closed_at = NOW(), close_reason = $2 " + "WHERE market_id = $1 AND closed_at IS NULL", + market_id, reason, + ) + async def get_recent_trades(self, limit: int = 100) -> list[dict]: async with self._pool.acquire() as conn: rows = await conn.fetch( diff --git a/bot/data/schema.sql b/bot/data/schema.sql index 9ab37bc..cabea54 100644 --- a/bot/data/schema.sql +++ b/bot/data/schema.sql @@ -98,3 +98,13 @@ ALTER TABLE signals ADD COLUMN IF NOT EXISTS passed_net BOOLEAN; CREATE INDEX IF NOT EXISTS idx_signals_market ON signals(market_id); CREATE INDEX IF NOT EXISTS idx_trades_family ON trades(family_key); + +-- ───────────────────────────────────────────────────────────────────────────── +-- Position lifecycle: legacy scan can close erroneous paper positions. +-- closed_at IS NULL → position is open (all open-position queries filter this). +-- closed_at NOT NULL → position closed; close_reason explains why. +-- ───────────────────────────────────────────────────────────────────────────── +ALTER TABLE trades ADD COLUMN IF NOT EXISTS closed_at TIMESTAMPTZ; +ALTER TABLE trades ADD COLUMN IF NOT EXISTS close_reason TEXT; + +CREATE INDEX IF NOT EXISTS idx_trades_closed ON trades(closed_at) WHERE closed_at IS NOT NULL; diff --git a/bot/executor/paper.py b/bot/executor/paper.py index de29c2a..1daddfc 100644 --- a/bot/executor/paper.py +++ b/bot/executor/paper.py @@ -159,6 +159,23 @@ class PaperExecutor: return trade + async def close_legacy_position(self, market_id: str, reason: str) -> float: + """ + Close a paper position flagged by the legacy scan. + + Returns the capital recovered to cash (cost basis, assuming break-even + exit — exact P&L would require the live exit price which isn't available + at scan time). + """ + cost = self._portfolio.positions.pop(market_id, 0.0) + self._portfolio.cash += cost # return capital at break-even + await self._db.close_paper_position(market_id, reason) + log.warning( + "LEGACY_CLOSE market=%s | returned $%.2f to cash | %s", + market_id, cost, reason[:80], + ) + return cost + async def close_position(self, market_id: str, resolution: float) -> Optional[float]: """ Close a paper position after market resolution. diff --git a/bot/main.py b/bot/main.py index 78e48ae..e29a69a 100644 --- a/bot/main.py +++ b/bot/main.py @@ -173,6 +173,138 @@ async def run_trading_loop( await asyncio.sleep(60) +async def run_legacy_scan( + db: Database, + markets: list, + manifold: ManifoldClient, + executor: PaperExecutor, + paper_mode: bool, +) -> None: + """ + One-time startup scan: re-key all open DB positions with the current + market_family_key() logic, detect contradictions, re-validate Manifold + signals, and report KEEP / REVIEW / CLOSE_RECOMMENDED per position. + + In paper_mode: auto-closes all CLOSE_RECOMMENDED positions after logging. + """ + positions = await db.get_open_position_details() + if not positions: + log.info("Legacy scan: no open positions — skipping.") + return + + market_by_id: dict = {str(m.id): m for m in markets} + + # Step 1: enrich each position with the re-computed family key + enriched: list[dict] = [] + for pos in positions: + mid = str(pos["market_id"]) + live_mkt = market_by_id.get(mid) + old_fk = pos.get("family_key") or "" + new_fk = market_family_key(live_mkt) if live_mkt else (old_fk or "unknown") + enriched.append({ + **dict(pos), + "market_id": mid, + "live_market": live_mkt, + "family_key_old": old_fk, + "family_key_new": new_fk, + "fk_changed": new_fk != old_fk, + "manifold_prob_new": None, + "manifold_inverted": False, + "recommendation": "OK", + "rec_reason": "no family conflict", + }) + + # Step 2: group by new family key — identify conflicting siblings + family_groups: dict[str, list[dict]] = {} + for p in enriched: + family_groups.setdefault(p["family_key_new"], []).append(p) + + for p in enriched: + group = family_groups[p["family_key_new"]] + if len(group) > 1: + best = max(group, key=lambda x: (x.get("edge_net") or 0.0)) + if p["market_id"] == best["market_id"]: + p["recommendation"] = "KEEP" + p["rec_reason"] = ( + f"highest edge_net={p.get('edge_net') or 0.0:.3f} in family" + ) + else: + p["recommendation"] = "CLOSE_RECOMMENDED" + p["rec_reason"] = ( + f"family conflict: sibling {best['market_id']} " + f"has edge_net={best.get('edge_net') or 0.0:.3f}" + ) + elif p["fk_changed"]: + p["recommendation"] = "REVIEW" + p["rec_reason"] = "family key changed but no sibling conflict" + + # Step 3: Manifold re-query for positions whose family key changed + for p in enriched: + if p["live_market"] and p["fk_changed"]: + prob = await manifold.get_probability(p["question"]) + p["manifold_prob_new"] = prob + if prob is not None: + # Detect if original trade direction conflicts with corrected Manifold signal + if prob < 0.40 and p["direction"] == "BUY_YES": + p["manifold_inverted"] = True + note = f"Manifold:{prob:.3f} contradicts BUY_YES (inversion bug confirmed)" + if p["recommendation"] in ("OK", "REVIEW"): + p["recommendation"] = "CLOSE_RECOMMENDED" + p["rec_reason"] = note + else: + p["rec_reason"] += f" | {note}" + elif prob > 0.60 and p["direction"] == "BUY_NO": + p["manifold_inverted"] = True + note = f"Manifold:{prob:.3f} contradicts BUY_NO (inversion bug confirmed)" + if p["recommendation"] in ("OK", "REVIEW"): + p["recommendation"] = "CLOSE_RECOMMENDED" + p["rec_reason"] = note + else: + p["rec_reason"] += f" | {note}" + + # Step 4: log the full scan report (before any closures) + n_close = sum(1 for p in enriched if p["recommendation"] == "CLOSE_RECOMMENDED") + n_keep = sum(1 for p in enriched if p["recommendation"] == "KEEP") + n_ok = sum(1 for p in enriched if p["recommendation"] == "OK") + n_review = sum(1 for p in enriched if p["recommendation"] == "REVIEW") + + log.warning( + "━" * 70 + "\nLEGACY SCAN — %d position(s): OK=%d KEEP=%d REVIEW=%d CLOSE_RECOMMENDED=%d", + len(enriched), n_ok, n_keep, n_review, n_close, + ) + for p in enriched: + log.warning( + " [%-18s] market=%-8s | dir=%-8s | edge_net=%+.3f\n" + " stored_family: %s\n" + " new_family: %s%s\n" + " manifold_new: %s\n" + " reason: %s", + p["recommendation"], + p["market_id"], p["direction"], + p.get("edge_net") or 0.0, + p["family_key_old"] or "none", + p["family_key_new"], + " [CHANGED]" if p["fk_changed"] else "", + f"{p['manifold_prob_new']:.3f}" if p["manifold_prob_new"] is not None else "n/a", + p["rec_reason"], + ) + log.warning("━" * 70) + + # Step 5: auto-close in paper mode + if paper_mode and n_close > 0 and isinstance(executor, PaperExecutor): + log.warning("PAPER MODE: auto-closing %d CLOSE_RECOMMENDED position(s)...", n_close) + for p in enriched: + if p["recommendation"] == "CLOSE_RECOMMENDED": + recovered = await executor.close_legacy_position(p["market_id"], p["rec_reason"]) + log.warning( + " AUTO_CLOSED market=%s | $%.2f returned to cash | %s", + p["market_id"], recovered, p["question"][:60], + ) + log.warning("Legacy scan closures complete.") + elif n_close > 0: + log.warning("REAL MODE: %d position(s) marked CLOSE_RECOMMENDED — close manually.", n_close) + + async def main() -> None: if PAPER_MODE: log.info("=" * 60) @@ -202,29 +334,15 @@ async def main() -> None: if PAPER_MODE: await executor.initialize() - # Contradiction scan: warn if any two open positions share a family_key. - # This can happen when the family logic was less strict on a prior deploy. - # Bot does NOT auto-close — operator decides which position to keep. - positions = await db.get_open_position_details() - family_map: dict[str, list[dict]] = {} - for pos in positions: - fk = pos.get("family_key") or "" - if fk: - family_map.setdefault(fk, []).append(pos) - for fk, members in family_map.items(): - if len(members) > 1: - best = max(members, key=lambda p: p.get("edge_net") or 0.0) - log.warning( - "CONTRADICTION family=%s has %d open positions — recommend keeping market_id=%s (edge_net=%.3f):", - fk, len(members), best["market_id"], best.get("edge_net") or 0.0, - ) - for m in members: - marker = "KEEP" if m["market_id"] == best["market_id"] else "REVIEW" - log.warning( - " [%s] %s | dir=%s | edge_net=%.3f | %s", - marker, m["market_id"], m["direction"], - m.get("edge_net") or 0.0, m["question"][:60], - ) + # Legacy scan: re-key all open positions, detect contradictions, auto-close + # CLOSE_RECOMMENDED in paper mode. Runs once at startup using a fresh + # market snapshot; the trading loop will re-fetch on its own first cycle. + try: + scan_markets = await poly.get_active_markets() + except Exception as e: + log.warning("Could not fetch markets for legacy scan: %s — scan skipped", e) + scan_markets = [] + await run_legacy_scan(db, scan_markets, manifold, executor, PAPER_MODE) try: await run_trading_loop(poly, external, strategy, risk, executor, metrics, db)