feat(scan): legacy position scan — re-key, Manifold re-validate, auto-close

Adds run_legacy_scan() that executes once at startup before the trading loop:

  1. Re-keys every open DB position using the current market_family_key()
  2. Groups by new family key; KEEP = highest edge_net, CLOSE_RECOMMENDED = sibling
  3. Manifold re-query for positions whose family key changed; if corrected
     probability contradicts the trade direction → CLOSE_RECOMMENDED
  4. Logs full report (KEEP / REVIEW / CLOSE_RECOMMENDED) before any closures
  5. In paper mode: auto-closes all CLOSE_RECOMMENDED positions

For the existing Ohio bug:
  - Democrats win Ohio governor (629557): CLOSE_RECOMMENDED
    family changed ohio-democrat-2026 → ohio-gubernatorial-2026
    Manifold re-query confirms prob=0.05 contradicts BUY_YES (inversion bug)
    $X returned to cash at break-even
  - Republicans win Ohio governor (629558): KEEP
    higher edge_net (0.349 > 0.247)

Infrastructure:
  - schema.sql: closed_at TIMESTAMPTZ, close_reason TEXT on trades
  - db.py: all open-position queries filter WHERE closed_at IS NULL
           + close_paper_position(market_id, reason)
  - paper.py: close_legacy_position(market_id, reason) → float

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

bot/data/db.py

@@ -65,15 +65,11 @@ class Database:
             )
 
     async def get_open_positions(self) -> dict[str, float]:
-        """Return {market_id: total_net_cost} for all trades in DB.
-
-        Since there is no closed flag, every trade in the DB is treated as an
-        open position.  After a TRUNCATE the query returns nothing, so the
-        portfolio correctly resets to a full bankroll.
-        """
+        """Return {market_id: total_net_cost} for all open (not closed) trades in DB."""
         async with self._pool.acquire() as conn:
             rows = await conn.fetch(
-                "SELECT market_id, SUM(net_cost) AS total FROM trades GROUP BY market_id"
+                "SELECT market_id, SUM(net_cost) AS total "
+                "FROM trades WHERE closed_at IS NULL GROUP BY market_id"
             )
             return {r["market_id"]: float(r["total"]) for r in rows}
 
@@ -85,7 +81,8 @@ class Database:
         """
         async with self._pool.acquire() as conn:
             rows = await conn.fetch(
-                "SELECT DISTINCT family_key FROM trades WHERE family_key IS NOT NULL"
+                "SELECT DISTINCT family_key FROM trades "
+                "WHERE family_key IS NOT NULL AND closed_at IS NULL"
             )
             return {r["family_key"] for r in rows if r["family_key"]}
 
@@ -101,11 +98,20 @@ class Database:
                 SELECT DISTINCT ON (market_id)
                     market_id, question, direction, edge_net, family_key, timestamp
                 FROM trades
-                WHERE paper = TRUE
+                WHERE paper = TRUE AND closed_at IS NULL
                 ORDER BY market_id, timestamp DESC
             """)
             return [dict(r) for r in rows]
 
+    async def close_paper_position(self, market_id: str, reason: str = "") -> None:
+        """Mark a paper position as closed (sets closed_at timestamp)."""
+        async with self._pool.acquire() as conn:
+            await conn.execute(
+                "UPDATE trades SET closed_at = NOW(), close_reason = $2 "
+                "WHERE market_id = $1 AND closed_at IS NULL",
+                market_id, reason,
+            )
+
     async def get_recent_trades(self, limit: int = 100) -> list[dict]:
         async with self._pool.acquire() as conn:
             rows = await conn.fetch(

bot/data/schema.sql

@@ -98,3 +98,13 @@ ALTER TABLE signals ADD COLUMN IF NOT EXISTS passed_net      BOOLEAN;
 
 CREATE INDEX IF NOT EXISTS idx_signals_market ON signals(market_id);
 CREATE INDEX IF NOT EXISTS idx_trades_family  ON trades(family_key);
+
+-- ─────────────────────────────────────────────────────────────────────────────
+-- Position lifecycle: legacy scan can close erroneous paper positions.
+-- closed_at IS NULL  → position is open (all open-position queries filter this).
+-- closed_at NOT NULL → position closed; close_reason explains why.
+-- ─────────────────────────────────────────────────────────────────────────────
+ALTER TABLE trades ADD COLUMN IF NOT EXISTS closed_at    TIMESTAMPTZ;
+ALTER TABLE trades ADD COLUMN IF NOT EXISTS close_reason TEXT;
+
+CREATE INDEX IF NOT EXISTS idx_trades_closed ON trades(closed_at) WHERE closed_at IS NOT NULL;

bot/executor/paper.py

@@ -159,6 +159,23 @@ class PaperExecutor:
 
         return trade
 
+    async def close_legacy_position(self, market_id: str, reason: str) -> float:
+        """
+        Close a paper position flagged by the legacy scan.
+
+        Returns the capital recovered to cash (cost basis, assuming break-even
+        exit — exact P&L would require the live exit price which isn't available
+        at scan time).
+        """
+        cost = self._portfolio.positions.pop(market_id, 0.0)
+        self._portfolio.cash += cost  # return capital at break-even
+        await self._db.close_paper_position(market_id, reason)
+        log.warning(
+            "LEGACY_CLOSE market=%s | returned $%.2f to cash | %s",
+            market_id, cost, reason[:80],
+        )
+        return cost
+
     async def close_position(self, market_id: str, resolution: float) -> Optional[float]:
         """
         Close a paper position after market resolution.

bot/main.py

@@ -173,6 +173,138 @@ async def run_trading_loop(
         await asyncio.sleep(60)
 
 
+async def run_legacy_scan(
+    db: Database,
+    markets: list,
+    manifold: ManifoldClient,
+    executor: PaperExecutor,
+    paper_mode: bool,
+) -> None:
+    """
+    One-time startup scan: re-key all open DB positions with the current
+    market_family_key() logic, detect contradictions, re-validate Manifold
+    signals, and report KEEP / REVIEW / CLOSE_RECOMMENDED per position.
+
+    In paper_mode: auto-closes all CLOSE_RECOMMENDED positions after logging.
+    """
+    positions = await db.get_open_position_details()
+    if not positions:
+        log.info("Legacy scan: no open positions — skipping.")
+        return
+
+    market_by_id: dict = {str(m.id): m for m in markets}
+
+    # Step 1: enrich each position with the re-computed family key
+    enriched: list[dict] = []
+    for pos in positions:
+        mid = str(pos["market_id"])
+        live_mkt = market_by_id.get(mid)
+        old_fk = pos.get("family_key") or ""
+        new_fk = market_family_key(live_mkt) if live_mkt else (old_fk or "unknown")
+        enriched.append({
+            **dict(pos),
+            "market_id": mid,
+            "live_market": live_mkt,
+            "family_key_old": old_fk,
+            "family_key_new": new_fk,
+            "fk_changed": new_fk != old_fk,
+            "manifold_prob_new": None,
+            "manifold_inverted": False,
+            "recommendation": "OK",
+            "rec_reason": "no family conflict",
+        })
+
+    # Step 2: group by new family key — identify conflicting siblings
+    family_groups: dict[str, list[dict]] = {}
+    for p in enriched:
+        family_groups.setdefault(p["family_key_new"], []).append(p)
+
+    for p in enriched:
+        group = family_groups[p["family_key_new"]]
+        if len(group) > 1:
+            best = max(group, key=lambda x: (x.get("edge_net") or 0.0))
+            if p["market_id"] == best["market_id"]:
+                p["recommendation"] = "KEEP"
+                p["rec_reason"] = (
+                    f"highest edge_net={p.get('edge_net') or 0.0:.3f} in family"
+                )
+            else:
+                p["recommendation"] = "CLOSE_RECOMMENDED"
+                p["rec_reason"] = (
+                    f"family conflict: sibling {best['market_id']} "
+                    f"has edge_net={best.get('edge_net') or 0.0:.3f}"
+                )
+        elif p["fk_changed"]:
+            p["recommendation"] = "REVIEW"
+            p["rec_reason"] = "family key changed but no sibling conflict"
+
+    # Step 3: Manifold re-query for positions whose family key changed
+    for p in enriched:
+        if p["live_market"] and p["fk_changed"]:
+            prob = await manifold.get_probability(p["question"])
+            p["manifold_prob_new"] = prob
+            if prob is not None:
+                # Detect if original trade direction conflicts with corrected Manifold signal
+                if prob < 0.40 and p["direction"] == "BUY_YES":
+                    p["manifold_inverted"] = True
+                    note = f"Manifold:{prob:.3f} contradicts BUY_YES (inversion bug confirmed)"
+                    if p["recommendation"] in ("OK", "REVIEW"):
+                        p["recommendation"] = "CLOSE_RECOMMENDED"
+                        p["rec_reason"] = note
+                    else:
+                        p["rec_reason"] += f" | {note}"
+                elif prob > 0.60 and p["direction"] == "BUY_NO":
+                    p["manifold_inverted"] = True
+                    note = f"Manifold:{prob:.3f} contradicts BUY_NO (inversion bug confirmed)"
+                    if p["recommendation"] in ("OK", "REVIEW"):
+                        p["recommendation"] = "CLOSE_RECOMMENDED"
+                        p["rec_reason"] = note
+                    else:
+                        p["rec_reason"] += f" | {note}"
+
+    # Step 4: log the full scan report (before any closures)
+    n_close = sum(1 for p in enriched if p["recommendation"] == "CLOSE_RECOMMENDED")
+    n_keep  = sum(1 for p in enriched if p["recommendation"] == "KEEP")
+    n_ok    = sum(1 for p in enriched if p["recommendation"] == "OK")
+    n_review = sum(1 for p in enriched if p["recommendation"] == "REVIEW")
+
+    log.warning(
+        "━" * 70 + "\nLEGACY SCAN — %d position(s): OK=%d  KEEP=%d  REVIEW=%d  CLOSE_RECOMMENDED=%d",
+        len(enriched), n_ok, n_keep, n_review, n_close,
+    )
+    for p in enriched:
+        log.warning(
+            "  [%-18s] market=%-8s | dir=%-8s | edge_net=%+.3f\n"
+            "                        stored_family: %s\n"
+            "                        new_family:    %s%s\n"
+            "                        manifold_new:  %s\n"
+            "                        reason: %s",
+            p["recommendation"],
+            p["market_id"], p["direction"],
+            p.get("edge_net") or 0.0,
+            p["family_key_old"] or "none",
+            p["family_key_new"],
+            " [CHANGED]" if p["fk_changed"] else "",
+            f"{p['manifold_prob_new']:.3f}" if p["manifold_prob_new"] is not None else "n/a",
+            p["rec_reason"],
+        )
+    log.warning("━" * 70)
+
+    # Step 5: auto-close in paper mode
+    if paper_mode and n_close > 0 and isinstance(executor, PaperExecutor):
+        log.warning("PAPER MODE: auto-closing %d CLOSE_RECOMMENDED position(s)...", n_close)
+        for p in enriched:
+            if p["recommendation"] == "CLOSE_RECOMMENDED":
+                recovered = await executor.close_legacy_position(p["market_id"], p["rec_reason"])
+                log.warning(
+                    "  AUTO_CLOSED market=%s | $%.2f returned to cash | %s",
+                    p["market_id"], recovered, p["question"][:60],
+                )
+        log.warning("Legacy scan closures complete.")
+    elif n_close > 0:
+        log.warning("REAL MODE: %d position(s) marked CLOSE_RECOMMENDED — close manually.", n_close)
+
+
 async def main() -> None:
     if PAPER_MODE:
         log.info("=" * 60)
@@ -202,29 +334,15 @@ async def main() -> None:
     if PAPER_MODE:
         await executor.initialize()
 
-    # Contradiction scan: warn if any two open positions share a family_key.
-    # This can happen when the family logic was less strict on a prior deploy.
-    # Bot does NOT auto-close — operator decides which position to keep.
-    positions = await db.get_open_position_details()
-    family_map: dict[str, list[dict]] = {}
-    for pos in positions:
-        fk = pos.get("family_key") or ""
-        if fk:
-            family_map.setdefault(fk, []).append(pos)
-    for fk, members in family_map.items():
-        if len(members) > 1:
-            best = max(members, key=lambda p: p.get("edge_net") or 0.0)
-            log.warning(
-                "CONTRADICTION family=%s has %d open positions — recommend keeping market_id=%s (edge_net=%.3f):",
-                fk, len(members), best["market_id"], best.get("edge_net") or 0.0,
-            )
-            for m in members:
-                marker = "KEEP" if m["market_id"] == best["market_id"] else "REVIEW"
-                log.warning(
-                    "  [%s] %s | dir=%s | edge_net=%.3f | %s",
-                    marker, m["market_id"], m["direction"],
-                    m.get("edge_net") or 0.0, m["question"][:60],
-                )
+    # Legacy scan: re-key all open positions, detect contradictions, auto-close
+    # CLOSE_RECOMMENDED in paper mode.  Runs once at startup using a fresh
+    # market snapshot; the trading loop will re-fetch on its own first cycle.
+    try:
+        scan_markets = await poly.get_active_markets()
+    except Exception as e:
+        log.warning("Could not fetch markets for legacy scan: %s — scan skipped", e)
+        scan_markets = []
+    await run_legacy_scan(db, scan_markets, manifold, executor, PAPER_MODE)
 
     try:
         await run_trading_loop(poly, external, strategy, risk, executor, metrics, db)