diff --git a/bot/data/db.py b/bot/data/db.py index 886830f..e7f5927 100644 --- a/bot/data/db.py +++ b/bot/data/db.py @@ -530,6 +530,8 @@ class Database: match_score: Optional[float], match_reason: Optional[str], match_status: str, + poly_outcome_type: Optional[str] = None, + mfld_outcome_type: Optional[str] = None, ) -> None: async with self._pool.acquire() as conn: await conn.execute(""" @@ -537,13 +539,15 @@ class Database: id, poly_market_id, poly_question, search_query, mfld_market_id, mfld_market_title, mfld_market_url, prob_raw, prob_final, inverted, - match_score, match_reason, match_status, used_in_trade - ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,FALSE) + match_score, match_reason, match_status, used_in_trade, + poly_outcome_type, mfld_outcome_type + ) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,FALSE,$14,$15) """, audit_id, poly_market_id, poly_question, search_query, mfld_market_id, mfld_market_title, mfld_market_url, prob_raw, prob_final, inverted, match_score, match_reason, match_status, + poly_outcome_type, mfld_outcome_type, ) async def mark_manifold_audit_used(self, audit_id: str) -> None: diff --git a/bot/data/manifold.py b/bot/data/manifold.py index 445b5f1..a8706aa 100644 --- a/bot/data/manifold.py +++ b/bot/data/manifold.py @@ -6,6 +6,15 @@ by keyword overlap and returns a ManifoldMatchResult with full audit metadata. Match threshold: >= 0.40 Jaccard overlap (raised from 0.25 for stricter semantics). +Outcome compatibility guard (conservative): + - Conditional Manifold markets ("If X, will Y?" / "Conditional on..." / "Assuming..." + / "Given that..." / mid-sentence "...if X is nominated, will...") are rejected: + a premise-gated question is not equivalent to a direct outcome question even when + token overlap is high. reason='conditional_market'. + - Each side is classified into an outcome_type (nomination | primary_win | + general_win | conditional | other). Matches with differing outcome_type — or any + conditional side — are rejected. reason='outcome_mismatch: poly=... manifold=...'. + Inversion guard (conservative): - If Polymarket question names a party (democrat/republican) AND the matched Manifold market names the OPPOSITE party → invert probability (1 - prob). @@ -59,6 +68,8 @@ class ManifoldMatchResult: match_reason: Optional[str] = None # human-readable explanation inverted: bool = False search_query: str = "" + poly_outcome_type: Optional[str] = None # nomination|primary_win|general_win|conditional|other + mfld_outcome_type: Optional[str] = None def _significant_words(text: str) -> set[str]: @@ -82,6 +93,45 @@ def _detect_party(text: str) -> Optional[str]: return None +# ── Conditional-market detection (Task 1) ────────────────────────────────────── +# A market is "conditional" when its resolution is gated on a premise rather than +# asking the outcome directly (e.g. "If X is the nominee, will he win?"). Such a +# market is NOT equivalent to a direct outcome question even with high token overlap. +_CONDITIONAL_PREFIXES = ("if ", "conditional on", "assuming ", "given that") +# " if ," — a mid-sentence conditional clause closed by a comma. +_CONDITIONAL_CLAUSE_RE = re.compile(r"\sif\s[^,]*,") + + +def _is_conditional(text: str) -> bool: + """True if the question is phrased conditionally (premise-gated).""" + t = (text or "").strip().lower() + if t.startswith(_CONDITIONAL_PREFIXES): + return True + return bool(_CONDITIONAL_CLAUSE_RE.search(t)) + + +def _classify_outcome(text: str) -> str: + """ + Coarse classification of what a question is *asking about*, used to reject + matches whose outcomes are not equivalent even when tokens overlap. + + Returns one of: nomination | primary_win | general_win | conditional | other. + Order matters: conditional is checked first (premise-gated), then nomination + (which subsumes "primary nominee"), then primary, then general election. + """ + t = (text or "").strip().lower() + if t.startswith(_CONDITIONAL_PREFIXES): + return "conditional" + if any(k in t for k in ("nominee", "nominated", "nomination")): + return "nomination" + if any(k in t for k in ("primary", "win the primary", "first round")): + return "primary_win" + if any(k in t for k in ("win the election", "win the race", + "win the seat", "general election")): + return "general_win" + return "other" + + def _find_best_candidate(poly_question: str, results: list[dict]) -> tuple[Optional[dict], float]: """Find the highest-scoring open binary Manifold market by Jaccard overlap.""" poly_words = _significant_words(poly_question) @@ -137,9 +187,14 @@ class ManifoldClient: if cached and (now - cached[0]) < CACHE_TTL_SEC: return cached[1] + poly_outcome = _classify_outcome(question) + query = _build_search_query(question) if not query: - result = ManifoldMatchResult(status="no_results", search_query="") + result = ManifoldMatchResult( + status="no_results", search_query="", + poly_outcome_type=poly_outcome, + ) self._cache[question] = (now, result) return result @@ -152,12 +207,18 @@ class ManifoldClient: results = resp.json() except Exception as exc: log.warning("Manifold API error for %r: %s", question[:40], exc) - result = ManifoldMatchResult(status="no_results", search_query=query) + result = ManifoldMatchResult( + status="no_results", search_query=query, + poly_outcome_type=poly_outcome, + ) self._cache[question] = (now, result) return result if not results: - result = ManifoldMatchResult(status="no_results", search_query=query) + result = ManifoldMatchResult( + status="no_results", search_query=query, + poly_outcome_type=poly_outcome, + ) self._cache[question] = (now, result) return result @@ -176,22 +237,36 @@ class ManifoldClient: match_score=score if best else None, match_reason=reason, search_query=query, + poly_outcome_type=poly_outcome, + mfld_outcome_type=_classify_outcome(best.get("question", "")) if best else None, ) self._cache[question] = (now, result) return result - # ── Inversion analysis (conservative) ──────────────────────────────── + # ── Outcome compatibility + inversion analysis (conservative) ───────── + mfld_title = best.get("question", "") + mfld_outcome = _classify_outcome(mfld_title) poly_party = _detect_party(question) - manifold_party = _detect_party(best.get("question", "")) + manifold_party = _detect_party(mfld_title) poly_words = _significant_words(question) - mfld_words = _significant_words(best.get("question", "")) + mfld_words = _significant_words(mfld_title) matched_tokens = sorted(poly_words & mfld_words)[:6] inverted = False rejection_reason: Optional[str] = None - if poly_party is not None: + # Task 1 — conditional Manifold market is never equivalent to a direct + # outcome question, regardless of token overlap. + if _is_conditional(mfld_title): + rejection_reason = "conditional_market: manifold question is conditional" + # Task 2 — outcome types must match; any conditional side is rejected. + elif (poly_outcome == "conditional" or mfld_outcome == "conditional" + or poly_outcome != mfld_outcome): + rejection_reason = ( + f"outcome_mismatch: poly={poly_outcome} manifold={mfld_outcome}" + ) + elif poly_party is not None: if manifold_party is None: # Poly specifies a party; Manifold does not → can't verify inversion safety rejection_reason = ( @@ -219,6 +294,8 @@ class ManifoldClient: f"jaccard={score:.2f}, tokens={matched_tokens}, {rejection_reason}" ), search_query=query, + poly_outcome_type=poly_outcome, + mfld_outcome_type=mfld_outcome, ) self._cache[question] = (now, result) return result @@ -257,6 +334,8 @@ class ManifoldClient: match_reason=match_reason, inverted=inverted, search_query=query, + poly_outcome_type=poly_outcome, + mfld_outcome_type=mfld_outcome, ) self._cache[question] = (now, result) return result diff --git a/bot/data/schema.sql b/bot/data/schema.sql index 26c1df5..bc699a5 100644 --- a/bot/data/schema.sql +++ b/bot/data/schema.sql @@ -207,13 +207,19 @@ CREATE TABLE IF NOT EXISTS manifold_match_audit ( match_score DOUBLE PRECISION, match_reason TEXT, match_status TEXT NOT NULL, - used_in_trade BOOLEAN DEFAULT FALSE + used_in_trade BOOLEAN DEFAULT FALSE, + poly_outcome_type TEXT, + mfld_outcome_type TEXT ); CREATE INDEX IF NOT EXISTS idx_mfld_audit_timestamp ON manifold_match_audit(timestamp DESC); CREATE INDEX IF NOT EXISTS idx_mfld_audit_status ON manifold_match_audit(match_status); CREATE INDEX IF NOT EXISTS idx_mfld_audit_poly_mkt ON manifold_match_audit(poly_market_id); +-- Backfill outcome-type columns on pre-existing tables (idempotent). +ALTER TABLE manifold_match_audit ADD COLUMN IF NOT EXISTS poly_outcome_type TEXT; +ALTER TABLE manifold_match_audit ADD COLUMN IF NOT EXISTS mfld_outcome_type TEXT; + -- ───────────────────────────────────────────────────────────────────────────── -- Metric exclusion — administrative closure flag -- diff --git a/bot/strategy/bayesian.py b/bot/strategy/bayesian.py index 007564e..7034834 100644 --- a/bot/strategy/bayesian.py +++ b/bot/strategy/bayesian.py @@ -470,6 +470,8 @@ class BayesianStrategy: match_score=manifold_result.match_score, match_reason=manifold_result.match_reason, match_status=manifold_result.status, + poly_outcome_type=manifold_result.poly_outcome_type, + mfld_outcome_type=manifold_result.mfld_outcome_type, ) except Exception as exc: log.warning("Failed to save manifold audit: %s", exc) diff --git a/tests/test_manifold_outcome.py b/tests/test_manifold_outcome.py new file mode 100644 index 0000000..0c3fcb8 --- /dev/null +++ b/tests/test_manifold_outcome.py @@ -0,0 +1,152 @@ +""" +Tests for the Manifold outcome-compatibility guard. + +Regression: a Polymarket *nomination* question must not match a Manifold +*conditional* question ("If X is the nominee, will he win?") even at Jaccard=1.0. +""" +import asyncio + +import pytest + +from bot.data.manifold import ( + ManifoldClient, + _classify_outcome, + _is_conditional, +) + + +# ── _is_conditional ──────────────────────────────────────────────────────────── +def test_is_conditional_prefixes(): + assert _is_conditional("If Graham Platner is the nominee, will he win?") + assert _is_conditional("Conditional on a recession, will rates fall?") + assert _is_conditional("Assuming Trump runs, will he win?") + assert _is_conditional("Given that X happens, will Y?") + + +def test_is_conditional_midsentence_clause(): + assert _is_conditional("Will Biden, if he is nominated, win the election?") + + +def test_is_not_conditional(): + assert not _is_conditional("Will Graham Platner be the Democratic nominee?") + assert not _is_conditional("Will the GOP win the Senate?") + # "if" without a closing comma clause is not flagged + assert not _is_conditional("What happens if everything goes right") + + +# ── _classify_outcome ─────────────────────────────────────────────────────────── +def test_classify_nomination(): + assert _classify_outcome("Will X be the Democratic nominee for Senate?") == "nomination" + assert _classify_outcome("Will X be nominated?") == "nomination" + # "primary nominee" → nomination (checked before primary) + assert _classify_outcome("Will X be the primary nominee?") == "nomination" + + +def test_classify_primary_win(): + assert _classify_outcome("Will X win the primary?") == "primary_win" + assert _classify_outcome("Will X advance in the first round?") == "primary_win" + + +def test_classify_general_win(): + assert _classify_outcome("Will X win the election?") == "general_win" + assert _classify_outcome("Will X win the seat?") == "general_win" + assert _classify_outcome("Will X win the general election?") == "general_win" + + +def test_classify_conditional(): + assert _classify_outcome("If X is the nominee, will he win?") == "conditional" + assert _classify_outcome("Assuming a runoff, who wins?") == "conditional" + + +def test_classify_other(): + assert _classify_outcome("Will it rain tomorrow?") == "other" + + +# ── End-to-end get_match with a stubbed Manifold API ──────────────────────────── +class _StubResponse: + def __init__(self, payload): + self._payload = payload + + def raise_for_status(self): + pass + + def json(self): + return self._payload + + +class _StubHTTP: + def __init__(self, payload): + self._payload = payload + + async def get(self, *args, **kwargs): + return _StubResponse(self._payload) + + async def aclose(self): + pass + + +async def _match(poly, mfld_market): + client = ManifoldClient() + client._client = _StubHTTP([mfld_market]) + try: + return await client.get_match(poly) + finally: + await client.close() + + +def test_graham_platner_conditional_rejected(): + """Poly nomination vs Manifold conditional → rejected (Task 4.1).""" + poly = ("Will Graham Platner be the Democratic nominee for Senate " + "in Maine in 2026?") + mfld_market = { + "outcomeType": "BINARY", + "probability": 0.55, + "question": ("If Graham Platner is the Democratic nominee for Senate " + "in Maine, will he win the general election?"), + "id": "abc123", + "slug": "graham-platner-win", + "creatorUsername": "someone", + } + result = asyncio.run(_match(poly, mfld_market)) + + assert result.status == "rejected" + assert result.match_reason is not None + assert ("conditional" in result.match_reason + or "outcome_mismatch" in result.match_reason) + # outcome types are classified and available for persistence + assert result.poly_outcome_type == "nomination" + assert result.mfld_outcome_type == "conditional" + + +def test_outcome_mismatch_nomination_vs_general_rejected(): + """Poly nomination vs Manifold general_win (non-conditional) → rejected.""" + poly = "Will Jane Doe be the Republican nominee for Governor?" + mfld_market = { + "outcomeType": "BINARY", + "probability": 0.4, + "question": "Will Jane Doe win the election for Governor?", + "id": "x", "slug": "jane-doe", "creatorUsername": "u", + } + result = asyncio.run(_match(poly, mfld_market)) + + assert result.status == "rejected" + assert "outcome_mismatch" in result.match_reason + assert result.poly_outcome_type == "nomination" + assert result.mfld_outcome_type == "general_win" + + +def test_matching_nomination_accepted(): + """Poly nomination vs Manifold nomination (same outcome) → accepted.""" + poly = "Will Graham Platner be the Democratic nominee for Senate in Maine?" + mfld_market = { + "outcomeType": "BINARY", + "probability": 0.62, + "question": "Will Graham Platner be the Democratic Senate nominee in Maine?", + "id": "ok", "slug": "platner-nominee", "creatorUsername": "u", + } + result = asyncio.run(_match(poly, mfld_market)) + + assert result.status == "accepted" + assert result.poly_outcome_type == "nomination" + assert result.mfld_outcome_type == "nomination" + assert result.prob_final == pytest.approx(0.62)