feat(manifold): add outcome compatibility guard and conditional market rejection
CI/CD / build-and-push (push) Successful in 7s
CI/CD / build-and-push (push) Successful in 7s
Reject false-positive matches where Jaccard overlap is high but the outcome is not equivalent (e.g. Poly nomination vs Manifold "If X is nominee, will he win"). - _is_conditional(): detect conditional Manifold markets (If/Conditional on/ Assuming/Given that prefixes + mid-sentence " if ...," clauses) -> reject with reason "conditional_market". - _classify_outcome(): classify into nomination|primary_win|general_win| conditional|other; reject when poly/mfld types differ or either is conditional -> reason "outcome_mismatch: poly=... manifold=...". - Persist poly_outcome_type/mfld_outcome_type on ManifoldMatchResult, in manifold_match_audit (CREATE + idempotent ALTER), save_manifold_audit() and the bayesian call site. - Tests covering classification, conditional detection and the Graham Platner regression (now rejected); valid nomination<->nomination still accepted. Untouched: _MATCH_THRESHOLD (0.40), MANIFOLD_LOGODDS_WEIGHT, edge thresholds, exposure, trading logic. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,152 @@
|
||||
"""
|
||||
Tests for the Manifold outcome-compatibility guard.
|
||||
|
||||
Regression: a Polymarket *nomination* question must not match a Manifold
|
||||
*conditional* question ("If X is the nominee, will he win?") even at Jaccard=1.0.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from bot.data.manifold import (
|
||||
ManifoldClient,
|
||||
_classify_outcome,
|
||||
_is_conditional,
|
||||
)
|
||||
|
||||
|
||||
# ── _is_conditional ────────────────────────────────────────────────────────────
|
||||
def test_is_conditional_prefixes():
|
||||
assert _is_conditional("If Graham Platner is the nominee, will he win?")
|
||||
assert _is_conditional("Conditional on a recession, will rates fall?")
|
||||
assert _is_conditional("Assuming Trump runs, will he win?")
|
||||
assert _is_conditional("Given that X happens, will Y?")
|
||||
|
||||
|
||||
def test_is_conditional_midsentence_clause():
|
||||
assert _is_conditional("Will Biden, if he is nominated, win the election?")
|
||||
|
||||
|
||||
def test_is_not_conditional():
|
||||
assert not _is_conditional("Will Graham Platner be the Democratic nominee?")
|
||||
assert not _is_conditional("Will the GOP win the Senate?")
|
||||
# "if" without a closing comma clause is not flagged
|
||||
assert not _is_conditional("What happens if everything goes right")
|
||||
|
||||
|
||||
# ── _classify_outcome ───────────────────────────────────────────────────────────
|
||||
def test_classify_nomination():
|
||||
assert _classify_outcome("Will X be the Democratic nominee for Senate?") == "nomination"
|
||||
assert _classify_outcome("Will X be nominated?") == "nomination"
|
||||
# "primary nominee" → nomination (checked before primary)
|
||||
assert _classify_outcome("Will X be the primary nominee?") == "nomination"
|
||||
|
||||
|
||||
def test_classify_primary_win():
|
||||
assert _classify_outcome("Will X win the primary?") == "primary_win"
|
||||
assert _classify_outcome("Will X advance in the first round?") == "primary_win"
|
||||
|
||||
|
||||
def test_classify_general_win():
|
||||
assert _classify_outcome("Will X win the election?") == "general_win"
|
||||
assert _classify_outcome("Will X win the seat?") == "general_win"
|
||||
assert _classify_outcome("Will X win the general election?") == "general_win"
|
||||
|
||||
|
||||
def test_classify_conditional():
|
||||
assert _classify_outcome("If X is the nominee, will he win?") == "conditional"
|
||||
assert _classify_outcome("Assuming a runoff, who wins?") == "conditional"
|
||||
|
||||
|
||||
def test_classify_other():
|
||||
assert _classify_outcome("Will it rain tomorrow?") == "other"
|
||||
|
||||
|
||||
# ── End-to-end get_match with a stubbed Manifold API ────────────────────────────
|
||||
class _StubResponse:
|
||||
def __init__(self, payload):
|
||||
self._payload = payload
|
||||
|
||||
def raise_for_status(self):
|
||||
pass
|
||||
|
||||
def json(self):
|
||||
return self._payload
|
||||
|
||||
|
||||
class _StubHTTP:
|
||||
def __init__(self, payload):
|
||||
self._payload = payload
|
||||
|
||||
async def get(self, *args, **kwargs):
|
||||
return _StubResponse(self._payload)
|
||||
|
||||
async def aclose(self):
|
||||
pass
|
||||
|
||||
|
||||
async def _match(poly, mfld_market):
|
||||
client = ManifoldClient()
|
||||
client._client = _StubHTTP([mfld_market])
|
||||
try:
|
||||
return await client.get_match(poly)
|
||||
finally:
|
||||
await client.close()
|
||||
|
||||
|
||||
def test_graham_platner_conditional_rejected():
|
||||
"""Poly nomination vs Manifold conditional → rejected (Task 4.1)."""
|
||||
poly = ("Will Graham Platner be the Democratic nominee for Senate "
|
||||
"in Maine in 2026?")
|
||||
mfld_market = {
|
||||
"outcomeType": "BINARY",
|
||||
"probability": 0.55,
|
||||
"question": ("If Graham Platner is the Democratic nominee for Senate "
|
||||
"in Maine, will he win the general election?"),
|
||||
"id": "abc123",
|
||||
"slug": "graham-platner-win",
|
||||
"creatorUsername": "someone",
|
||||
}
|
||||
result = asyncio.run(_match(poly, mfld_market))
|
||||
|
||||
assert result.status == "rejected"
|
||||
assert result.match_reason is not None
|
||||
assert ("conditional" in result.match_reason
|
||||
or "outcome_mismatch" in result.match_reason)
|
||||
# outcome types are classified and available for persistence
|
||||
assert result.poly_outcome_type == "nomination"
|
||||
assert result.mfld_outcome_type == "conditional"
|
||||
|
||||
|
||||
def test_outcome_mismatch_nomination_vs_general_rejected():
|
||||
"""Poly nomination vs Manifold general_win (non-conditional) → rejected."""
|
||||
poly = "Will Jane Doe be the Republican nominee for Governor?"
|
||||
mfld_market = {
|
||||
"outcomeType": "BINARY",
|
||||
"probability": 0.4,
|
||||
"question": "Will Jane Doe win the election for Governor?",
|
||||
"id": "x", "slug": "jane-doe", "creatorUsername": "u",
|
||||
}
|
||||
result = asyncio.run(_match(poly, mfld_market))
|
||||
|
||||
assert result.status == "rejected"
|
||||
assert "outcome_mismatch" in result.match_reason
|
||||
assert result.poly_outcome_type == "nomination"
|
||||
assert result.mfld_outcome_type == "general_win"
|
||||
|
||||
|
||||
def test_matching_nomination_accepted():
|
||||
"""Poly nomination vs Manifold nomination (same outcome) → accepted."""
|
||||
poly = "Will Graham Platner be the Democratic nominee for Senate in Maine?"
|
||||
mfld_market = {
|
||||
"outcomeType": "BINARY",
|
||||
"probability": 0.62,
|
||||
"question": "Will Graham Platner be the Democratic Senate nominee in Maine?",
|
||||
"id": "ok", "slug": "platner-nominee", "creatorUsername": "u",
|
||||
}
|
||||
result = asyncio.run(_match(poly, mfld_market))
|
||||
|
||||
assert result.status == "accepted"
|
||||
assert result.poly_outcome_type == "nomination"
|
||||
assert result.mfld_outcome_type == "nomination"
|
||||
assert result.prob_final == pytest.approx(0.62)
|
||||
Reference in New Issue
Block a user