feat(manifold): add outcome compatibility guard and conditional market rejection
CI/CD / build-and-push (push) Successful in 7s
CI/CD / build-and-push (push) Successful in 7s
Reject false-positive matches where Jaccard overlap is high but the outcome is not equivalent (e.g. Poly nomination vs Manifold "If X is nominee, will he win"). - _is_conditional(): detect conditional Manifold markets (If/Conditional on/ Assuming/Given that prefixes + mid-sentence " if ...," clauses) -> reject with reason "conditional_market". - _classify_outcome(): classify into nomination|primary_win|general_win| conditional|other; reject when poly/mfld types differ or either is conditional -> reason "outcome_mismatch: poly=... manifold=...". - Persist poly_outcome_type/mfld_outcome_type on ManifoldMatchResult, in manifold_match_audit (CREATE + idempotent ALTER), save_manifold_audit() and the bayesian call site. - Tests covering classification, conditional detection and the Graham Platner regression (now rejected); valid nomination<->nomination still accepted. Untouched: _MATCH_THRESHOLD (0.40), MANIFOLD_LOGODDS_WEIGHT, edge thresholds, exposure, trading logic. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+6
-2
@@ -530,6 +530,8 @@ class Database:
|
||||
match_score: Optional[float],
|
||||
match_reason: Optional[str],
|
||||
match_status: str,
|
||||
poly_outcome_type: Optional[str] = None,
|
||||
mfld_outcome_type: Optional[str] = None,
|
||||
) -> None:
|
||||
async with self._pool.acquire() as conn:
|
||||
await conn.execute("""
|
||||
@@ -537,13 +539,15 @@ class Database:
|
||||
id, poly_market_id, poly_question, search_query,
|
||||
mfld_market_id, mfld_market_title, mfld_market_url,
|
||||
prob_raw, prob_final, inverted,
|
||||
match_score, match_reason, match_status, used_in_trade
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,FALSE)
|
||||
match_score, match_reason, match_status, used_in_trade,
|
||||
poly_outcome_type, mfld_outcome_type
|
||||
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,FALSE,$14,$15)
|
||||
""",
|
||||
audit_id, poly_market_id, poly_question, search_query,
|
||||
mfld_market_id, mfld_market_title, mfld_market_url,
|
||||
prob_raw, prob_final, inverted,
|
||||
match_score, match_reason, match_status,
|
||||
poly_outcome_type, mfld_outcome_type,
|
||||
)
|
||||
|
||||
async def mark_manifold_audit_used(self, audit_id: str) -> None:
|
||||
|
||||
+86
-7
@@ -6,6 +6,15 @@ by keyword overlap and returns a ManifoldMatchResult with full audit metadata.
|
||||
|
||||
Match threshold: >= 0.40 Jaccard overlap (raised from 0.25 for stricter semantics).
|
||||
|
||||
Outcome compatibility guard (conservative):
|
||||
- Conditional Manifold markets ("If X, will Y?" / "Conditional on..." / "Assuming..."
|
||||
/ "Given that..." / mid-sentence "...if X is nominated, will...") are rejected:
|
||||
a premise-gated question is not equivalent to a direct outcome question even when
|
||||
token overlap is high. reason='conditional_market'.
|
||||
- Each side is classified into an outcome_type (nomination | primary_win |
|
||||
general_win | conditional | other). Matches with differing outcome_type — or any
|
||||
conditional side — are rejected. reason='outcome_mismatch: poly=... manifold=...'.
|
||||
|
||||
Inversion guard (conservative):
|
||||
- If Polymarket question names a party (democrat/republican) AND the matched
|
||||
Manifold market names the OPPOSITE party → invert probability (1 - prob).
|
||||
@@ -59,6 +68,8 @@ class ManifoldMatchResult:
|
||||
match_reason: Optional[str] = None # human-readable explanation
|
||||
inverted: bool = False
|
||||
search_query: str = ""
|
||||
poly_outcome_type: Optional[str] = None # nomination|primary_win|general_win|conditional|other
|
||||
mfld_outcome_type: Optional[str] = None
|
||||
|
||||
|
||||
def _significant_words(text: str) -> set[str]:
|
||||
@@ -82,6 +93,45 @@ def _detect_party(text: str) -> Optional[str]:
|
||||
return None
|
||||
|
||||
|
||||
# ── Conditional-market detection (Task 1) ──────────────────────────────────────
|
||||
# A market is "conditional" when its resolution is gated on a premise rather than
|
||||
# asking the outcome directly (e.g. "If X is the nominee, will he win?"). Such a
|
||||
# market is NOT equivalent to a direct outcome question even with high token overlap.
|
||||
_CONDITIONAL_PREFIXES = ("if ", "conditional on", "assuming ", "given that")
|
||||
# " if <clause>," — a mid-sentence conditional clause closed by a comma.
|
||||
_CONDITIONAL_CLAUSE_RE = re.compile(r"\sif\s[^,]*,")
|
||||
|
||||
|
||||
def _is_conditional(text: str) -> bool:
|
||||
"""True if the question is phrased conditionally (premise-gated)."""
|
||||
t = (text or "").strip().lower()
|
||||
if t.startswith(_CONDITIONAL_PREFIXES):
|
||||
return True
|
||||
return bool(_CONDITIONAL_CLAUSE_RE.search(t))
|
||||
|
||||
|
||||
def _classify_outcome(text: str) -> str:
|
||||
"""
|
||||
Coarse classification of what a question is *asking about*, used to reject
|
||||
matches whose outcomes are not equivalent even when tokens overlap.
|
||||
|
||||
Returns one of: nomination | primary_win | general_win | conditional | other.
|
||||
Order matters: conditional is checked first (premise-gated), then nomination
|
||||
(which subsumes "primary nominee"), then primary, then general election.
|
||||
"""
|
||||
t = (text or "").strip().lower()
|
||||
if t.startswith(_CONDITIONAL_PREFIXES):
|
||||
return "conditional"
|
||||
if any(k in t for k in ("nominee", "nominated", "nomination")):
|
||||
return "nomination"
|
||||
if any(k in t for k in ("primary", "win the primary", "first round")):
|
||||
return "primary_win"
|
||||
if any(k in t for k in ("win the election", "win the race",
|
||||
"win the seat", "general election")):
|
||||
return "general_win"
|
||||
return "other"
|
||||
|
||||
|
||||
def _find_best_candidate(poly_question: str, results: list[dict]) -> tuple[Optional[dict], float]:
|
||||
"""Find the highest-scoring open binary Manifold market by Jaccard overlap."""
|
||||
poly_words = _significant_words(poly_question)
|
||||
@@ -137,9 +187,14 @@ class ManifoldClient:
|
||||
if cached and (now - cached[0]) < CACHE_TTL_SEC:
|
||||
return cached[1]
|
||||
|
||||
poly_outcome = _classify_outcome(question)
|
||||
|
||||
query = _build_search_query(question)
|
||||
if not query:
|
||||
result = ManifoldMatchResult(status="no_results", search_query="")
|
||||
result = ManifoldMatchResult(
|
||||
status="no_results", search_query="",
|
||||
poly_outcome_type=poly_outcome,
|
||||
)
|
||||
self._cache[question] = (now, result)
|
||||
return result
|
||||
|
||||
@@ -152,12 +207,18 @@ class ManifoldClient:
|
||||
results = resp.json()
|
||||
except Exception as exc:
|
||||
log.warning("Manifold API error for %r: %s", question[:40], exc)
|
||||
result = ManifoldMatchResult(status="no_results", search_query=query)
|
||||
result = ManifoldMatchResult(
|
||||
status="no_results", search_query=query,
|
||||
poly_outcome_type=poly_outcome,
|
||||
)
|
||||
self._cache[question] = (now, result)
|
||||
return result
|
||||
|
||||
if not results:
|
||||
result = ManifoldMatchResult(status="no_results", search_query=query)
|
||||
result = ManifoldMatchResult(
|
||||
status="no_results", search_query=query,
|
||||
poly_outcome_type=poly_outcome,
|
||||
)
|
||||
self._cache[question] = (now, result)
|
||||
return result
|
||||
|
||||
@@ -176,22 +237,36 @@ class ManifoldClient:
|
||||
match_score=score if best else None,
|
||||
match_reason=reason,
|
||||
search_query=query,
|
||||
poly_outcome_type=poly_outcome,
|
||||
mfld_outcome_type=_classify_outcome(best.get("question", "")) if best else None,
|
||||
)
|
||||
self._cache[question] = (now, result)
|
||||
return result
|
||||
|
||||
# ── Inversion analysis (conservative) ────────────────────────────────
|
||||
# ── Outcome compatibility + inversion analysis (conservative) ─────────
|
||||
mfld_title = best.get("question", "")
|
||||
mfld_outcome = _classify_outcome(mfld_title)
|
||||
poly_party = _detect_party(question)
|
||||
manifold_party = _detect_party(best.get("question", ""))
|
||||
manifold_party = _detect_party(mfld_title)
|
||||
|
||||
poly_words = _significant_words(question)
|
||||
mfld_words = _significant_words(best.get("question", ""))
|
||||
mfld_words = _significant_words(mfld_title)
|
||||
matched_tokens = sorted(poly_words & mfld_words)[:6]
|
||||
|
||||
inverted = False
|
||||
rejection_reason: Optional[str] = None
|
||||
|
||||
if poly_party is not None:
|
||||
# Task 1 — conditional Manifold market is never equivalent to a direct
|
||||
# outcome question, regardless of token overlap.
|
||||
if _is_conditional(mfld_title):
|
||||
rejection_reason = "conditional_market: manifold question is conditional"
|
||||
# Task 2 — outcome types must match; any conditional side is rejected.
|
||||
elif (poly_outcome == "conditional" or mfld_outcome == "conditional"
|
||||
or poly_outcome != mfld_outcome):
|
||||
rejection_reason = (
|
||||
f"outcome_mismatch: poly={poly_outcome} manifold={mfld_outcome}"
|
||||
)
|
||||
elif poly_party is not None:
|
||||
if manifold_party is None:
|
||||
# Poly specifies a party; Manifold does not → can't verify inversion safety
|
||||
rejection_reason = (
|
||||
@@ -219,6 +294,8 @@ class ManifoldClient:
|
||||
f"jaccard={score:.2f}, tokens={matched_tokens}, {rejection_reason}"
|
||||
),
|
||||
search_query=query,
|
||||
poly_outcome_type=poly_outcome,
|
||||
mfld_outcome_type=mfld_outcome,
|
||||
)
|
||||
self._cache[question] = (now, result)
|
||||
return result
|
||||
@@ -257,6 +334,8 @@ class ManifoldClient:
|
||||
match_reason=match_reason,
|
||||
inverted=inverted,
|
||||
search_query=query,
|
||||
poly_outcome_type=poly_outcome,
|
||||
mfld_outcome_type=mfld_outcome,
|
||||
)
|
||||
self._cache[question] = (now, result)
|
||||
return result
|
||||
|
||||
+7
-1
@@ -207,13 +207,19 @@ CREATE TABLE IF NOT EXISTS manifold_match_audit (
|
||||
match_score DOUBLE PRECISION,
|
||||
match_reason TEXT,
|
||||
match_status TEXT NOT NULL,
|
||||
used_in_trade BOOLEAN DEFAULT FALSE
|
||||
used_in_trade BOOLEAN DEFAULT FALSE,
|
||||
poly_outcome_type TEXT,
|
||||
mfld_outcome_type TEXT
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_mfld_audit_timestamp ON manifold_match_audit(timestamp DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_mfld_audit_status ON manifold_match_audit(match_status);
|
||||
CREATE INDEX IF NOT EXISTS idx_mfld_audit_poly_mkt ON manifold_match_audit(poly_market_id);
|
||||
|
||||
-- Backfill outcome-type columns on pre-existing tables (idempotent).
|
||||
ALTER TABLE manifold_match_audit ADD COLUMN IF NOT EXISTS poly_outcome_type TEXT;
|
||||
ALTER TABLE manifold_match_audit ADD COLUMN IF NOT EXISTS mfld_outcome_type TEXT;
|
||||
|
||||
-- ─────────────────────────────────────────────────────────────────────────────
|
||||
-- Metric exclusion — administrative closure flag
|
||||
--
|
||||
|
||||
Reference in New Issue
Block a user