feat(manifold): add outcome compatibility guard and conditional market rejection
CI/CD / build-and-push (push) Successful in 7s
CI/CD / build-and-push (push) Successful in 7s
Reject false-positive matches where Jaccard overlap is high but the outcome is not equivalent (e.g. Poly nomination vs Manifold "If X is nominee, will he win"). - _is_conditional(): detect conditional Manifold markets (If/Conditional on/ Assuming/Given that prefixes + mid-sentence " if ...," clauses) -> reject with reason "conditional_market". - _classify_outcome(): classify into nomination|primary_win|general_win| conditional|other; reject when poly/mfld types differ or either is conditional -> reason "outcome_mismatch: poly=... manifold=...". - Persist poly_outcome_type/mfld_outcome_type on ManifoldMatchResult, in manifold_match_audit (CREATE + idempotent ALTER), save_manifold_audit() and the bayesian call site. - Tests covering classification, conditional detection and the Graham Platner regression (now rejected); valid nomination<->nomination still accepted. Untouched: _MATCH_THRESHOLD (0.40), MANIFOLD_LOGODDS_WEIGHT, edge thresholds, exposure, trading logic. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
+86
-7
@@ -6,6 +6,15 @@ by keyword overlap and returns a ManifoldMatchResult with full audit metadata.
|
||||
|
||||
Match threshold: >= 0.40 Jaccard overlap (raised from 0.25 for stricter semantics).
|
||||
|
||||
Outcome compatibility guard (conservative):
|
||||
- Conditional Manifold markets ("If X, will Y?" / "Conditional on..." / "Assuming..."
|
||||
/ "Given that..." / mid-sentence "...if X is nominated, will...") are rejected:
|
||||
a premise-gated question is not equivalent to a direct outcome question even when
|
||||
token overlap is high. reason='conditional_market'.
|
||||
- Each side is classified into an outcome_type (nomination | primary_win |
|
||||
general_win | conditional | other). Matches with differing outcome_type — or any
|
||||
conditional side — are rejected. reason='outcome_mismatch: poly=... manifold=...'.
|
||||
|
||||
Inversion guard (conservative):
|
||||
- If Polymarket question names a party (democrat/republican) AND the matched
|
||||
Manifold market names the OPPOSITE party → invert probability (1 - prob).
|
||||
@@ -59,6 +68,8 @@ class ManifoldMatchResult:
|
||||
match_reason: Optional[str] = None # human-readable explanation
|
||||
inverted: bool = False
|
||||
search_query: str = ""
|
||||
poly_outcome_type: Optional[str] = None # nomination|primary_win|general_win|conditional|other
|
||||
mfld_outcome_type: Optional[str] = None
|
||||
|
||||
|
||||
def _significant_words(text: str) -> set[str]:
|
||||
@@ -82,6 +93,45 @@ def _detect_party(text: str) -> Optional[str]:
|
||||
return None
|
||||
|
||||
|
||||
# ── Conditional-market detection (Task 1) ──────────────────────────────────────
|
||||
# A market is "conditional" when its resolution is gated on a premise rather than
|
||||
# asking the outcome directly (e.g. "If X is the nominee, will he win?"). Such a
|
||||
# market is NOT equivalent to a direct outcome question even with high token overlap.
|
||||
_CONDITIONAL_PREFIXES = ("if ", "conditional on", "assuming ", "given that")
|
||||
# " if <clause>," — a mid-sentence conditional clause closed by a comma.
|
||||
_CONDITIONAL_CLAUSE_RE = re.compile(r"\sif\s[^,]*,")
|
||||
|
||||
|
||||
def _is_conditional(text: str) -> bool:
|
||||
"""True if the question is phrased conditionally (premise-gated)."""
|
||||
t = (text or "").strip().lower()
|
||||
if t.startswith(_CONDITIONAL_PREFIXES):
|
||||
return True
|
||||
return bool(_CONDITIONAL_CLAUSE_RE.search(t))
|
||||
|
||||
|
||||
def _classify_outcome(text: str) -> str:
|
||||
"""
|
||||
Coarse classification of what a question is *asking about*, used to reject
|
||||
matches whose outcomes are not equivalent even when tokens overlap.
|
||||
|
||||
Returns one of: nomination | primary_win | general_win | conditional | other.
|
||||
Order matters: conditional is checked first (premise-gated), then nomination
|
||||
(which subsumes "primary nominee"), then primary, then general election.
|
||||
"""
|
||||
t = (text or "").strip().lower()
|
||||
if t.startswith(_CONDITIONAL_PREFIXES):
|
||||
return "conditional"
|
||||
if any(k in t for k in ("nominee", "nominated", "nomination")):
|
||||
return "nomination"
|
||||
if any(k in t for k in ("primary", "win the primary", "first round")):
|
||||
return "primary_win"
|
||||
if any(k in t for k in ("win the election", "win the race",
|
||||
"win the seat", "general election")):
|
||||
return "general_win"
|
||||
return "other"
|
||||
|
||||
|
||||
def _find_best_candidate(poly_question: str, results: list[dict]) -> tuple[Optional[dict], float]:
|
||||
"""Find the highest-scoring open binary Manifold market by Jaccard overlap."""
|
||||
poly_words = _significant_words(poly_question)
|
||||
@@ -137,9 +187,14 @@ class ManifoldClient:
|
||||
if cached and (now - cached[0]) < CACHE_TTL_SEC:
|
||||
return cached[1]
|
||||
|
||||
poly_outcome = _classify_outcome(question)
|
||||
|
||||
query = _build_search_query(question)
|
||||
if not query:
|
||||
result = ManifoldMatchResult(status="no_results", search_query="")
|
||||
result = ManifoldMatchResult(
|
||||
status="no_results", search_query="",
|
||||
poly_outcome_type=poly_outcome,
|
||||
)
|
||||
self._cache[question] = (now, result)
|
||||
return result
|
||||
|
||||
@@ -152,12 +207,18 @@ class ManifoldClient:
|
||||
results = resp.json()
|
||||
except Exception as exc:
|
||||
log.warning("Manifold API error for %r: %s", question[:40], exc)
|
||||
result = ManifoldMatchResult(status="no_results", search_query=query)
|
||||
result = ManifoldMatchResult(
|
||||
status="no_results", search_query=query,
|
||||
poly_outcome_type=poly_outcome,
|
||||
)
|
||||
self._cache[question] = (now, result)
|
||||
return result
|
||||
|
||||
if not results:
|
||||
result = ManifoldMatchResult(status="no_results", search_query=query)
|
||||
result = ManifoldMatchResult(
|
||||
status="no_results", search_query=query,
|
||||
poly_outcome_type=poly_outcome,
|
||||
)
|
||||
self._cache[question] = (now, result)
|
||||
return result
|
||||
|
||||
@@ -176,22 +237,36 @@ class ManifoldClient:
|
||||
match_score=score if best else None,
|
||||
match_reason=reason,
|
||||
search_query=query,
|
||||
poly_outcome_type=poly_outcome,
|
||||
mfld_outcome_type=_classify_outcome(best.get("question", "")) if best else None,
|
||||
)
|
||||
self._cache[question] = (now, result)
|
||||
return result
|
||||
|
||||
# ── Inversion analysis (conservative) ────────────────────────────────
|
||||
# ── Outcome compatibility + inversion analysis (conservative) ─────────
|
||||
mfld_title = best.get("question", "")
|
||||
mfld_outcome = _classify_outcome(mfld_title)
|
||||
poly_party = _detect_party(question)
|
||||
manifold_party = _detect_party(best.get("question", ""))
|
||||
manifold_party = _detect_party(mfld_title)
|
||||
|
||||
poly_words = _significant_words(question)
|
||||
mfld_words = _significant_words(best.get("question", ""))
|
||||
mfld_words = _significant_words(mfld_title)
|
||||
matched_tokens = sorted(poly_words & mfld_words)[:6]
|
||||
|
||||
inverted = False
|
||||
rejection_reason: Optional[str] = None
|
||||
|
||||
if poly_party is not None:
|
||||
# Task 1 — conditional Manifold market is never equivalent to a direct
|
||||
# outcome question, regardless of token overlap.
|
||||
if _is_conditional(mfld_title):
|
||||
rejection_reason = "conditional_market: manifold question is conditional"
|
||||
# Task 2 — outcome types must match; any conditional side is rejected.
|
||||
elif (poly_outcome == "conditional" or mfld_outcome == "conditional"
|
||||
or poly_outcome != mfld_outcome):
|
||||
rejection_reason = (
|
||||
f"outcome_mismatch: poly={poly_outcome} manifold={mfld_outcome}"
|
||||
)
|
||||
elif poly_party is not None:
|
||||
if manifold_party is None:
|
||||
# Poly specifies a party; Manifold does not → can't verify inversion safety
|
||||
rejection_reason = (
|
||||
@@ -219,6 +294,8 @@ class ManifoldClient:
|
||||
f"jaccard={score:.2f}, tokens={matched_tokens}, {rejection_reason}"
|
||||
),
|
||||
search_query=query,
|
||||
poly_outcome_type=poly_outcome,
|
||||
mfld_outcome_type=mfld_outcome,
|
||||
)
|
||||
self._cache[question] = (now, result)
|
||||
return result
|
||||
@@ -257,6 +334,8 @@ class ManifoldClient:
|
||||
match_reason=match_reason,
|
||||
inverted=inverted,
|
||||
search_query=query,
|
||||
poly_outcome_type=poly_outcome,
|
||||
mfld_outcome_type=mfld_outcome,
|
||||
)
|
||||
self._cache[question] = (now, result)
|
||||
return result
|
||||
|
||||
Reference in New Issue
Block a user