chemavx/polymarket-bot
1
0
Fork 0
Code Issues Pull Requests 10 Actions Packages Projects Releases Wiki Activity

feat(manifold): audit matching quality with ManifoldMatchResult and manifold_match_audit table
CI/CD / build-and-push (push) Successful in 14s
Details

Browse Source 
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
chemavx
2026-05-27 15:57:48 +00:00
parent ae7c737153
commit 9abaae44fd
8 changed files with 431 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bot/data/db.py
+83 -2
View File
@@ -36,11 +36,15 @@ class Database:
entry_price, shares, fee_usdc, net_cost, timestamp, reasoning, paper,
edge_gross, edge_net, prior_prob, final_prob,
mid_price, spread_estimate, commission, family_key,
feat_fg_lo, feat_mom_lo, feat_news_lo, feat_mfld_lo, feat_btc_dom_lo
feat_fg_lo, feat_mom_lo, feat_news_lo, feat_mfld_lo, feat_btc_dom_lo,
mfld_market_id, mfld_market_title, mfld_market_url,
mfld_prob_raw, mfld_prob_final, mfld_inverted,
mfld_match_score, mfld_match_reason, mfld_match_status
) VALUES (
$1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,
$13,$14,$15,$16,$17,$18,$19,$20,
$21,$22,$23,$24,$25
$21,$22,$23,$24,$25,
$26,$27,$28,$29,$30,$31,$32,$33,$34
)
ON CONFLICT (id) DO NOTHING
""",
@@ -53,6 +57,10 @@ class Database:
# Phase 6 feature log-odds
trade.feat_fg_lo, trade.feat_mom_lo, trade.feat_news_lo,
trade.feat_mfld_lo, trade.feat_btc_dom_lo,
# Manifold audit fields
trade.mfld_market_id, trade.mfld_market_title, trade.mfld_market_url,
trade.mfld_prob_raw, trade.mfld_prob_final, trade.mfld_inverted,
trade.mfld_match_score, trade.mfld_match_reason, trade.mfld_match_status,
)
async def save_daily_metrics(self, metrics: dict) -> None:
@@ -493,6 +501,79 @@ class Database:
return result
async def save_manifold_audit(
self,
audit_id: str,
poly_market_id: str,
poly_question: str,
search_query: str,
mfld_market_id: Optional[str],
mfld_market_title: Optional[str],
mfld_market_url: Optional[str],
prob_raw: Optional[float],
prob_final: Optional[float],
inverted: bool,
match_score: Optional[float],
match_reason: Optional[str],
match_status: str,
) -> None:
async with self._pool.acquire() as conn:
await conn.execute("""
INSERT INTO manifold_match_audit (
id, poly_market_id, poly_question, search_query,
mfld_market_id, mfld_market_title, mfld_market_url,
prob_raw, prob_final, inverted,
match_score, match_reason, match_status, used_in_trade
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,FALSE)
""",
audit_id, poly_market_id, poly_question, search_query,
mfld_market_id, mfld_market_title, mfld_market_url,
prob_raw, prob_final, inverted,
match_score, match_reason, match_status,
)
async def mark_manifold_audit_used(self, audit_id: str) -> None:
async with self._pool.acquire() as conn:
await conn.execute(
"UPDATE manifold_match_audit SET used_in_trade = TRUE WHERE id = $1",
audit_id,
)
async def get_manifold_matches(self, limit: int = 50) -> dict:
async with self._pool.acquire() as conn:
summary = await conn.fetchrow("""
SELECT
COUNT(*) FILTER (WHERE match_status = 'accepted') AS total_accepted,
COUNT(*) FILTER (WHERE match_status = 'rejected') AS total_rejected,
COUNT(*) FILTER (WHERE match_status = 'no_results') AS total_no_results,
AVG(match_score) FILTER (WHERE match_status = 'accepted') AS avg_match_score
FROM manifold_match_audit
""")
mfld_dominated = await conn.fetchrow("""
SELECT COUNT(*) AS cnt FROM trades
WHERE feat_mfld_lo IS NOT NULL
AND ABS(feat_mfld_lo) > 0.0001
AND ABS(feat_mfld_lo) > ABS(COALESCE(feat_fg_lo, 0))
AND ABS(feat_mfld_lo) > ABS(COALESCE(feat_mom_lo, 0))
AND ABS(feat_mfld_lo) > ABS(COALESCE(feat_news_lo, 0))
AND ABS(feat_mfld_lo) > ABS(COALESCE(feat_btc_dom_lo, 0))
""")
rows = await conn.fetch(
"SELECT * FROM manifold_match_audit ORDER BY timestamp DESC LIMIT $1",
limit,
)
return {
"summary": {
"total_accepted": int(summary["total_accepted"] or 0),
"total_rejected": int(summary["total_rejected"] or 0),
"total_no_results": int(summary["total_no_results"] or 0),
"avg_match_score": _f(summary["avg_match_score"]),
"trades_dominated_by_mfld": int(mfld_dominated["cnt"] or 0),
},
"recent_matches": [dict(r) for r in rows],
}
def _f(v) -> Optional[float]:
"""None-safe float cast for asyncpg Decimal/None values."""
return float(v) if v is not None else None
bot/data/manifold.py
+144 -77
View File
@@ -2,24 +2,24 @@
Manifold Markets client — cross-platform prediction market probability signals.
For each Polymarket question, searches Manifold for a matching binary market
by keyword overlap and returns its probability as a calibration signal.
by keyword overlap and returns a ManifoldMatchResult with full audit metadata.
Inversion guard: if the Manifold market's winning side (Republican / Democrat)
is the complement of the Polymarket question's winning side, the probability is
automatically inverted (1 - prob). This prevents "Democrats win Ohio governor"
from consuming the probability of a Manifold market titled "Republicans win Ohio
governor" without adjustment.
Match threshold: >= 0.40 Jaccard overlap (raised from 0.25 for stricter semantics).
Rejection guard: if the match score falls below _MATCH_THRESHOLD the market is
rejected, even if inversion would otherwise apply. All decisions are logged at
INFO so they can be audited per-cycle.
Inversion guard (conservative):
- If Polymarket question names a party (democrat/republican) AND the matched
Manifold market names the OPPOSITE party → invert probability (1 - prob).
- If Polymarket question names a party AND Manifold market has NO party keyword
→ reject with reason='ambiguous_inversion' (can't determine if inversion applies).
- All other cases: no inversion, accept if score >= threshold.
- Ante duda, reject.
Cache TTL: 30 minutes (Manifold markets move slowly vs our 60 s cycle).
Match threshold: >= 0.25 keyword overlap ratio between significant tokens.
Cache TTL: 30 minutes.
"""
import logging
import re
import time
from dataclasses import dataclass, field
from typing import Optional
import httpx
@@ -29,7 +29,7 @@ CACHE_TTL_SEC = 1800 # 30 minutes
log = logging.getLogger(__name__)
_MATCH_THRESHOLD = 0.25
_MATCH_THRESHOLD = 0.40 # raised from 0.25
_STOP_WORDS = frozenset([
"will", "the", "a", "an", "is", "are", "was", "were", "be", "been",
@@ -43,9 +43,22 @@ _STOP_WORDS = frozenset([
"before", "during", "until", "against", "between", "through",
])
# Mutually exclusive political parties used for complement detection
_REPUBLICAN_WORDS = frozenset(["republican", "republicans", "gop"])
_DEMOCRAT_WORDS = frozenset(["democrat", "democrats", "democratic"])
_DEMOCRAT_WORDS = frozenset(["democrat", "democrats", "democratic"])
@dataclass
class ManifoldMatchResult:
status: str # 'accepted' | 'rejected' | 'no_results'
prob_final: Optional[float] = None
prob_raw: Optional[float] = None
market_id: Optional[str] = None # Manifold internal market ID
market_title: Optional[str] = None
market_url: Optional[str] = None
match_score: Optional[float] = None # 0-1 Jaccard
match_reason: Optional[str] = None # human-readable explanation
inverted: bool = False
search_query: str = ""
def _significant_words(text: str) -> set[str]:
@@ -69,27 +82,14 @@ def _detect_party(text: str) -> Optional[str]:
return None
def _best_match_with_audit(
poly_question: str,
results: list[dict],
) -> tuple[Optional[dict], float, bool]:
"""
Find the best-matching open binary Manifold market.
Returns (match, score, needs_inversion):
match — best result dict, or None if below threshold
score — keyword overlap score of best candidate (even if rejected)
needs_inversion — True when Manifold market favours the OPPOSITE party/side
to the Polymarket question (probability should be 1 - prob)
"""
def _find_best_candidate(poly_question: str, results: list[dict]) -> tuple[Optional[dict], float]:
"""Find the highest-scoring open binary Manifold market by Jaccard overlap."""
poly_words = _significant_words(poly_question)
poly_party = _detect_party(poly_question)
if not poly_words:
return None, 0.0, False
return None, 0.0
best_score = 0.0
best: Optional[dict] = None
best_needs_inv = False
for result in results:
if result.get("outcomeType") != "BINARY":
@@ -106,18 +106,14 @@ def _best_match_with_audit(
if score > best_score:
best_score = score
best = result
manifold_party = _detect_party(title)
# Inversion is warranted only when both sides are unambiguously detected
# and they are confirmed opposites (republican ≠ democrat).
best_needs_inv = (
poly_party is not None
and manifold_party is not None
and poly_party != manifold_party
)
if best_score >= _MATCH_THRESHOLD and best is not None:
return best, best_score, best_needs_inv
return None, best_score, False
return best, best_score
def _market_url(match: dict) -> Optional[str]:
slug = match.get("slug", "")
creator = match.get("creatorUsername", "")
return f"https://manifold.markets/{creator}/{slug}" if slug else None
class ManifoldClient:
@@ -125,17 +121,16 @@ class ManifoldClient:
def __init__(self) -> None:
self._client = httpx.AsyncClient(timeout=15)
# question → (fetched_at_monotonic, probability_or_None)
self._cache: dict[str, tuple[float, Optional[float]]] = {}
# question → (fetched_at_monotonic, ManifoldMatchResult)
self._cache: dict[str, tuple[float, ManifoldMatchResult]] = {}
async def get_probability(self, question: str) -> Optional[float]:
async def get_match(self, question: str) -> ManifoldMatchResult:
"""
Return Manifold probability for a matching market, or None.
Return a ManifoldMatchResult for the given Polymarket question.
Probability is already adjusted for party-direction inversion when
the matched Manifold market is the complement of our question.
Full audit log is emitted at INFO for every resolved query.
status='accepted' → prob_final is set and ready to use as signal
status='rejected' → match found but failed quality/inversion check
status='no_results' → API returned no results or call failed
"""
now = time.monotonic()
cached = self._cache.get(question)
@@ -144,8 +139,9 @@ class ManifoldClient:
query = _build_search_query(question)
if not query:
self._cache[question] = (now, None)
return None
result = ManifoldMatchResult(status="no_results", search_query="")
self._cache[question] = (now, result)
return result
try:
resp = await self._client.get(
@@ -154,45 +150,116 @@ class ManifoldClient:
)
resp.raise_for_status()
results = resp.json()
except Exception as e:
log.warning("Manifold API error for %r: %s", question[:40], e)
self._cache[question] = (now, None)
return None
except Exception as exc:
log.warning("Manifold API error for %r: %s", question[:40], exc)
result = ManifoldMatchResult(status="no_results", search_query=query)
self._cache[question] = (now, result)
return result
match, score, needs_inv = _best_match_with_audit(question, results)
if not results:
result = ManifoldMatchResult(status="no_results", search_query=query)
self._cache[question] = (now, result)
return result
if match is None:
best, score = _find_best_candidate(question, results)
# ── Score threshold ───────────────────────────────────────────────────
if best is None or score < _MATCH_THRESHOLD:
reason = f"jaccard={score:.2f}<{_MATCH_THRESHOLD:.2f}"
log.info(
"Manifold no_match: %-50s | best_score=%.2f < %.2f | query=%r",
"Manifold REJECTED %-50s | score=%.2f < threshold=%.2f | query=%r",
question[:50], score, _MATCH_THRESHOLD, query,
)
self._cache[question] = (now, None)
return None
result = ManifoldMatchResult(
status="rejected",
market_title=best.get("question") if best else None,
match_score=score if best else None,
match_reason=reason,
search_query=query,
)
self._cache[question] = (now, result)
return result
prob_raw = float(match["probability"])
prob_final = (1.0 - prob_raw) if needs_inv else prob_raw
# ── Inversion analysis (conservative) ────────────────────────────────
poly_party = _detect_party(question)
manifold_party = _detect_party(best.get("question", ""))
# Build market URL from slug (best-effort; may be missing)
slug = match.get("slug", "")
creator = match.get("creatorUsername", "")
url = f"https://manifold.markets/{creator}/{slug}" if slug else "n/a"
poly_words = _significant_words(question)
mfld_words = _significant_words(best.get("question", ""))
matched_tokens = sorted(poly_words & mfld_words)[:6]
inverted = False
rejection_reason: Optional[str] = None
if poly_party is not None:
if manifold_party is None:
# Poly specifies a party; Manifold does not → can't verify inversion safety
rejection_reason = (
f"ambiguous_inversion: poly_party={poly_party}, mfld_party=none"
)
elif manifold_party != poly_party:
# Clear opposite parties — apply inversion
inverted = True
# manifold_party == poly_party → same party, no inversion needed
if rejection_reason is not None:
url = _market_url(best)
log.info(
"Manifold REJECTED %-50s | score=%.2f | reason=%s\n"
" mfld_title: %s",
question[:50], score, rejection_reason, best.get("question", "")[:70],
)
result = ManifoldMatchResult(
status="rejected",
market_id=str(best.get("id", "")) or None,
market_title=best.get("question"),
market_url=url,
match_score=score,
match_reason=(
f"jaccard={score:.2f}, tokens={matched_tokens}, {rejection_reason}"
),
search_query=query,
)
self._cache[question] = (now, result)
return result
# ── Accepted ──────────────────────────────────────────────────────────
prob_raw = float(best["probability"])
prob_final = (1.0 - prob_raw) if inverted else prob_raw
url = _market_url(best)
match_reason = f"jaccard={score:.2f}, tokens={matched_tokens}"
if inverted:
match_reason += f", inverted=party({poly_party}≠{manifold_party})"
log.info(
"Manifold %s: %-50s\n"
" poly_question: %s\n"
" manifold_title: %s\n"
" manifold_url: %s\n"
" match_score: %.2f | prob_raw=%.3f | inverted=%s | prob_final=%.3f",
"MATCH_INVERTED" if needs_inv else "MATCH",
"Manifold %s %-50s\n"
" poly: %s\n"
" mfld: %s\n"
" url: %s\n"
" score=%.2f | raw=%.3f | inverted=%s | final=%.3f",
"ACCEPTED_INVERTED" if inverted else "ACCEPTED ",
question[:50],
question,
match.get("question", ""),
url,
score, prob_raw, needs_inv, prob_final,
best.get("question", ""),
url or "n/a",
score, prob_raw, inverted, prob_final,
)
self._cache[question] = (now, prob_final)
return prob_final
result = ManifoldMatchResult(
status="accepted",
prob_final=prob_final,
prob_raw=prob_raw,
market_id=str(best.get("id", "")) or None,
market_title=best.get("question"),
market_url=url,
match_score=score,
match_reason=match_reason,
inverted=inverted,
search_query=query,
)
self._cache[question] = (now, result)
return result
async def close(self) -> None:
await self._client.aclose()
bot/data/schema.sql
+46
View File
@@ -168,6 +168,52 @@ ALTER TABLE trades ADD COLUMN IF NOT EXISTS feat_btc_dom_lo DOUBLE PRECISION;
CREATE INDEX IF NOT EXISTS idx_trades_feat_fg ON trades(feat_fg_lo) WHERE feat_fg_lo IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_trades_feat_mfld ON trades(feat_mfld_lo) WHERE feat_mfld_lo IS NOT NULL;
-- ─────────────────────────────────────────────────────────────────────────────
-- Manifold match audit — per-trade columns in trades
--
-- Persisted for every trade where Manifold was queried (status='accepted').
-- mfld_match_status: 'accepted' | 'rejected' | 'no_results'
-- mfld_inverted: TRUE when prob_final = 1 - prob_raw (party complement match)
-- ─────────────────────────────────────────────────────────────────────────────
ALTER TABLE trades ADD COLUMN IF NOT EXISTS mfld_market_id TEXT;
ALTER TABLE trades ADD COLUMN IF NOT EXISTS mfld_market_title TEXT;
ALTER TABLE trades ADD COLUMN IF NOT EXISTS mfld_market_url TEXT;
ALTER TABLE trades ADD COLUMN IF NOT EXISTS mfld_prob_raw DOUBLE PRECISION;
ALTER TABLE trades ADD COLUMN IF NOT EXISTS mfld_prob_final DOUBLE PRECISION;
ALTER TABLE trades ADD COLUMN IF NOT EXISTS mfld_inverted BOOLEAN;
ALTER TABLE trades ADD COLUMN IF NOT EXISTS mfld_match_score DOUBLE PRECISION;
ALTER TABLE trades ADD COLUMN IF NOT EXISTS mfld_match_reason TEXT;
ALTER TABLE trades ADD COLUMN IF NOT EXISTS mfld_match_status TEXT;
-- ─────────────────────────────────────────────────────────────────────────────
-- Manifold match audit table — records every Manifold query attempt
--
-- Populated for ALL queries: accepted, rejected, and no_results.
-- used_in_trade=TRUE is set after executor confirms a trade was executed.
-- poly_market_id: Market.id from the Polymarket Market dataclass (never NULL).
-- ─────────────────────────────────────────────────────────────────────────────
CREATE TABLE IF NOT EXISTS manifold_match_audit (
id TEXT PRIMARY KEY,
timestamp TIMESTAMPTZ DEFAULT NOW(),
poly_market_id TEXT NOT NULL,
poly_question TEXT NOT NULL,
search_query TEXT,
mfld_market_id TEXT,
mfld_market_title TEXT,
mfld_market_url TEXT,
prob_raw DOUBLE PRECISION,
prob_final DOUBLE PRECISION,
inverted BOOLEAN DEFAULT FALSE,
match_score DOUBLE PRECISION,
match_reason TEXT,
match_status TEXT NOT NULL,
used_in_trade BOOLEAN DEFAULT FALSE
);
CREATE INDEX IF NOT EXISTS idx_mfld_audit_timestamp ON manifold_match_audit(timestamp DESC);
CREATE INDEX IF NOT EXISTS idx_mfld_audit_status ON manifold_match_audit(match_status);
CREATE INDEX IF NOT EXISTS idx_mfld_audit_poly_mkt ON manifold_match_audit(poly_market_id);
-- ─────────────────────────────────────────────────────────────────────────────
-- Fix 3: extended metrics_daily columns for DB-computed metrics
--