diff --git a/api/main.py b/api/main.py index 705182d..52b2b93 100644 --- a/api/main.py +++ b/api/main.py @@ -243,6 +243,32 @@ async def get_manifold_matches(): return data +@app.get("/api/metrics/manifold-coverage") +async def get_manifold_coverage(): + """Manifold coverage by semantic market category, counted by UNIQUE market. + + Unlike the raw audit counters (which count per-attempt and are inflated by the + trading loop re-evaluating the same markets), this measures real coverage: + how many DISTINCT markets in each category the matcher found an accepted + Manifold counterpart for. Base table is manifold_match_audit filtered to the + current matcher (v3_outcome_guard); category is inferred from trade family_key + when available, otherwise from the question text. + + coverage_by_category — one entry per category, ordered by unique_evaluated DESC: + category — gubernatorial | mayoral | senate | primary-republican | + primary-democrat | big-tech | geopolitics | other + unique_evaluated — distinct markets audited in this category + unique_accepted — distinct markets with at least one accepted match + unique_rejected — distinct markets with at least one rejected match + unique_no_results — distinct markets with at least one no_results outcome + coverage_rate — unique_accepted / unique_evaluated (null if evaluated=0) + summary — total_unique_evaluated, total_unique_accepted, overall_coverage_rate + (null if nothing evaluated), categories_with_coverage (categories with + unique_accepted > 0). + """ + return await db.get_manifold_coverage_by_category() + + @app.get("/api/summary") async def get_summary(): """Dashboard summary card data. diff --git a/bot/data/db.py b/bot/data/db.py index c3d430e..fb72cb9 100644 --- a/bot/data/db.py +++ b/bot/data/db.py @@ -688,6 +688,113 @@ class Database: "recent_matches": [dict(r) for r in rows], } + async def get_manifold_coverage_by_category(self) -> dict: + """Manifold coverage by semantic market category, counted by UNIQUE market. + + Base table is manifold_match_audit filtered to the current matcher + (v3_outcome_guard). Each poly_market_id is collapsed to one row first, so + a market is counted once regardless of how many audit attempts or trades it + has — this measures coverage, not retry volume. + + Category is inferred from the market's trade family_key when available, else + from poly_question (LEFT JOIN: audited markets that never produced a trade + are kept). Buckets (accepted/rejected/no_results) are not mutually exclusive + at the market level — a market that was no_results then later rejected counts + in both — matching COUNT(DISTINCT CASE WHEN status=...) semantics. + """ + async with self._pool.acquire() as conn: + rows = await conn.fetch(""" + WITH audit AS ( + SELECT + poly_market_id, + MAX(poly_question) AS poly_question, + bool_or(match_status = 'accepted') AS has_accepted, + bool_or(match_status = 'rejected') AS has_rejected, + bool_or(match_status = 'no_results') AS has_no_results + FROM manifold_match_audit + WHERE matcher_version = 'v3_outcome_guard' + GROUP BY poly_market_id + ), + fam AS ( + SELECT market_id, MAX(family_key) AS family_key + FROM trades + GROUP BY market_id + ), + categorized AS ( + SELECT + a.has_accepted, a.has_rejected, a.has_no_results, + CASE + WHEN f.family_key ILIKE '%gubernatorial%' THEN 'gubernatorial' + WHEN f.family_key ILIKE '%mayoral%' THEN 'mayoral' + WHEN f.family_key ILIKE '%senate%' THEN 'senate' + WHEN f.family_key ILIKE '%republican%' THEN 'primary-republican' + WHEN f.family_key ILIKE '%democrat%' THEN 'primary-democrat' + WHEN f.family_key ILIKE '%openai%' + OR f.family_key ILIKE '%nvidia%' + OR f.family_key ILIKE '%anthropic%' THEN 'big-tech' + -- family_key NULL or unmatched → infer from question + WHEN a.poly_question ILIKE '%governor%' + OR a.poly_question ILIKE '%gubernatorial%' THEN 'gubernatorial' + WHEN a.poly_question ILIKE '%mayor%' + OR a.poly_question ILIKE '%mayoral%' THEN 'mayoral' + WHEN a.poly_question ILIKE '%senate%' THEN 'senate' + WHEN a.poly_question ILIKE '%republican primary%' THEN 'primary-republican' + WHEN a.poly_question ILIKE '%democratic primary%' + OR a.poly_question ILIKE '%democrat primary%' THEN 'primary-democrat' + WHEN a.poly_question ILIKE '%openai%' + OR a.poly_question ILIKE '%nvidia%' + OR a.poly_question ILIKE '%anthropic%' THEN 'big-tech' + WHEN a.poly_question ILIKE '%russia%' + OR a.poly_question ILIKE '%ukraine%' + OR a.poly_question ILIKE '%israel%' + OR a.poly_question ILIKE '%ceasefire%' + OR a.poly_question ILIKE '%military%' THEN 'geopolitics' + ELSE 'other' + END AS category + FROM audit a + LEFT JOIN fam f ON f.market_id = a.poly_market_id + ) + SELECT + category, + COUNT(*) AS unique_evaluated, + COUNT(*) FILTER (WHERE has_accepted) AS unique_accepted, + COUNT(*) FILTER (WHERE has_rejected) AS unique_rejected, + COUNT(*) FILTER (WHERE has_no_results) AS unique_no_results + FROM categorized + GROUP BY category + ORDER BY unique_evaluated DESC + """) + + coverage_by_category = [] + total_evaluated = 0 + total_accepted = 0 + categories_with_coverage = 0 + for r in rows: + evaluated = int(r["unique_evaluated"] or 0) + accepted = int(r["unique_accepted"] or 0) + total_evaluated += evaluated + total_accepted += accepted + if accepted > 0: + categories_with_coverage += 1 + coverage_by_category.append({ + "category": r["category"], + "unique_evaluated": evaluated, + "unique_accepted": accepted, + "unique_rejected": int(r["unique_rejected"] or 0), + "unique_no_results": int(r["unique_no_results"] or 0), + "coverage_rate": (accepted / evaluated) if evaluated else None, + }) + + return { + "coverage_by_category": coverage_by_category, + "summary": { + "total_unique_evaluated": total_evaluated, + "total_unique_accepted": total_accepted, + "overall_coverage_rate": (total_accepted / total_evaluated) if total_evaluated else None, + "categories_with_coverage": categories_with_coverage, + }, + } + def _f(v) -> Optional[float]: """None-safe float cast for asyncpg Decimal/None values."""