feat(api): add /api/metrics/manifold-coverage by-category endpoint
CI/CD / build-and-push (push) Successful in 8s
CI/CD / build-and-push (push) Successful in 8s
Measure real Manifold coverage per semantic market category counted by UNIQUE market (not audit rows, which are inflated by repeated re-evaluation). Base table is manifold_match_audit filtered to v3_outcome_guard; each poly_market_id is collapsed to one row, LEFT JOIN trades for family_key, category inferred from family_key when present else from poly_question (gubernatorial / mayoral / senate / primary-republican / primary-democrat / big-tech / geopolitics / other). Per category: unique_evaluated / unique_accepted / unique_rejected / unique_no_results / coverage_rate, ordered by unique_evaluated DESC. Summary adds total_unique_evaluated, total_unique_accepted, overall_coverage_rate, categories_with_coverage. Read-only: new db method + endpoint only; matcher, scheduler, cooldowns, thresholds and exposure untouched. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
chemavx
co-authored by
Claude Opus 4.8
Claude Opus 4.8
parent
98abd96fd2
commit
823914789d
+107
@@ -688,6 +688,113 @@ class Database:
|
||||
"recent_matches": [dict(r) for r in rows],
|
||||
}
|
||||
|
||||
async def get_manifold_coverage_by_category(self) -> dict:
|
||||
"""Manifold coverage by semantic market category, counted by UNIQUE market.
|
||||
|
||||
Base table is manifold_match_audit filtered to the current matcher
|
||||
(v3_outcome_guard). Each poly_market_id is collapsed to one row first, so
|
||||
a market is counted once regardless of how many audit attempts or trades it
|
||||
has — this measures coverage, not retry volume.
|
||||
|
||||
Category is inferred from the market's trade family_key when available, else
|
||||
from poly_question (LEFT JOIN: audited markets that never produced a trade
|
||||
are kept). Buckets (accepted/rejected/no_results) are not mutually exclusive
|
||||
at the market level — a market that was no_results then later rejected counts
|
||||
in both — matching COUNT(DISTINCT CASE WHEN status=...) semantics.
|
||||
"""
|
||||
async with self._pool.acquire() as conn:
|
||||
rows = await conn.fetch("""
|
||||
WITH audit AS (
|
||||
SELECT
|
||||
poly_market_id,
|
||||
MAX(poly_question) AS poly_question,
|
||||
bool_or(match_status = 'accepted') AS has_accepted,
|
||||
bool_or(match_status = 'rejected') AS has_rejected,
|
||||
bool_or(match_status = 'no_results') AS has_no_results
|
||||
FROM manifold_match_audit
|
||||
WHERE matcher_version = 'v3_outcome_guard'
|
||||
GROUP BY poly_market_id
|
||||
),
|
||||
fam AS (
|
||||
SELECT market_id, MAX(family_key) AS family_key
|
||||
FROM trades
|
||||
GROUP BY market_id
|
||||
),
|
||||
categorized AS (
|
||||
SELECT
|
||||
a.has_accepted, a.has_rejected, a.has_no_results,
|
||||
CASE
|
||||
WHEN f.family_key ILIKE '%gubernatorial%' THEN 'gubernatorial'
|
||||
WHEN f.family_key ILIKE '%mayoral%' THEN 'mayoral'
|
||||
WHEN f.family_key ILIKE '%senate%' THEN 'senate'
|
||||
WHEN f.family_key ILIKE '%republican%' THEN 'primary-republican'
|
||||
WHEN f.family_key ILIKE '%democrat%' THEN 'primary-democrat'
|
||||
WHEN f.family_key ILIKE '%openai%'
|
||||
OR f.family_key ILIKE '%nvidia%'
|
||||
OR f.family_key ILIKE '%anthropic%' THEN 'big-tech'
|
||||
-- family_key NULL or unmatched → infer from question
|
||||
WHEN a.poly_question ILIKE '%governor%'
|
||||
OR a.poly_question ILIKE '%gubernatorial%' THEN 'gubernatorial'
|
||||
WHEN a.poly_question ILIKE '%mayor%'
|
||||
OR a.poly_question ILIKE '%mayoral%' THEN 'mayoral'
|
||||
WHEN a.poly_question ILIKE '%senate%' THEN 'senate'
|
||||
WHEN a.poly_question ILIKE '%republican primary%' THEN 'primary-republican'
|
||||
WHEN a.poly_question ILIKE '%democratic primary%'
|
||||
OR a.poly_question ILIKE '%democrat primary%' THEN 'primary-democrat'
|
||||
WHEN a.poly_question ILIKE '%openai%'
|
||||
OR a.poly_question ILIKE '%nvidia%'
|
||||
OR a.poly_question ILIKE '%anthropic%' THEN 'big-tech'
|
||||
WHEN a.poly_question ILIKE '%russia%'
|
||||
OR a.poly_question ILIKE '%ukraine%'
|
||||
OR a.poly_question ILIKE '%israel%'
|
||||
OR a.poly_question ILIKE '%ceasefire%'
|
||||
OR a.poly_question ILIKE '%military%' THEN 'geopolitics'
|
||||
ELSE 'other'
|
||||
END AS category
|
||||
FROM audit a
|
||||
LEFT JOIN fam f ON f.market_id = a.poly_market_id
|
||||
)
|
||||
SELECT
|
||||
category,
|
||||
COUNT(*) AS unique_evaluated,
|
||||
COUNT(*) FILTER (WHERE has_accepted) AS unique_accepted,
|
||||
COUNT(*) FILTER (WHERE has_rejected) AS unique_rejected,
|
||||
COUNT(*) FILTER (WHERE has_no_results) AS unique_no_results
|
||||
FROM categorized
|
||||
GROUP BY category
|
||||
ORDER BY unique_evaluated DESC
|
||||
""")
|
||||
|
||||
coverage_by_category = []
|
||||
total_evaluated = 0
|
||||
total_accepted = 0
|
||||
categories_with_coverage = 0
|
||||
for r in rows:
|
||||
evaluated = int(r["unique_evaluated"] or 0)
|
||||
accepted = int(r["unique_accepted"] or 0)
|
||||
total_evaluated += evaluated
|
||||
total_accepted += accepted
|
||||
if accepted > 0:
|
||||
categories_with_coverage += 1
|
||||
coverage_by_category.append({
|
||||
"category": r["category"],
|
||||
"unique_evaluated": evaluated,
|
||||
"unique_accepted": accepted,
|
||||
"unique_rejected": int(r["unique_rejected"] or 0),
|
||||
"unique_no_results": int(r["unique_no_results"] or 0),
|
||||
"coverage_rate": (accepted / evaluated) if evaluated else None,
|
||||
})
|
||||
|
||||
return {
|
||||
"coverage_by_category": coverage_by_category,
|
||||
"summary": {
|
||||
"total_unique_evaluated": total_evaluated,
|
||||
"total_unique_accepted": total_accepted,
|
||||
"overall_coverage_rate": (total_accepted / total_evaluated) if total_evaluated else None,
|
||||
"categories_with_coverage": categories_with_coverage,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _f(v) -> Optional[float]:
|
||||
"""None-safe float cast for asyncpg Decimal/None values."""
|
||||
|
||||
Reference in New Issue
Block a user