feat(api): add /api/metrics/manifold-coverage by-category endpoint
CI/CD / build-and-push (push) Successful in 8s
CI/CD / build-and-push (push) Successful in 8s
Measure real Manifold coverage per semantic market category counted by UNIQUE market (not audit rows, which are inflated by repeated re-evaluation). Base table is manifold_match_audit filtered to v3_outcome_guard; each poly_market_id is collapsed to one row, LEFT JOIN trades for family_key, category inferred from family_key when present else from poly_question (gubernatorial / mayoral / senate / primary-republican / primary-democrat / big-tech / geopolitics / other). Per category: unique_evaluated / unique_accepted / unique_rejected / unique_no_results / coverage_rate, ordered by unique_evaluated DESC. Summary adds total_unique_evaluated, total_unique_accepted, overall_coverage_rate, categories_with_coverage. Read-only: new db method + endpoint only; matcher, scheduler, cooldowns, thresholds and exposure untouched. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
98abd96fd2
commit
823914789d
+26
@@ -243,6 +243,32 @@ async def get_manifold_matches():
|
|||||||
return data
|
return data
|
||||||
|
|
||||||
|
|
||||||
|
@app.get("/api/metrics/manifold-coverage")
|
||||||
|
async def get_manifold_coverage():
|
||||||
|
"""Manifold coverage by semantic market category, counted by UNIQUE market.
|
||||||
|
|
||||||
|
Unlike the raw audit counters (which count per-attempt and are inflated by the
|
||||||
|
trading loop re-evaluating the same markets), this measures real coverage:
|
||||||
|
how many DISTINCT markets in each category the matcher found an accepted
|
||||||
|
Manifold counterpart for. Base table is manifold_match_audit filtered to the
|
||||||
|
current matcher (v3_outcome_guard); category is inferred from trade family_key
|
||||||
|
when available, otherwise from the question text.
|
||||||
|
|
||||||
|
coverage_by_category — one entry per category, ordered by unique_evaluated DESC:
|
||||||
|
category — gubernatorial | mayoral | senate | primary-republican |
|
||||||
|
primary-democrat | big-tech | geopolitics | other
|
||||||
|
unique_evaluated — distinct markets audited in this category
|
||||||
|
unique_accepted — distinct markets with at least one accepted match
|
||||||
|
unique_rejected — distinct markets with at least one rejected match
|
||||||
|
unique_no_results — distinct markets with at least one no_results outcome
|
||||||
|
coverage_rate — unique_accepted / unique_evaluated (null if evaluated=0)
|
||||||
|
summary — total_unique_evaluated, total_unique_accepted, overall_coverage_rate
|
||||||
|
(null if nothing evaluated), categories_with_coverage (categories with
|
||||||
|
unique_accepted > 0).
|
||||||
|
"""
|
||||||
|
return await db.get_manifold_coverage_by_category()
|
||||||
|
|
||||||
|
|
||||||
@app.get("/api/summary")
|
@app.get("/api/summary")
|
||||||
async def get_summary():
|
async def get_summary():
|
||||||
"""Dashboard summary card data.
|
"""Dashboard summary card data.
|
||||||
|
|||||||
+107
@@ -688,6 +688,113 @@ class Database:
|
|||||||
"recent_matches": [dict(r) for r in rows],
|
"recent_matches": [dict(r) for r in rows],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async def get_manifold_coverage_by_category(self) -> dict:
|
||||||
|
"""Manifold coverage by semantic market category, counted by UNIQUE market.
|
||||||
|
|
||||||
|
Base table is manifold_match_audit filtered to the current matcher
|
||||||
|
(v3_outcome_guard). Each poly_market_id is collapsed to one row first, so
|
||||||
|
a market is counted once regardless of how many audit attempts or trades it
|
||||||
|
has — this measures coverage, not retry volume.
|
||||||
|
|
||||||
|
Category is inferred from the market's trade family_key when available, else
|
||||||
|
from poly_question (LEFT JOIN: audited markets that never produced a trade
|
||||||
|
are kept). Buckets (accepted/rejected/no_results) are not mutually exclusive
|
||||||
|
at the market level — a market that was no_results then later rejected counts
|
||||||
|
in both — matching COUNT(DISTINCT CASE WHEN status=...) semantics.
|
||||||
|
"""
|
||||||
|
async with self._pool.acquire() as conn:
|
||||||
|
rows = await conn.fetch("""
|
||||||
|
WITH audit AS (
|
||||||
|
SELECT
|
||||||
|
poly_market_id,
|
||||||
|
MAX(poly_question) AS poly_question,
|
||||||
|
bool_or(match_status = 'accepted') AS has_accepted,
|
||||||
|
bool_or(match_status = 'rejected') AS has_rejected,
|
||||||
|
bool_or(match_status = 'no_results') AS has_no_results
|
||||||
|
FROM manifold_match_audit
|
||||||
|
WHERE matcher_version = 'v3_outcome_guard'
|
||||||
|
GROUP BY poly_market_id
|
||||||
|
),
|
||||||
|
fam AS (
|
||||||
|
SELECT market_id, MAX(family_key) AS family_key
|
||||||
|
FROM trades
|
||||||
|
GROUP BY market_id
|
||||||
|
),
|
||||||
|
categorized AS (
|
||||||
|
SELECT
|
||||||
|
a.has_accepted, a.has_rejected, a.has_no_results,
|
||||||
|
CASE
|
||||||
|
WHEN f.family_key ILIKE '%gubernatorial%' THEN 'gubernatorial'
|
||||||
|
WHEN f.family_key ILIKE '%mayoral%' THEN 'mayoral'
|
||||||
|
WHEN f.family_key ILIKE '%senate%' THEN 'senate'
|
||||||
|
WHEN f.family_key ILIKE '%republican%' THEN 'primary-republican'
|
||||||
|
WHEN f.family_key ILIKE '%democrat%' THEN 'primary-democrat'
|
||||||
|
WHEN f.family_key ILIKE '%openai%'
|
||||||
|
OR f.family_key ILIKE '%nvidia%'
|
||||||
|
OR f.family_key ILIKE '%anthropic%' THEN 'big-tech'
|
||||||
|
-- family_key NULL or unmatched → infer from question
|
||||||
|
WHEN a.poly_question ILIKE '%governor%'
|
||||||
|
OR a.poly_question ILIKE '%gubernatorial%' THEN 'gubernatorial'
|
||||||
|
WHEN a.poly_question ILIKE '%mayor%'
|
||||||
|
OR a.poly_question ILIKE '%mayoral%' THEN 'mayoral'
|
||||||
|
WHEN a.poly_question ILIKE '%senate%' THEN 'senate'
|
||||||
|
WHEN a.poly_question ILIKE '%republican primary%' THEN 'primary-republican'
|
||||||
|
WHEN a.poly_question ILIKE '%democratic primary%'
|
||||||
|
OR a.poly_question ILIKE '%democrat primary%' THEN 'primary-democrat'
|
||||||
|
WHEN a.poly_question ILIKE '%openai%'
|
||||||
|
OR a.poly_question ILIKE '%nvidia%'
|
||||||
|
OR a.poly_question ILIKE '%anthropic%' THEN 'big-tech'
|
||||||
|
WHEN a.poly_question ILIKE '%russia%'
|
||||||
|
OR a.poly_question ILIKE '%ukraine%'
|
||||||
|
OR a.poly_question ILIKE '%israel%'
|
||||||
|
OR a.poly_question ILIKE '%ceasefire%'
|
||||||
|
OR a.poly_question ILIKE '%military%' THEN 'geopolitics'
|
||||||
|
ELSE 'other'
|
||||||
|
END AS category
|
||||||
|
FROM audit a
|
||||||
|
LEFT JOIN fam f ON f.market_id = a.poly_market_id
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
category,
|
||||||
|
COUNT(*) AS unique_evaluated,
|
||||||
|
COUNT(*) FILTER (WHERE has_accepted) AS unique_accepted,
|
||||||
|
COUNT(*) FILTER (WHERE has_rejected) AS unique_rejected,
|
||||||
|
COUNT(*) FILTER (WHERE has_no_results) AS unique_no_results
|
||||||
|
FROM categorized
|
||||||
|
GROUP BY category
|
||||||
|
ORDER BY unique_evaluated DESC
|
||||||
|
""")
|
||||||
|
|
||||||
|
coverage_by_category = []
|
||||||
|
total_evaluated = 0
|
||||||
|
total_accepted = 0
|
||||||
|
categories_with_coverage = 0
|
||||||
|
for r in rows:
|
||||||
|
evaluated = int(r["unique_evaluated"] or 0)
|
||||||
|
accepted = int(r["unique_accepted"] or 0)
|
||||||
|
total_evaluated += evaluated
|
||||||
|
total_accepted += accepted
|
||||||
|
if accepted > 0:
|
||||||
|
categories_with_coverage += 1
|
||||||
|
coverage_by_category.append({
|
||||||
|
"category": r["category"],
|
||||||
|
"unique_evaluated": evaluated,
|
||||||
|
"unique_accepted": accepted,
|
||||||
|
"unique_rejected": int(r["unique_rejected"] or 0),
|
||||||
|
"unique_no_results": int(r["unique_no_results"] or 0),
|
||||||
|
"coverage_rate": (accepted / evaluated) if evaluated else None,
|
||||||
|
})
|
||||||
|
|
||||||
|
return {
|
||||||
|
"coverage_by_category": coverage_by_category,
|
||||||
|
"summary": {
|
||||||
|
"total_unique_evaluated": total_evaluated,
|
||||||
|
"total_unique_accepted": total_accepted,
|
||||||
|
"overall_coverage_rate": (total_accepted / total_evaluated) if total_evaluated else None,
|
||||||
|
"categories_with_coverage": categories_with_coverage,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _f(v) -> Optional[float]:
|
def _f(v) -> Optional[float]:
|
||||||
"""None-safe float cast for asyncpg Decimal/None values."""
|
"""None-safe float cast for asyncpg Decimal/None values."""
|
||||||
|
|||||||
Reference in New Issue
Block a user