feat: check de colisión de tema (topic.collision) en rules + validate
Dos posts sobre el mismo caso se canibalizan en la SERP (2026-07-10: se publicó un segundo Kecksburg con otro ya programado, y había un título casi gemelo del de Malmstrom en la cola). Nueva función pura topic_collision(post, corpus) en seo_rules.py — corpus-aware, fuera de RULES — que dispara por: caso+año compartidos (años < 2020; los de la era de noticias no son identidad de caso), hooks de título ≥70% similares (calibrado: el par nuclear da 0.742, el par legítimo más cercano 0.65), o slugs ≥50% solapados. seo_validate.py obtiene el corpus via ghst-en (--limit all) y lo reporta como grupo topic_collision; --no-collision lo omite para follow-ups intencionados. Calibrado contra los 36 posts reales: dispara solo el duplicado Kecksburg (HIGH) y el par Grusch preview/evento (MED, intencionado). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
beae7c154a
commit
bae1cc949c
+106
@@ -178,6 +178,112 @@ def r_jsonld(p):
|
||||
return [Violation("jsonld.missing", MED, "no BlogPosting JSON-LD", "add JSON-LD")]
|
||||
|
||||
|
||||
# ---- topic collision (corpus-aware; NOT in RULES) ---------------------------
|
||||
# Two posts about the same case cannibalize each other in the SERP (2026-07-10:
|
||||
# a second Kecksburg post was published while another sat scheduled; a "When
|
||||
# Nuclear ... Went/Go Silent" near-twin title was already queued). RULES functions
|
||||
# are (post) -> violations; this one also needs the rest of the site, so callers
|
||||
# (seo_validate.py) pass the corpus explicitly: published + scheduled posts as
|
||||
# dicts with at least {id, title, slug}.
|
||||
|
||||
TOPIC_STOPWORDS = {
|
||||
# english glue
|
||||
"the", "a", "an", "of", "and", "in", "at", "on", "to", "that", "what",
|
||||
"when", "who", "why", "how", "its", "his", "her", "their", "our", "one",
|
||||
"still", "cant", "couldnt", "went", "go", "goes", "most", "from", "with",
|
||||
"they", "them", "these", "this", "are", "were", "was", "is", "be", "been",
|
||||
"has", "have", "had", "but", "for", "all", "than", "then", "ever", "never",
|
||||
# domain-generic (present in half the catalog — carry no case identity)
|
||||
"ufo", "ufos", "uap", "uaps", "incident", "incidents", "case", "cases",
|
||||
"file", "files", "mystery", "declassified", "declassification", "pentagon",
|
||||
"government", "military", "congress", "secret", "program", "investigation",
|
||||
"evidence", "witness", "witnesses", "document", "documents", "documented",
|
||||
"unexplained", "encounter", "sighting", "sightings", "alien", "aliens",
|
||||
"phenomena", "aerial", "unidentified", "extraordinary", "americas",
|
||||
"american", "video", "footage",
|
||||
}
|
||||
# 0.70 calibrated 2026-07-10: the "When Nuclear Weapons Go / Arsenal Went
|
||||
# Silent" near-twin pair scores 0.742 (char-level penalizes weapons/arsenal);
|
||||
# the closest legit-distinct pair in the catalog scores 0.65.
|
||||
TITLE_HOOK_SIM_MIN = 0.70 # SequenceMatcher on the pre-colon hook
|
||||
SLUG_JACCARD_MIN = 0.5 # shared slug-token ratio
|
||||
# Years >= this are "news era", not case identity: every contemporary post
|
||||
# carries the current year (PURSUE 2026, Grusch 2026...) without being the same
|
||||
# story. Case years in the catalog run 1947-2019.
|
||||
NEWS_YEAR_MIN = 2020
|
||||
|
||||
_YEAR_RE = re.compile(r"\b(19|20)\d{2}\b")
|
||||
|
||||
|
||||
def _tokens(text):
|
||||
return set(re.findall(r"[a-z0-9]+", _s(text).lower()))
|
||||
|
||||
|
||||
def _case_years(title, slug):
|
||||
"""Historical case years (pre news-era) found in title+slug."""
|
||||
return {m.group(0) for m in _YEAR_RE.finditer(title + " " + slug)
|
||||
if int(m.group(0)) < NEWS_YEAR_MIN}
|
||||
|
||||
|
||||
def _sig_tokens(title, slug):
|
||||
"""Case-identity tokens: title+slug minus glue/domain words, years and
|
||||
fragments shorter than 3 chars (possessive 's', initials...)."""
|
||||
toks = _tokens(title) | _tokens(slug.replace("-", " "))
|
||||
return {t for t in toks
|
||||
if len(t) >= 3 and t not in TOPIC_STOPWORDS and not _YEAR_RE.fullmatch(t)}
|
||||
|
||||
|
||||
def _hook(title):
|
||||
return _s(title).split(":")[0].strip().lower()
|
||||
|
||||
|
||||
def topic_collision(post, corpus):
|
||||
"""Compare one candidate post against the site corpus → list[Violation].
|
||||
|
||||
Fires when the candidate and an existing post look like the same story:
|
||||
- share a case year AND a case-identity token (Kecksburg+1965), or
|
||||
- their pre-colon title hooks read nearly the same, or
|
||||
- their slugs share most of their tokens.
|
||||
"""
|
||||
from difflib import SequenceMatcher
|
||||
|
||||
out = []
|
||||
c_years = _case_years(_s(post.get("title")), _s(post.get("slug")))
|
||||
c_sig = _sig_tokens(post.get("title"), _s(post.get("slug")))
|
||||
c_hook = _hook(post.get("title"))
|
||||
c_slug_toks = _tokens(_s(post.get("slug")).replace("-", " "))
|
||||
|
||||
for other in corpus:
|
||||
if other.get("id") == post.get("id"):
|
||||
continue
|
||||
o_title, o_slug = _s(other.get("title")), _s(other.get("slug"))
|
||||
o_years = _case_years(o_title, o_slug)
|
||||
o_sig = _sig_tokens(o_title, o_slug)
|
||||
|
||||
reasons = []
|
||||
if (c_years & o_years) and (c_sig & o_sig):
|
||||
shared = ", ".join(sorted(c_sig & o_sig)[:3] + sorted(c_years & o_years))
|
||||
reasons.append((HIGH, f"same case + year ({shared})"))
|
||||
hook_sim = SequenceMatcher(None, c_hook, _hook(o_title)).ratio()
|
||||
if c_hook and hook_sim >= TITLE_HOOK_SIM_MIN:
|
||||
reasons.append((MED, f"title hooks {hook_sim:.0%} similar"))
|
||||
o_slug_toks = _tokens(o_slug.replace("-", " "))
|
||||
union = c_slug_toks | o_slug_toks
|
||||
if union:
|
||||
jac = len(c_slug_toks & o_slug_toks) / len(union)
|
||||
if jac >= SLUG_JACCARD_MIN:
|
||||
reasons.append((MED, f"slugs {jac:.0%} overlapping"))
|
||||
|
||||
if reasons:
|
||||
sev = max(s for s, _ in reasons)
|
||||
why = "; ".join(r for _, r in reasons)
|
||||
out.append(Violation(
|
||||
"topic.collision", sev,
|
||||
f"collides with [{other.get('status', '?')}] \"{o_title[:60]}\" — {why}",
|
||||
"merge, retitle to a distinct angle, or interlink deliberately"))
|
||||
return out
|
||||
|
||||
|
||||
RULES = [
|
||||
r_meta_title,
|
||||
r_meta_description,
|
||||
|
||||
Reference in New Issue
Block a user