diff --git a/src/seo/.rules.sha256 b/src/seo/.rules.sha256 index dac6e23..287383b 100644 --- a/src/seo/.rules.sha256 +++ b/src/seo/.rules.sha256 @@ -1 +1 @@ -b8faa93b1f7727d3870e18f69b68283d9a97ed9da819ef0cfa79a60cc2c4ab70 +73872aaf4079e5a0b690fbbe7de7e7967edc2297c8bad79142308714452c60ef diff --git a/src/seo/rules.py b/src/seo/rules.py index aaf9ee4..9e5e850 100644 --- a/src/seo/rules.py +++ b/src/seo/rules.py @@ -195,6 +195,112 @@ def r_jsonld(p): return [Violation("jsonld.missing", MED, "no BlogPosting JSON-LD", "add JSON-LD")] +# ---- topic collision (corpus-aware; NOT in RULES) --------------------------- +# Two posts about the same case cannibalize each other in the SERP (2026-07-10: +# a second Kecksburg post was published while another sat scheduled; a "When +# Nuclear ... Went/Go Silent" near-twin title was already queued). RULES functions +# are (post) -> violations; this one also needs the rest of the site, so callers +# (seo_validate.py) pass the corpus explicitly: published + scheduled posts as +# dicts with at least {id, title, slug}. + +TOPIC_STOPWORDS = { + # english glue + "the", "a", "an", "of", "and", "in", "at", "on", "to", "that", "what", + "when", "who", "why", "how", "its", "his", "her", "their", "our", "one", + "still", "cant", "couldnt", "went", "go", "goes", "most", "from", "with", + "they", "them", "these", "this", "are", "were", "was", "is", "be", "been", + "has", "have", "had", "but", "for", "all", "than", "then", "ever", "never", + # domain-generic (present in half the catalog — carry no case identity) + "ufo", "ufos", "uap", "uaps", "incident", "incidents", "case", "cases", + "file", "files", "mystery", "declassified", "declassification", "pentagon", + "government", "military", "congress", "secret", "program", "investigation", + "evidence", "witness", "witnesses", "document", "documents", "documented", + "unexplained", "encounter", "sighting", "sightings", "alien", "aliens", + "phenomena", "aerial", "unidentified", "extraordinary", "americas", + "american", "video", "footage", +} +# 0.70 calibrated 2026-07-10: the "When Nuclear Weapons Go / Arsenal Went +# Silent" near-twin pair scores 0.742 (char-level penalizes weapons/arsenal); +# the closest legit-distinct pair in the catalog scores 0.65. +TITLE_HOOK_SIM_MIN = 0.70 # SequenceMatcher on the pre-colon hook +SLUG_JACCARD_MIN = 0.5 # shared slug-token ratio +# Years >= this are "news era", not case identity: every contemporary post +# carries the current year (PURSUE 2026, Grusch 2026...) without being the same +# story. Case years in the catalog run 1947-2019. +NEWS_YEAR_MIN = 2020 + +_YEAR_RE = re.compile(r"\b(19|20)\d{2}\b") + + +def _tokens(text): + return set(re.findall(r"[a-z0-9]+", _s(text).lower())) + + +def _case_years(title, slug): + """Historical case years (pre news-era) found in title+slug.""" + return {m.group(0) for m in _YEAR_RE.finditer(title + " " + slug) + if int(m.group(0)) < NEWS_YEAR_MIN} + + +def _sig_tokens(title, slug): + """Case-identity tokens: title+slug minus glue/domain words, years and + fragments shorter than 3 chars (possessive 's', initials...).""" + toks = _tokens(title) | _tokens(slug.replace("-", " ")) + return {t for t in toks + if len(t) >= 3 and t not in TOPIC_STOPWORDS and not _YEAR_RE.fullmatch(t)} + + +def _hook(title): + return _s(title).split(":")[0].strip().lower() + + +def topic_collision(post, corpus): + """Compare one candidate post against the site corpus → list[Violation]. + + Fires when the candidate and an existing post look like the same story: + - share a case year AND a case-identity token (Kecksburg+1965), or + - their pre-colon title hooks read nearly the same, or + - their slugs share most of their tokens. + """ + from difflib import SequenceMatcher + + out = [] + c_years = _case_years(_s(post.get("title")), _s(post.get("slug"))) + c_sig = _sig_tokens(post.get("title"), _s(post.get("slug"))) + c_hook = _hook(post.get("title")) + c_slug_toks = _tokens(_s(post.get("slug")).replace("-", " ")) + + for other in corpus: + if other.get("id") == post.get("id"): + continue + o_title, o_slug = _s(other.get("title")), _s(other.get("slug")) + o_years = _case_years(o_title, o_slug) + o_sig = _sig_tokens(o_title, o_slug) + + reasons = [] + if (c_years & o_years) and (c_sig & o_sig): + shared = ", ".join(sorted(c_sig & o_sig)[:3] + sorted(c_years & o_years)) + reasons.append((HIGH, f"same case + year ({shared})")) + hook_sim = SequenceMatcher(None, c_hook, _hook(o_title)).ratio() + if c_hook and hook_sim >= TITLE_HOOK_SIM_MIN: + reasons.append((MED, f"title hooks {hook_sim:.0%} similar")) + o_slug_toks = _tokens(o_slug.replace("-", " ")) + union = c_slug_toks | o_slug_toks + if union: + jac = len(c_slug_toks & o_slug_toks) / len(union) + if jac >= SLUG_JACCARD_MIN: + reasons.append((MED, f"slugs {jac:.0%} overlapping")) + + if reasons: + sev = max(s for s, _ in reasons) + why = "; ".join(r for _, r in reasons) + out.append(Violation( + "topic.collision", sev, + f"collides with [{other.get('status', '?')}] \"{o_title[:60]}\" — {why}", + "merge, retitle to a distinct angle, or interlink deliberately")) + return out + + RULES = [ r_meta_title, r_meta_description,