diff --git a/seo_rules.py b/seo_rules.py index 08d122a..2ec363f 100644 --- a/seo_rules.py +++ b/seo_rules.py @@ -178,6 +178,112 @@ def r_jsonld(p): return [Violation("jsonld.missing", MED, "no BlogPosting JSON-LD", "add JSON-LD")] +# ---- topic collision (corpus-aware; NOT in RULES) --------------------------- +# Two posts about the same case cannibalize each other in the SERP (2026-07-10: +# a second Kecksburg post was published while another sat scheduled; a "When +# Nuclear ... Went/Go Silent" near-twin title was already queued). RULES functions +# are (post) -> violations; this one also needs the rest of the site, so callers +# (seo_validate.py) pass the corpus explicitly: published + scheduled posts as +# dicts with at least {id, title, slug}. + +TOPIC_STOPWORDS = { + # english glue + "the", "a", "an", "of", "and", "in", "at", "on", "to", "that", "what", + "when", "who", "why", "how", "its", "his", "her", "their", "our", "one", + "still", "cant", "couldnt", "went", "go", "goes", "most", "from", "with", + "they", "them", "these", "this", "are", "were", "was", "is", "be", "been", + "has", "have", "had", "but", "for", "all", "than", "then", "ever", "never", + # domain-generic (present in half the catalog — carry no case identity) + "ufo", "ufos", "uap", "uaps", "incident", "incidents", "case", "cases", + "file", "files", "mystery", "declassified", "declassification", "pentagon", + "government", "military", "congress", "secret", "program", "investigation", + "evidence", "witness", "witnesses", "document", "documents", "documented", + "unexplained", "encounter", "sighting", "sightings", "alien", "aliens", + "phenomena", "aerial", "unidentified", "extraordinary", "americas", + "american", "video", "footage", +} +# 0.70 calibrated 2026-07-10: the "When Nuclear Weapons Go / Arsenal Went +# Silent" near-twin pair scores 0.742 (char-level penalizes weapons/arsenal); +# the closest legit-distinct pair in the catalog scores 0.65. +TITLE_HOOK_SIM_MIN = 0.70 # SequenceMatcher on the pre-colon hook +SLUG_JACCARD_MIN = 0.5 # shared slug-token ratio +# Years >= this are "news era", not case identity: every contemporary post +# carries the current year (PURSUE 2026, Grusch 2026...) without being the same +# story. Case years in the catalog run 1947-2019. +NEWS_YEAR_MIN = 2020 + +_YEAR_RE = re.compile(r"\b(19|20)\d{2}\b") + + +def _tokens(text): + return set(re.findall(r"[a-z0-9]+", _s(text).lower())) + + +def _case_years(title, slug): + """Historical case years (pre news-era) found in title+slug.""" + return {m.group(0) for m in _YEAR_RE.finditer(title + " " + slug) + if int(m.group(0)) < NEWS_YEAR_MIN} + + +def _sig_tokens(title, slug): + """Case-identity tokens: title+slug minus glue/domain words, years and + fragments shorter than 3 chars (possessive 's', initials...).""" + toks = _tokens(title) | _tokens(slug.replace("-", " ")) + return {t for t in toks + if len(t) >= 3 and t not in TOPIC_STOPWORDS and not _YEAR_RE.fullmatch(t)} + + +def _hook(title): + return _s(title).split(":")[0].strip().lower() + + +def topic_collision(post, corpus): + """Compare one candidate post against the site corpus → list[Violation]. + + Fires when the candidate and an existing post look like the same story: + - share a case year AND a case-identity token (Kecksburg+1965), or + - their pre-colon title hooks read nearly the same, or + - their slugs share most of their tokens. + """ + from difflib import SequenceMatcher + + out = [] + c_years = _case_years(_s(post.get("title")), _s(post.get("slug"))) + c_sig = _sig_tokens(post.get("title"), _s(post.get("slug"))) + c_hook = _hook(post.get("title")) + c_slug_toks = _tokens(_s(post.get("slug")).replace("-", " ")) + + for other in corpus: + if other.get("id") == post.get("id"): + continue + o_title, o_slug = _s(other.get("title")), _s(other.get("slug")) + o_years = _case_years(o_title, o_slug) + o_sig = _sig_tokens(o_title, o_slug) + + reasons = [] + if (c_years & o_years) and (c_sig & o_sig): + shared = ", ".join(sorted(c_sig & o_sig)[:3] + sorted(c_years & o_years)) + reasons.append((HIGH, f"same case + year ({shared})")) + hook_sim = SequenceMatcher(None, c_hook, _hook(o_title)).ratio() + if c_hook and hook_sim >= TITLE_HOOK_SIM_MIN: + reasons.append((MED, f"title hooks {hook_sim:.0%} similar")) + o_slug_toks = _tokens(o_slug.replace("-", " ")) + union = c_slug_toks | o_slug_toks + if union: + jac = len(c_slug_toks & o_slug_toks) / len(union) + if jac >= SLUG_JACCARD_MIN: + reasons.append((MED, f"slugs {jac:.0%} overlapping")) + + if reasons: + sev = max(s for s, _ in reasons) + why = "; ".join(r for _, r in reasons) + out.append(Violation( + "topic.collision", sev, + f"collides with [{other.get('status', '?')}] \"{o_title[:60]}\" — {why}", + "merge, retitle to a distinct angle, or interlink deliberately")) + return out + + RULES = [ r_meta_title, r_meta_description, diff --git a/seo_validate.py b/seo_validate.py index 01d5203..8d68b60 100644 --- a/seo_validate.py +++ b/seo_validate.py @@ -39,9 +39,31 @@ GROUPS = [ ("feature_image", ("feature_image.",)), ("feature_image_alt", ("feature_image_alt.",)), ("internal_links", ("internal_links.",)), + ("topic_collision", ("topic.",)), ] +def fetch_corpus(): + """All published+scheduled posts (id, title, slug, status) via ghst-en, for + the topic-collision check. Returns [] (with a warning) if the fetch fails — + the validator still runs the per-post rules.""" + fd, path = tempfile.mkstemp(suffix=".json", prefix="seo_corpus_") + os.close(fd) + try: + cmd = (f"ghst-en --json post list --limit all " + f"--fields id,title,slug,status > {path}") + r = subprocess.run(cmd, shell=True, stderr=subprocess.PIPE, text=True) + if r.returncode != 0: + print(f" [warn] corpus fetch failed — topic collision NOT checked:\n{r.stderr}", + file=sys.stderr) + return [] + data = json.load(open(path)) + posts = data.get("posts", data) if isinstance(data, dict) else data + return [p for p in posts if p.get("status") in ("published", "scheduled")] + finally: + os.unlink(path) + + def fetch_one(selector, value): """Fetch a single post (draft or published) via ghst-en → post dict. @@ -76,10 +98,13 @@ def load_json(path): return _unwrap(json.load(open(path))) -def validate(post): - """Run the shared engine and print a per-group PASS/FAIL report. +def validate(post, corpus=None): + """Run the shared engine (plus the corpus-aware topic-collision check when a + corpus is given) and print a per-group PASS/FAIL report. Returns the number of actionable (non-INFO) violations.""" violations = R.check_post(post) + if corpus: + violations.extend(R.topic_collision(post, corpus)) actionable = [v for v in violations if v.severity > R.INFO] info = [v for v in violations if v.severity == R.INFO] @@ -129,6 +154,8 @@ def main(): g.add_argument("--slug") g.add_argument("--id") g.add_argument("--json", metavar="FILE") + ap.add_argument("--no-collision", action="store_true", + help="skip the topic-collision check (intentional follow-up piece)") args = ap.parse_args() if args.json: @@ -138,7 +165,8 @@ def main(): else: post = fetch_one("id", args.id) - n = validate(post) + corpus = [] if args.no_collision else fetch_corpus() + n = validate(post, corpus) sys.exit(1 if n else 0)