feat: check de colisión de tema (topic.collision) en rules + validate
Dos posts sobre el mismo caso se canibalizan en la SERP (2026-07-10: se publicó un segundo Kecksburg con otro ya programado, y había un título casi gemelo del de Malmstrom en la cola). Nueva función pura topic_collision(post, corpus) en seo_rules.py — corpus-aware, fuera de RULES — que dispara por: caso+año compartidos (años < 2020; los de la era de noticias no son identidad de caso), hooks de título ≥70% similares (calibrado: el par nuclear da 0.742, el par legítimo más cercano 0.65), o slugs ≥50% solapados. seo_validate.py obtiene el corpus via ghst-en (--limit all) y lo reporta como grupo topic_collision; --no-collision lo omite para follow-ups intencionados. Calibrado contra los 36 posts reales: dispara solo el duplicado Kecksburg (HIGH) y el par Grusch preview/evento (MED, intencionado). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
beae7c154a
commit
bae1cc949c
+106
@@ -178,6 +178,112 @@ def r_jsonld(p):
|
|||||||
return [Violation("jsonld.missing", MED, "no BlogPosting JSON-LD", "add JSON-LD")]
|
return [Violation("jsonld.missing", MED, "no BlogPosting JSON-LD", "add JSON-LD")]
|
||||||
|
|
||||||
|
|
||||||
|
# ---- topic collision (corpus-aware; NOT in RULES) ---------------------------
|
||||||
|
# Two posts about the same case cannibalize each other in the SERP (2026-07-10:
|
||||||
|
# a second Kecksburg post was published while another sat scheduled; a "When
|
||||||
|
# Nuclear ... Went/Go Silent" near-twin title was already queued). RULES functions
|
||||||
|
# are (post) -> violations; this one also needs the rest of the site, so callers
|
||||||
|
# (seo_validate.py) pass the corpus explicitly: published + scheduled posts as
|
||||||
|
# dicts with at least {id, title, slug}.
|
||||||
|
|
||||||
|
TOPIC_STOPWORDS = {
|
||||||
|
# english glue
|
||||||
|
"the", "a", "an", "of", "and", "in", "at", "on", "to", "that", "what",
|
||||||
|
"when", "who", "why", "how", "its", "his", "her", "their", "our", "one",
|
||||||
|
"still", "cant", "couldnt", "went", "go", "goes", "most", "from", "with",
|
||||||
|
"they", "them", "these", "this", "are", "were", "was", "is", "be", "been",
|
||||||
|
"has", "have", "had", "but", "for", "all", "than", "then", "ever", "never",
|
||||||
|
# domain-generic (present in half the catalog — carry no case identity)
|
||||||
|
"ufo", "ufos", "uap", "uaps", "incident", "incidents", "case", "cases",
|
||||||
|
"file", "files", "mystery", "declassified", "declassification", "pentagon",
|
||||||
|
"government", "military", "congress", "secret", "program", "investigation",
|
||||||
|
"evidence", "witness", "witnesses", "document", "documents", "documented",
|
||||||
|
"unexplained", "encounter", "sighting", "sightings", "alien", "aliens",
|
||||||
|
"phenomena", "aerial", "unidentified", "extraordinary", "americas",
|
||||||
|
"american", "video", "footage",
|
||||||
|
}
|
||||||
|
# 0.70 calibrated 2026-07-10: the "When Nuclear Weapons Go / Arsenal Went
|
||||||
|
# Silent" near-twin pair scores 0.742 (char-level penalizes weapons/arsenal);
|
||||||
|
# the closest legit-distinct pair in the catalog scores 0.65.
|
||||||
|
TITLE_HOOK_SIM_MIN = 0.70 # SequenceMatcher on the pre-colon hook
|
||||||
|
SLUG_JACCARD_MIN = 0.5 # shared slug-token ratio
|
||||||
|
# Years >= this are "news era", not case identity: every contemporary post
|
||||||
|
# carries the current year (PURSUE 2026, Grusch 2026...) without being the same
|
||||||
|
# story. Case years in the catalog run 1947-2019.
|
||||||
|
NEWS_YEAR_MIN = 2020
|
||||||
|
|
||||||
|
_YEAR_RE = re.compile(r"\b(19|20)\d{2}\b")
|
||||||
|
|
||||||
|
|
||||||
|
def _tokens(text):
|
||||||
|
return set(re.findall(r"[a-z0-9]+", _s(text).lower()))
|
||||||
|
|
||||||
|
|
||||||
|
def _case_years(title, slug):
|
||||||
|
"""Historical case years (pre news-era) found in title+slug."""
|
||||||
|
return {m.group(0) for m in _YEAR_RE.finditer(title + " " + slug)
|
||||||
|
if int(m.group(0)) < NEWS_YEAR_MIN}
|
||||||
|
|
||||||
|
|
||||||
|
def _sig_tokens(title, slug):
|
||||||
|
"""Case-identity tokens: title+slug minus glue/domain words, years and
|
||||||
|
fragments shorter than 3 chars (possessive 's', initials...)."""
|
||||||
|
toks = _tokens(title) | _tokens(slug.replace("-", " "))
|
||||||
|
return {t for t in toks
|
||||||
|
if len(t) >= 3 and t not in TOPIC_STOPWORDS and not _YEAR_RE.fullmatch(t)}
|
||||||
|
|
||||||
|
|
||||||
|
def _hook(title):
|
||||||
|
return _s(title).split(":")[0].strip().lower()
|
||||||
|
|
||||||
|
|
||||||
|
def topic_collision(post, corpus):
|
||||||
|
"""Compare one candidate post against the site corpus → list[Violation].
|
||||||
|
|
||||||
|
Fires when the candidate and an existing post look like the same story:
|
||||||
|
- share a case year AND a case-identity token (Kecksburg+1965), or
|
||||||
|
- their pre-colon title hooks read nearly the same, or
|
||||||
|
- their slugs share most of their tokens.
|
||||||
|
"""
|
||||||
|
from difflib import SequenceMatcher
|
||||||
|
|
||||||
|
out = []
|
||||||
|
c_years = _case_years(_s(post.get("title")), _s(post.get("slug")))
|
||||||
|
c_sig = _sig_tokens(post.get("title"), _s(post.get("slug")))
|
||||||
|
c_hook = _hook(post.get("title"))
|
||||||
|
c_slug_toks = _tokens(_s(post.get("slug")).replace("-", " "))
|
||||||
|
|
||||||
|
for other in corpus:
|
||||||
|
if other.get("id") == post.get("id"):
|
||||||
|
continue
|
||||||
|
o_title, o_slug = _s(other.get("title")), _s(other.get("slug"))
|
||||||
|
o_years = _case_years(o_title, o_slug)
|
||||||
|
o_sig = _sig_tokens(o_title, o_slug)
|
||||||
|
|
||||||
|
reasons = []
|
||||||
|
if (c_years & o_years) and (c_sig & o_sig):
|
||||||
|
shared = ", ".join(sorted(c_sig & o_sig)[:3] + sorted(c_years & o_years))
|
||||||
|
reasons.append((HIGH, f"same case + year ({shared})"))
|
||||||
|
hook_sim = SequenceMatcher(None, c_hook, _hook(o_title)).ratio()
|
||||||
|
if c_hook and hook_sim >= TITLE_HOOK_SIM_MIN:
|
||||||
|
reasons.append((MED, f"title hooks {hook_sim:.0%} similar"))
|
||||||
|
o_slug_toks = _tokens(o_slug.replace("-", " "))
|
||||||
|
union = c_slug_toks | o_slug_toks
|
||||||
|
if union:
|
||||||
|
jac = len(c_slug_toks & o_slug_toks) / len(union)
|
||||||
|
if jac >= SLUG_JACCARD_MIN:
|
||||||
|
reasons.append((MED, f"slugs {jac:.0%} overlapping"))
|
||||||
|
|
||||||
|
if reasons:
|
||||||
|
sev = max(s for s, _ in reasons)
|
||||||
|
why = "; ".join(r for _, r in reasons)
|
||||||
|
out.append(Violation(
|
||||||
|
"topic.collision", sev,
|
||||||
|
f"collides with [{other.get('status', '?')}] \"{o_title[:60]}\" — {why}",
|
||||||
|
"merge, retitle to a distinct angle, or interlink deliberately"))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
RULES = [
|
RULES = [
|
||||||
r_meta_title,
|
r_meta_title,
|
||||||
r_meta_description,
|
r_meta_description,
|
||||||
|
|||||||
+31
-3
@@ -39,9 +39,31 @@ GROUPS = [
|
|||||||
("feature_image", ("feature_image.",)),
|
("feature_image", ("feature_image.",)),
|
||||||
("feature_image_alt", ("feature_image_alt.",)),
|
("feature_image_alt", ("feature_image_alt.",)),
|
||||||
("internal_links", ("internal_links.",)),
|
("internal_links", ("internal_links.",)),
|
||||||
|
("topic_collision", ("topic.",)),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_corpus():
|
||||||
|
"""All published+scheduled posts (id, title, slug, status) via ghst-en, for
|
||||||
|
the topic-collision check. Returns [] (with a warning) if the fetch fails —
|
||||||
|
the validator still runs the per-post rules."""
|
||||||
|
fd, path = tempfile.mkstemp(suffix=".json", prefix="seo_corpus_")
|
||||||
|
os.close(fd)
|
||||||
|
try:
|
||||||
|
cmd = (f"ghst-en --json post list --limit all "
|
||||||
|
f"--fields id,title,slug,status > {path}")
|
||||||
|
r = subprocess.run(cmd, shell=True, stderr=subprocess.PIPE, text=True)
|
||||||
|
if r.returncode != 0:
|
||||||
|
print(f" [warn] corpus fetch failed — topic collision NOT checked:\n{r.stderr}",
|
||||||
|
file=sys.stderr)
|
||||||
|
return []
|
||||||
|
data = json.load(open(path))
|
||||||
|
posts = data.get("posts", data) if isinstance(data, dict) else data
|
||||||
|
return [p for p in posts if p.get("status") in ("published", "scheduled")]
|
||||||
|
finally:
|
||||||
|
os.unlink(path)
|
||||||
|
|
||||||
|
|
||||||
def fetch_one(selector, value):
|
def fetch_one(selector, value):
|
||||||
"""Fetch a single post (draft or published) via ghst-en → post dict.
|
"""Fetch a single post (draft or published) via ghst-en → post dict.
|
||||||
|
|
||||||
@@ -76,10 +98,13 @@ def load_json(path):
|
|||||||
return _unwrap(json.load(open(path)))
|
return _unwrap(json.load(open(path)))
|
||||||
|
|
||||||
|
|
||||||
def validate(post):
|
def validate(post, corpus=None):
|
||||||
"""Run the shared engine and print a per-group PASS/FAIL report.
|
"""Run the shared engine (plus the corpus-aware topic-collision check when a
|
||||||
|
corpus is given) and print a per-group PASS/FAIL report.
|
||||||
Returns the number of actionable (non-INFO) violations."""
|
Returns the number of actionable (non-INFO) violations."""
|
||||||
violations = R.check_post(post)
|
violations = R.check_post(post)
|
||||||
|
if corpus:
|
||||||
|
violations.extend(R.topic_collision(post, corpus))
|
||||||
actionable = [v for v in violations if v.severity > R.INFO]
|
actionable = [v for v in violations if v.severity > R.INFO]
|
||||||
info = [v for v in violations if v.severity == R.INFO]
|
info = [v for v in violations if v.severity == R.INFO]
|
||||||
|
|
||||||
@@ -129,6 +154,8 @@ def main():
|
|||||||
g.add_argument("--slug")
|
g.add_argument("--slug")
|
||||||
g.add_argument("--id")
|
g.add_argument("--id")
|
||||||
g.add_argument("--json", metavar="FILE")
|
g.add_argument("--json", metavar="FILE")
|
||||||
|
ap.add_argument("--no-collision", action="store_true",
|
||||||
|
help="skip the topic-collision check (intentional follow-up piece)")
|
||||||
args = ap.parse_args()
|
args = ap.parse_args()
|
||||||
|
|
||||||
if args.json:
|
if args.json:
|
||||||
@@ -138,7 +165,8 @@ def main():
|
|||||||
else:
|
else:
|
||||||
post = fetch_one("id", args.id)
|
post = fetch_one("id", args.id)
|
||||||
|
|
||||||
n = validate(post)
|
corpus = [] if args.no_collision else fetch_corpus()
|
||||||
|
n = validate(post, corpus)
|
||||||
sys.exit(1 if n else 0)
|
sys.exit(1 if n else 0)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user