Compare commits
2
Commits
4ee9ad064e
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
dadc030d16 | ||
|
|
0e9a2e5b36 |
@@ -612,6 +612,7 @@ class OutputGenerator:
|
|||||||
return ""
|
return ""
|
||||||
title = _extract_title(full_output) or topic
|
title = _extract_title(full_output) or topic
|
||||||
mode = _resolve_seo_mode(seo_override)
|
mode = _resolve_seo_mode(seo_override)
|
||||||
|
collision_note = await self._collision_note(lang, title)
|
||||||
|
|
||||||
if mode in ("on", "dryrun"):
|
if mode in ("on", "dryrun"):
|
||||||
try:
|
try:
|
||||||
@@ -636,13 +637,17 @@ class OutputGenerator:
|
|||||||
# surface the proposal so Jose can inspect before trusting writes.
|
# surface the proposal so Jose can inspect before trusting writes.
|
||||||
result = await ghost.publish_draft(title, full_output)
|
result = await ghost.publish_draft(title, full_output)
|
||||||
post = result["posts"][0]
|
post = result["posts"][0]
|
||||||
self.last_publish_notice = _seo_dryrun_message(ghost, post, seo, inserted_pairs)
|
self.last_publish_notice = (
|
||||||
|
_seo_dryrun_message(ghost, post, seo, inserted_pairs)
|
||||||
|
+ collision_note)
|
||||||
else:
|
else:
|
||||||
result = await ghost.publish_draft(
|
result = await ghost.publish_draft(
|
||||||
title, full_output, tags=seo["tags"], seo=seo,
|
title, full_output, tags=seo["tags"], seo=seo,
|
||||||
body_html=linked_html)
|
body_html=linked_html)
|
||||||
post = result["posts"][0]
|
post = result["posts"][0]
|
||||||
self.last_publish_notice = _seo_live_message(ghost, post, seo, inserted_pairs)
|
self.last_publish_notice = (
|
||||||
|
_seo_live_message(ghost, post, seo, inserted_pairs)
|
||||||
|
+ collision_note)
|
||||||
logger.info("Auto-published blog to Ghost",
|
logger.info("Auto-published blog to Ghost",
|
||||||
mode=mode, post_id=post["id"], links=len(inserted_pairs))
|
mode=mode, post_id=post["id"], links=len(inserted_pairs))
|
||||||
return ""
|
return ""
|
||||||
@@ -661,11 +666,27 @@ class OutputGenerator:
|
|||||||
result = await ghost.publish_draft(title, full_output)
|
result = await ghost.publish_draft(title, full_output)
|
||||||
post = result["posts"][0]
|
post = result["posts"][0]
|
||||||
logger.info("Auto-published blog to Ghost (bare)", post_id=post["id"])
|
logger.info("Auto-published blog to Ghost (bare)", post_id=post["id"])
|
||||||
return _bare_ghost_notice(ghost, post)
|
return _bare_ghost_notice(ghost, post) + collision_note
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("Auto-publish to Ghost failed", error=str(e))
|
logger.warning("Auto-publish to Ghost failed", error=str(e))
|
||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
async def _collision_note(self, lang: str, title: str) -> str:
|
||||||
|
"""Aviso de canibalización del título propuesto contra los posts
|
||||||
|
published+scheduled del sitio. Solo EN (las stopwords del motor son
|
||||||
|
inglesas). Nunca lanza y nunca bloquea: el draft se publica igual y
|
||||||
|
el aviso viaja en el notice de Telegram.
|
||||||
|
"""
|
||||||
|
if lang != "en":
|
||||||
|
return ""
|
||||||
|
try:
|
||||||
|
from src.seo.autofill import collision_notice, fetch_collision_corpus
|
||||||
|
corpus = await fetch_collision_corpus(lang)
|
||||||
|
return collision_notice(title, corpus) or ""
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("Topic collision check failed — skipped", error=str(e))
|
||||||
|
return ""
|
||||||
|
|
||||||
async def generate(self, session_id: int, output_type: OutputType,
|
async def generate(self, session_id: int, output_type: OutputType,
|
||||||
progress_callback=None, lang: str = "es",
|
progress_callback=None, lang: str = "es",
|
||||||
seo_override: str | None = None) -> str:
|
seo_override: str | None = None) -> str:
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
b8faa93b1f7727d3870e18f69b68283d9a97ed9da819ef0cfa79a60cc2c4ab70
|
73872aaf4079e5a0b690fbbe7de7e7967edc2297c8bad79142308714452c60ef
|
||||||
|
|||||||
@@ -650,3 +650,62 @@ def insert_internal_links(
|
|||||||
phrase=phrase, slug=slug)
|
phrase=phrase, slug=slug)
|
||||||
|
|
||||||
return "".join(tokens), inserted_pairs
|
return "".join(tokens), inserted_pairs
|
||||||
|
|
||||||
|
# ─── 4. Topic collision (aviso pre-publish) ──────────────────────────────────
|
||||||
|
|
||||||
|
_MD_UNSAFE = re.compile(r"[*_`\[\]]")
|
||||||
|
|
||||||
|
|
||||||
|
def _slugify_title(title: str) -> str:
|
||||||
|
"""Aproximación del slug que Ghost generará del título — el draft aún no
|
||||||
|
tiene slug real, y topic_collision usa el slug como una de sus señales."""
|
||||||
|
return re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-")
|
||||||
|
|
||||||
|
|
||||||
|
async def fetch_collision_corpus(lang: str) -> list[dict]:
|
||||||
|
"""Posts published+scheduled del sitio (id, slug, title, status) para el
|
||||||
|
check de colisión de tema. A diferencia de fetch_published_menu incluye
|
||||||
|
los PROGRAMADOS: chocar con la cola de vacaciones es justo el caso a
|
||||||
|
cazar (2026-07-10: segundo Kecksburg publicado con otro ya en cola).
|
||||||
|
Aislamiento total: cualquier fallo → [] y log, nunca raise.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# Lazy import to avoid a heavy/circular import at module load.
|
||||||
|
from src.generator.generator import GhostPublisher
|
||||||
|
|
||||||
|
pub = GhostPublisher(lang=lang)
|
||||||
|
if not pub.is_configured():
|
||||||
|
return []
|
||||||
|
data = await pub._admin_get(
|
||||||
|
"posts/?filter=status:[published,scheduled]"
|
||||||
|
"&fields=id,slug,title,status&limit=all",
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
if data is None:
|
||||||
|
return []
|
||||||
|
corpus = [p for p in data.get("posts", []) if p.get("slug")]
|
||||||
|
logger.info("seo.collision: corpus fetched", lang=lang, count=len(corpus))
|
||||||
|
return corpus
|
||||||
|
except Exception as e: # noqa: BLE001 — isolation guarantee
|
||||||
|
logger.warning("seo.collision: corpus fetch failed", lang=lang, error=str(e))
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def collision_notice(title: str, corpus: list[dict]) -> str | None:
|
||||||
|
"""Aviso (Markdown seguro para Telegram) si el título propuesto colisiona
|
||||||
|
con un post existente, vía el motor vendorizado R.topic_collision.
|
||||||
|
None si no hay colisión. Los títulos ajenos se sanean de entidades
|
||||||
|
Markdown para no romper el parseo del mensaje (ver regla de _safe_send).
|
||||||
|
"""
|
||||||
|
if not corpus:
|
||||||
|
return None
|
||||||
|
candidate = {"id": None, "title": title, "slug": _slugify_title(title)}
|
||||||
|
violations = R.topic_collision(candidate, corpus)
|
||||||
|
if not violations:
|
||||||
|
return None
|
||||||
|
lines = [_MD_UNSAFE.sub("", v.message) for v in violations[:3]]
|
||||||
|
return (
|
||||||
|
"\n\n🚨 *Posible colisión de tema* — el draft se ha creado igualmente:\n"
|
||||||
|
+ "\n".join(f"• {line}" for line in lines)
|
||||||
|
+ "\nAntes de publicar: fusionar, retitular a otro ángulo o enlazar a propósito."
|
||||||
|
)
|
||||||
|
|||||||
@@ -195,6 +195,112 @@ def r_jsonld(p):
|
|||||||
return [Violation("jsonld.missing", MED, "no BlogPosting JSON-LD", "add JSON-LD")]
|
return [Violation("jsonld.missing", MED, "no BlogPosting JSON-LD", "add JSON-LD")]
|
||||||
|
|
||||||
|
|
||||||
|
# ---- topic collision (corpus-aware; NOT in RULES) ---------------------------
|
||||||
|
# Two posts about the same case cannibalize each other in the SERP (2026-07-10:
|
||||||
|
# a second Kecksburg post was published while another sat scheduled; a "When
|
||||||
|
# Nuclear ... Went/Go Silent" near-twin title was already queued). RULES functions
|
||||||
|
# are (post) -> violations; this one also needs the rest of the site, so callers
|
||||||
|
# (seo_validate.py) pass the corpus explicitly: published + scheduled posts as
|
||||||
|
# dicts with at least {id, title, slug}.
|
||||||
|
|
||||||
|
TOPIC_STOPWORDS = {
|
||||||
|
# english glue
|
||||||
|
"the", "a", "an", "of", "and", "in", "at", "on", "to", "that", "what",
|
||||||
|
"when", "who", "why", "how", "its", "his", "her", "their", "our", "one",
|
||||||
|
"still", "cant", "couldnt", "went", "go", "goes", "most", "from", "with",
|
||||||
|
"they", "them", "these", "this", "are", "were", "was", "is", "be", "been",
|
||||||
|
"has", "have", "had", "but", "for", "all", "than", "then", "ever", "never",
|
||||||
|
# domain-generic (present in half the catalog — carry no case identity)
|
||||||
|
"ufo", "ufos", "uap", "uaps", "incident", "incidents", "case", "cases",
|
||||||
|
"file", "files", "mystery", "declassified", "declassification", "pentagon",
|
||||||
|
"government", "military", "congress", "secret", "program", "investigation",
|
||||||
|
"evidence", "witness", "witnesses", "document", "documents", "documented",
|
||||||
|
"unexplained", "encounter", "sighting", "sightings", "alien", "aliens",
|
||||||
|
"phenomena", "aerial", "unidentified", "extraordinary", "americas",
|
||||||
|
"american", "video", "footage",
|
||||||
|
}
|
||||||
|
# 0.70 calibrated 2026-07-10: the "When Nuclear Weapons Go / Arsenal Went
|
||||||
|
# Silent" near-twin pair scores 0.742 (char-level penalizes weapons/arsenal);
|
||||||
|
# the closest legit-distinct pair in the catalog scores 0.65.
|
||||||
|
TITLE_HOOK_SIM_MIN = 0.70 # SequenceMatcher on the pre-colon hook
|
||||||
|
SLUG_JACCARD_MIN = 0.5 # shared slug-token ratio
|
||||||
|
# Years >= this are "news era", not case identity: every contemporary post
|
||||||
|
# carries the current year (PURSUE 2026, Grusch 2026...) without being the same
|
||||||
|
# story. Case years in the catalog run 1947-2019.
|
||||||
|
NEWS_YEAR_MIN = 2020
|
||||||
|
|
||||||
|
_YEAR_RE = re.compile(r"\b(19|20)\d{2}\b")
|
||||||
|
|
||||||
|
|
||||||
|
def _tokens(text):
|
||||||
|
return set(re.findall(r"[a-z0-9]+", _s(text).lower()))
|
||||||
|
|
||||||
|
|
||||||
|
def _case_years(title, slug):
|
||||||
|
"""Historical case years (pre news-era) found in title+slug."""
|
||||||
|
return {m.group(0) for m in _YEAR_RE.finditer(title + " " + slug)
|
||||||
|
if int(m.group(0)) < NEWS_YEAR_MIN}
|
||||||
|
|
||||||
|
|
||||||
|
def _sig_tokens(title, slug):
|
||||||
|
"""Case-identity tokens: title+slug minus glue/domain words, years and
|
||||||
|
fragments shorter than 3 chars (possessive 's', initials...)."""
|
||||||
|
toks = _tokens(title) | _tokens(slug.replace("-", " "))
|
||||||
|
return {t for t in toks
|
||||||
|
if len(t) >= 3 and t not in TOPIC_STOPWORDS and not _YEAR_RE.fullmatch(t)}
|
||||||
|
|
||||||
|
|
||||||
|
def _hook(title):
|
||||||
|
return _s(title).split(":")[0].strip().lower()
|
||||||
|
|
||||||
|
|
||||||
|
def topic_collision(post, corpus):
|
||||||
|
"""Compare one candidate post against the site corpus → list[Violation].
|
||||||
|
|
||||||
|
Fires when the candidate and an existing post look like the same story:
|
||||||
|
- share a case year AND a case-identity token (Kecksburg+1965), or
|
||||||
|
- their pre-colon title hooks read nearly the same, or
|
||||||
|
- their slugs share most of their tokens.
|
||||||
|
"""
|
||||||
|
from difflib import SequenceMatcher
|
||||||
|
|
||||||
|
out = []
|
||||||
|
c_years = _case_years(_s(post.get("title")), _s(post.get("slug")))
|
||||||
|
c_sig = _sig_tokens(post.get("title"), _s(post.get("slug")))
|
||||||
|
c_hook = _hook(post.get("title"))
|
||||||
|
c_slug_toks = _tokens(_s(post.get("slug")).replace("-", " "))
|
||||||
|
|
||||||
|
for other in corpus:
|
||||||
|
if other.get("id") == post.get("id"):
|
||||||
|
continue
|
||||||
|
o_title, o_slug = _s(other.get("title")), _s(other.get("slug"))
|
||||||
|
o_years = _case_years(o_title, o_slug)
|
||||||
|
o_sig = _sig_tokens(o_title, o_slug)
|
||||||
|
|
||||||
|
reasons = []
|
||||||
|
if (c_years & o_years) and (c_sig & o_sig):
|
||||||
|
shared = ", ".join(sorted(c_sig & o_sig)[:3] + sorted(c_years & o_years))
|
||||||
|
reasons.append((HIGH, f"same case + year ({shared})"))
|
||||||
|
hook_sim = SequenceMatcher(None, c_hook, _hook(o_title)).ratio()
|
||||||
|
if c_hook and hook_sim >= TITLE_HOOK_SIM_MIN:
|
||||||
|
reasons.append((MED, f"title hooks {hook_sim:.0%} similar"))
|
||||||
|
o_slug_toks = _tokens(o_slug.replace("-", " "))
|
||||||
|
union = c_slug_toks | o_slug_toks
|
||||||
|
if union:
|
||||||
|
jac = len(c_slug_toks & o_slug_toks) / len(union)
|
||||||
|
if jac >= SLUG_JACCARD_MIN:
|
||||||
|
reasons.append((MED, f"slugs {jac:.0%} overlapping"))
|
||||||
|
|
||||||
|
if reasons:
|
||||||
|
sev = max(s for s, _ in reasons)
|
||||||
|
why = "; ".join(r for _, r in reasons)
|
||||||
|
out.append(Violation(
|
||||||
|
"topic.collision", sev,
|
||||||
|
f"collides with [{other.get('status', '?')}] \"{o_title[:60]}\" — {why}",
|
||||||
|
"merge, retitle to a distinct angle, or interlink deliberately"))
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
RULES = [
|
RULES = [
|
||||||
r_meta_title,
|
r_meta_title,
|
||||||
r_meta_description,
|
r_meta_description,
|
||||||
|
|||||||
@@ -37,3 +37,47 @@ def test_system_prompt_lists_allowed_tags_per_lang():
|
|||||||
assert 'never use "investigacion"' in en
|
assert 'never use "investigacion"' in en
|
||||||
assert 'never use "investigacion"' not in es
|
assert 'never use "investigacion"' not in es
|
||||||
assert "ONLY from this exact list" in es
|
assert "ONLY from this exact list" in es
|
||||||
|
|
||||||
|
|
||||||
|
# ─── topic collision ─────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
from src.seo.autofill import collision_notice, _slugify_title
|
||||||
|
|
||||||
|
CORPUS = [
|
||||||
|
{"id": "1", "status": "published", "slug": "kecksburg-1965-acorn-ufo-missing-nasa-files",
|
||||||
|
"title": 'Kecksburg 1965: The Acorn-Shaped Object, the Missing NASA Files, and "Pennsylvania\'s Roswell"'},
|
||||||
|
{"id": "2", "status": "scheduled", "slug": "uss-russell-2019-pyramid-uap-channel-islands",
|
||||||
|
"title": "USS Russell 2019: The Pyramid UAP Video and the Channel Islands Drone Swarm"},
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_collision_fires_on_same_case_and_year():
|
||||||
|
note = collision_notice("Kecksburg 1965: New Acorn Evidence", CORPUS)
|
||||||
|
assert note is not None
|
||||||
|
assert "kecksburg" in note.lower()
|
||||||
|
assert "1965" in note
|
||||||
|
|
||||||
|
|
||||||
|
def test_collision_none_on_distinct_case():
|
||||||
|
assert collision_notice("Tehran 1976: The Jet-Disabling Encounter", CORPUS) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_collision_none_on_empty_corpus():
|
||||||
|
assert collision_notice("Kecksburg 1965: Anything", []) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_collision_note_is_markdown_safe():
|
||||||
|
corpus = [{"id": "9", "status": "published", "slug": "weird-1990-case",
|
||||||
|
"title": "Weird *1990* [Case] with_underscores and `ticks`"}]
|
||||||
|
note = collision_notice("Weird 1990: Case Revisited", corpus)
|
||||||
|
assert note is not None
|
||||||
|
# las entidades Markdown de títulos ajenos se sanean (solo quedan las nuestras)
|
||||||
|
bullets = [line for line in note.split("\n") if line.startswith("• ")]
|
||||||
|
assert bullets
|
||||||
|
for line in bullets:
|
||||||
|
for ch in "*_`[]":
|
||||||
|
assert ch not in line
|
||||||
|
|
||||||
|
|
||||||
|
def test_slugify_title():
|
||||||
|
assert _slugify_title("USS Russell 2019: The Pyramid UAP!") == "uss-russell-2019-the-pyramid-uap"
|
||||||
|
|||||||
Reference in New Issue
Block a user