vendor: sincroniza el motor SEO (stopwords ES + tokenizador sin acentos)
Build & Deploy ResearchOwl / build-and-push (push) Successful in 1m9s

make sync-seo tras chemavx-seo-tools 31c9d3e. Solo afecta a topic_collision;
las RULES de check_post no usan el tokenizador. Los 9 tests de seo pasan.
This commit is contained in:
ChemaVX
2026-07-21 07:15:45 +00:00
parent dadc030d16
commit 2af654d344
2 changed files with 39 additions and 2 deletions
+38 -1
View File
@@ -30,6 +30,7 @@ rule, write a function (post) -> list[Violation] and append it to RULES. The aud
and the validator both just call check_post(); they never re-implement a check.
"""
import re
import unicodedata
from collections import namedtuple
SITE_HOST = "theexclusionzone.com"
@@ -218,6 +219,33 @@ TOPIC_STOPWORDS = {
"unexplained", "encounter", "sighting", "sightings", "alien", "aliens",
"phenomena", "aerial", "unidentified", "extraordinary", "americas",
"american", "video", "footage",
# spanish glue (added 2026-07-21 with the ES site — zonadeexclusion.com).
# Without these, "que"/"los"/"del" counted as case identity: Kenneth Arnold
# 1947 "collided" with Roswell 1947 on nothing but «que» + the shared year.
# Cost: "los" no longer identifies Los Alamos on EN — "alamos" still does,
# and the EN corpus reports the same collisions before and after.
"los", "las", "una", "unos", "unas", "del", "por", "para", "con", "sin",
"sus", "que", "cual", "cuales", "quien", "quienes", "donde", "cuando",
"como", "pero", "porque", "aunque", "sobre", "entre", "hasta", "desde",
"tras", "ante", "bajo", "durante", "segun", "este", "esta", "esto",
"estos", "estas", "ese", "esa", "eso", "esos", "esas", "aquel", "aquella",
"otro", "otra", "otros", "otras", "todo", "toda", "todos", "todas",
"mismo", "misma", "cada", "algo", "alguien", "nada", "nadie", "mas", "muy",
"aun", "solo", "tambien", "siempre", "nunca", "jamas", "casi", "menos",
"fue", "fueron", "era", "eran", "ser", "son", "estan", "estaba",
"estaban", "haber", "habia", "han", "hay", "hizo", "hacer", "hace",
"tiene", "tienen", "tenia", "puede", "pueden", "podria", "sigue",
"siguen", "sabe", "dice", "dicen", "ano", "anos", "dia", "dias", "vez",
"veces", "despues", "antes", "hoy", "ahora",
# domain-generic ES — mirror of the English block above
"ovni", "ovnis", "fenomeno", "fenomenos", "caso", "casos", "incidente",
"incidentes", "misterio", "misterios", "expediente", "expedientes",
"archivo", "archivos", "documento", "documentos", "desclasificado",
"desclasificados", "desclasificacion", "gobierno", "militar", "militares",
"ejercito", "secreto", "secretos", "investigacion", "testigo", "testigos",
"avistamiento", "avistamientos", "encuentro", "encuentros",
"extraterrestre", "extraterrestres", "alienigena", "alienigenas",
"inexplicable", "inexplicables", "aereo", "aerea", "videos",
}
# 0.70 calibrated 2026-07-10: the "When Nuclear Weapons Go / Arsenal Went
# Silent" near-twin pair scores 0.742 (char-level penalizes weapons/arsenal);
@@ -232,8 +260,17 @@ NEWS_YEAR_MIN = 2020
_YEAR_RE = re.compile(r"\b(19|20)\d{2}\b")
def _deaccent(text):
"""Fold accents to ASCII. Required for Spanish: the [a-z0-9]+ tokenizer
SPLITS on any accented char, so "Pentágono" became {pent, gono} and
"Fenómenos" became {fen, menos} — 3-char garbage that no stopword list can
ever cover, and that never matched the (already accent-free) Ghost slug.
No-op on EN, whose only non-ASCII are dashes and curly apostrophes."""
return unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode()
def _tokens(text):
return set(re.findall(r"[a-z0-9]+", _s(text).lower()))
return set(re.findall(r"[a-z0-9]+", _deaccent(_s(text)).lower()))
def _case_years(title, slug):