vendor: sincroniza el motor SEO (stopwords ES + tokenizador sin acentos)
Build & Deploy ResearchOwl / build-and-push (push) Successful in 1m9s
Build & Deploy ResearchOwl / build-and-push (push) Successful in 1m9s
make sync-seo tras chemavx-seo-tools 31c9d3e. Solo afecta a topic_collision; las RULES de check_post no usan el tokenizador. Los 9 tests de seo pasan.
This commit is contained in:
+38
-1
@@ -30,6 +30,7 @@ rule, write a function (post) -> list[Violation] and append it to RULES. The aud
|
||||
and the validator both just call check_post(); they never re-implement a check.
|
||||
"""
|
||||
import re
|
||||
import unicodedata
|
||||
from collections import namedtuple
|
||||
|
||||
SITE_HOST = "theexclusionzone.com"
|
||||
@@ -218,6 +219,33 @@ TOPIC_STOPWORDS = {
|
||||
"unexplained", "encounter", "sighting", "sightings", "alien", "aliens",
|
||||
"phenomena", "aerial", "unidentified", "extraordinary", "americas",
|
||||
"american", "video", "footage",
|
||||
# spanish glue (added 2026-07-21 with the ES site — zonadeexclusion.com).
|
||||
# Without these, "que"/"los"/"del" counted as case identity: Kenneth Arnold
|
||||
# 1947 "collided" with Roswell 1947 on nothing but «que» + the shared year.
|
||||
# Cost: "los" no longer identifies Los Alamos on EN — "alamos" still does,
|
||||
# and the EN corpus reports the same collisions before and after.
|
||||
"los", "las", "una", "unos", "unas", "del", "por", "para", "con", "sin",
|
||||
"sus", "que", "cual", "cuales", "quien", "quienes", "donde", "cuando",
|
||||
"como", "pero", "porque", "aunque", "sobre", "entre", "hasta", "desde",
|
||||
"tras", "ante", "bajo", "durante", "segun", "este", "esta", "esto",
|
||||
"estos", "estas", "ese", "esa", "eso", "esos", "esas", "aquel", "aquella",
|
||||
"otro", "otra", "otros", "otras", "todo", "toda", "todos", "todas",
|
||||
"mismo", "misma", "cada", "algo", "alguien", "nada", "nadie", "mas", "muy",
|
||||
"aun", "solo", "tambien", "siempre", "nunca", "jamas", "casi", "menos",
|
||||
"fue", "fueron", "era", "eran", "ser", "son", "estan", "estaba",
|
||||
"estaban", "haber", "habia", "han", "hay", "hizo", "hacer", "hace",
|
||||
"tiene", "tienen", "tenia", "puede", "pueden", "podria", "sigue",
|
||||
"siguen", "sabe", "dice", "dicen", "ano", "anos", "dia", "dias", "vez",
|
||||
"veces", "despues", "antes", "hoy", "ahora",
|
||||
# domain-generic ES — mirror of the English block above
|
||||
"ovni", "ovnis", "fenomeno", "fenomenos", "caso", "casos", "incidente",
|
||||
"incidentes", "misterio", "misterios", "expediente", "expedientes",
|
||||
"archivo", "archivos", "documento", "documentos", "desclasificado",
|
||||
"desclasificados", "desclasificacion", "gobierno", "militar", "militares",
|
||||
"ejercito", "secreto", "secretos", "investigacion", "testigo", "testigos",
|
||||
"avistamiento", "avistamientos", "encuentro", "encuentros",
|
||||
"extraterrestre", "extraterrestres", "alienigena", "alienigenas",
|
||||
"inexplicable", "inexplicables", "aereo", "aerea", "videos",
|
||||
}
|
||||
# 0.70 calibrated 2026-07-10: the "When Nuclear Weapons Go / Arsenal Went
|
||||
# Silent" near-twin pair scores 0.742 (char-level penalizes weapons/arsenal);
|
||||
@@ -232,8 +260,17 @@ NEWS_YEAR_MIN = 2020
|
||||
_YEAR_RE = re.compile(r"\b(19|20)\d{2}\b")
|
||||
|
||||
|
||||
def _deaccent(text):
|
||||
"""Fold accents to ASCII. Required for Spanish: the [a-z0-9]+ tokenizer
|
||||
SPLITS on any accented char, so "Pentágono" became {pent, gono} and
|
||||
"Fenómenos" became {fen, menos} — 3-char garbage that no stopword list can
|
||||
ever cover, and that never matched the (already accent-free) Ghost slug.
|
||||
No-op on EN, whose only non-ASCII are dashes and curly apostrophes."""
|
||||
return unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode()
|
||||
|
||||
|
||||
def _tokens(text):
|
||||
return set(re.findall(r"[a-z0-9]+", _s(text).lower()))
|
||||
return set(re.findall(r"[a-z0-9]+", _deaccent(_s(text)).lower()))
|
||||
|
||||
|
||||
def _case_years(title, slug):
|
||||
|
||||
Reference in New Issue
Block a user