vendor: sincroniza el motor SEO (stopwords ES + tokenizador sin acentos)
Build & Deploy ResearchOwl / build-and-push (push) Successful in 1m9s
Build & Deploy ResearchOwl / build-and-push (push) Successful in 1m9s
make sync-seo tras chemavx-seo-tools 31c9d3e. Solo afecta a topic_collision; las RULES de check_post no usan el tokenizador. Los 9 tests de seo pasan.
This commit is contained in:
@@ -1 +1 @@
|
|||||||
73872aaf4079e5a0b690fbbe7de7e7967edc2297c8bad79142308714452c60ef
|
d373af317c287d1516d4487e1bcb874e98d0b76a402b9115c59280cef0050b97
|
||||||
|
|||||||
+38
-1
@@ -30,6 +30,7 @@ rule, write a function (post) -> list[Violation] and append it to RULES. The aud
|
|||||||
and the validator both just call check_post(); they never re-implement a check.
|
and the validator both just call check_post(); they never re-implement a check.
|
||||||
"""
|
"""
|
||||||
import re
|
import re
|
||||||
|
import unicodedata
|
||||||
from collections import namedtuple
|
from collections import namedtuple
|
||||||
|
|
||||||
SITE_HOST = "theexclusionzone.com"
|
SITE_HOST = "theexclusionzone.com"
|
||||||
@@ -218,6 +219,33 @@ TOPIC_STOPWORDS = {
|
|||||||
"unexplained", "encounter", "sighting", "sightings", "alien", "aliens",
|
"unexplained", "encounter", "sighting", "sightings", "alien", "aliens",
|
||||||
"phenomena", "aerial", "unidentified", "extraordinary", "americas",
|
"phenomena", "aerial", "unidentified", "extraordinary", "americas",
|
||||||
"american", "video", "footage",
|
"american", "video", "footage",
|
||||||
|
# spanish glue (added 2026-07-21 with the ES site — zonadeexclusion.com).
|
||||||
|
# Without these, "que"/"los"/"del" counted as case identity: Kenneth Arnold
|
||||||
|
# 1947 "collided" with Roswell 1947 on nothing but «que» + the shared year.
|
||||||
|
# Cost: "los" no longer identifies Los Alamos on EN — "alamos" still does,
|
||||||
|
# and the EN corpus reports the same collisions before and after.
|
||||||
|
"los", "las", "una", "unos", "unas", "del", "por", "para", "con", "sin",
|
||||||
|
"sus", "que", "cual", "cuales", "quien", "quienes", "donde", "cuando",
|
||||||
|
"como", "pero", "porque", "aunque", "sobre", "entre", "hasta", "desde",
|
||||||
|
"tras", "ante", "bajo", "durante", "segun", "este", "esta", "esto",
|
||||||
|
"estos", "estas", "ese", "esa", "eso", "esos", "esas", "aquel", "aquella",
|
||||||
|
"otro", "otra", "otros", "otras", "todo", "toda", "todos", "todas",
|
||||||
|
"mismo", "misma", "cada", "algo", "alguien", "nada", "nadie", "mas", "muy",
|
||||||
|
"aun", "solo", "tambien", "siempre", "nunca", "jamas", "casi", "menos",
|
||||||
|
"fue", "fueron", "era", "eran", "ser", "son", "estan", "estaba",
|
||||||
|
"estaban", "haber", "habia", "han", "hay", "hizo", "hacer", "hace",
|
||||||
|
"tiene", "tienen", "tenia", "puede", "pueden", "podria", "sigue",
|
||||||
|
"siguen", "sabe", "dice", "dicen", "ano", "anos", "dia", "dias", "vez",
|
||||||
|
"veces", "despues", "antes", "hoy", "ahora",
|
||||||
|
# domain-generic ES — mirror of the English block above
|
||||||
|
"ovni", "ovnis", "fenomeno", "fenomenos", "caso", "casos", "incidente",
|
||||||
|
"incidentes", "misterio", "misterios", "expediente", "expedientes",
|
||||||
|
"archivo", "archivos", "documento", "documentos", "desclasificado",
|
||||||
|
"desclasificados", "desclasificacion", "gobierno", "militar", "militares",
|
||||||
|
"ejercito", "secreto", "secretos", "investigacion", "testigo", "testigos",
|
||||||
|
"avistamiento", "avistamientos", "encuentro", "encuentros",
|
||||||
|
"extraterrestre", "extraterrestres", "alienigena", "alienigenas",
|
||||||
|
"inexplicable", "inexplicables", "aereo", "aerea", "videos",
|
||||||
}
|
}
|
||||||
# 0.70 calibrated 2026-07-10: the "When Nuclear Weapons Go / Arsenal Went
|
# 0.70 calibrated 2026-07-10: the "When Nuclear Weapons Go / Arsenal Went
|
||||||
# Silent" near-twin pair scores 0.742 (char-level penalizes weapons/arsenal);
|
# Silent" near-twin pair scores 0.742 (char-level penalizes weapons/arsenal);
|
||||||
@@ -232,8 +260,17 @@ NEWS_YEAR_MIN = 2020
|
|||||||
_YEAR_RE = re.compile(r"\b(19|20)\d{2}\b")
|
_YEAR_RE = re.compile(r"\b(19|20)\d{2}\b")
|
||||||
|
|
||||||
|
|
||||||
|
def _deaccent(text):
|
||||||
|
"""Fold accents to ASCII. Required for Spanish: the [a-z0-9]+ tokenizer
|
||||||
|
SPLITS on any accented char, so "Pentágono" became {pent, gono} and
|
||||||
|
"Fenómenos" became {fen, menos} — 3-char garbage that no stopword list can
|
||||||
|
ever cover, and that never matched the (already accent-free) Ghost slug.
|
||||||
|
No-op on EN, whose only non-ASCII are dashes and curly apostrophes."""
|
||||||
|
return unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode()
|
||||||
|
|
||||||
|
|
||||||
def _tokens(text):
|
def _tokens(text):
|
||||||
return set(re.findall(r"[a-z0-9]+", _s(text).lower()))
|
return set(re.findall(r"[a-z0-9]+", _deaccent(_s(text)).lower()))
|
||||||
|
|
||||||
|
|
||||||
def _case_years(title, slug):
|
def _case_years(title, slug):
|
||||||
|
|||||||
Reference in New Issue
Block a user