feat(short): generación y render de Shorts vía shortsmith
Build & Deploy ResearchOwl / build-and-push (push) Successful in 9s

Añade /generate short_en y /short_spec. El pipeline genera un shot spec
con Haiku, verifica cada cifra, fecha y cita contra los chunks de la
sesión, lo renderiza en shortsmith y entrega el MP4 por Telegram junto
a un informe de claims.

- ShortsmithClient con sondeo y fallback al spec JSON si el render falla
- Contrato de plantillas obtenido de GET /templates, no codificado
- Comprobación de fundamento determinista, sin LLM
- outputs.published_url para enlazar el artículo de Ghost
- Normalización de comillas rectas a tipográficas (ver KNOWN-ISSUES.md)

Lo que no aparece en los chunks se contrasta contra el ejemplo del
prompt: si casa ahí es fuga, no invención, y se informa como tal. El
purgado de sesiones se lleva también su MP4.

La subida a YouTube queda fuera a propósito: fase 3.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
ChemaVX
2026-08-01 21:55:42 +00:00
co-authored by Claude Opus 5
parent 8b81ef87e4
commit 20c8d03aa7
27 changed files with 4350 additions and 23 deletions
+64
View File
@@ -0,0 +1,64 @@
"""El mensaje de revisión del Short.
Es la puerta humana: si este mensaje no sale, o sale sin los avisos, se está
publicando lo que el modelo recuerde en vez de lo que dicen las fuentes. Por eso
tiene test propio aparte del pipeline.
"""
import json
from src.bot.bot import _claims_message
from src.generator.grounding import check_grounding
from src.generator.short import ShortResult
SPEC = {
"version": 1,
"meta": {"id": "x", "title": "X"},
"shots": [{"template": "scale_bars", "duration": 30.0, "props": {
"headline": "REPORTED SCALE",
"bars": [{"label": "BOEING 747", "value": 232, "unit": "FT"}]}}],
}
CHUNKS = [{"content": "A Boeing 747 is 232 ft long.", "url": "https://a.test/1"}]
def result_with(**kw):
base = dict(topic="Caso X", spec=SPEC, title="X", attempts=1,
cost_usd=0.0042, duration_s=30.0,
article_url="https://www.theexclusionzone.com/caso-x/",
grounding=check_grounding(SPEC, CHUNKS))
base.update(kw)
return ShortResult(**base)
def test_a_clean_report_still_says_so():
"""Un éxito silencioso enseña al lector a dejar de mirar."""
text = _claims_message(result_with())
assert "0 sin encontrar" in text
assert "1 chunks de 1 URLs" in text
assert "Coste: $0.0042" in text
def test_ungrounded_claims_are_listed_one_by_one():
invented = json.loads(json.dumps(SPEC))
invented["shots"][0]["props"]["headline"] = "41,000 FT"
text = _claims_message(result_with(spec=invented,
grounding=check_grounding(invented, CHUNKS)))
assert "1 sin encontrar" in text
assert "41,000 FT" in text
def test_a_session_without_an_article_url_says_what_to_run():
text = _claims_message(result_with(article_url=None))
assert "/generate blog en" in text
def test_render_warnings_reach_the_human():
text = _claims_message(result_with(render_warnings=[
{"template": "data_card", "text": "UNA FILA DEMASIADO LARGA",
"requested": 44, "size": 38}]))
assert "recortados" in text and "data_card" in text
def test_there_is_a_report_even_when_there_was_no_spec():
text = _claims_message(ShortResult(topic="Caso X"))
assert "Sin comprobación de fundamento" in text
assert "Coste:" in text
+288
View File
@@ -0,0 +1,288 @@
"""El comprobador de fundamento, contra el spec de referencia y contra copias
deliberadamente corrompidas.
Los chunks de abajo son material de fuente sintético pero escrito como escribe
una fuente real: fechas en otro orden que el spec, unidades con la palabra
entera, comillas tipográficas, números con separador de millares. Si el
comprobador sólo supiera comparar cadenas idénticas, este fichero lo delataría.
"""
import copy
import json
from pathlib import Path
import pytest
from src.generator.grounding import (
check_grounding, extract_claims, normalize,
)
EXAMPLE = Path(__file__).resolve().parents[1] / "src/generator/examples/jal1628.json"
@pytest.fixture
def spec():
return json.loads(EXAMPLE.read_text())
#: Cada chunk imita una fuente distinta. Entre los cuatro está TODO lo que
#: afirma examples/jal1628.json, pero casi nunca con las mismas palabras.
CHUNKS = [
{
"url": "https://www.faa.gov/foia/jal1628",
"content": (
"On November 17, 1986, Japan Air Lines flight JAL 1628, a Boeing 747 "
"cargo aircraft, was cruising at 35,000 feet and roughly 600 mph over "
"Alaska, en route from Fort Yukon toward Anchorage by way of Fairbanks "
"and Talkeetna. The flight crew reported two lights pacing the aircraft."
),
},
{
"url": "https://example.org/terauchi-testimony",
"content": (
"The pilot in command was Captain Kenju Terauchi, an ex-fighter pilot "
"with the JASDF, 29 years of flying experience and more than 10,000 "
"flight hours. Terauchi described the object as “twice the size of an "
"aircraft carrier”, an estimate that would put it between 1,600 and "
"2,000 feet across — against the 232 ft length of his own Boeing 747. "
"The unidentified contact held its relative position through a full "
"360° turn and a descent of 4,000 ft."
),
},
{
"url": "https://example.org/radar-records",
"content": (
"Three independent sources logged the encounter. The onboard radar "
"showed a contact 78 nm out at the 10 o'clock position. Anchorage "
"Center recorded primary returns through the turns. The Elmendorf ROCC "
"tracked what it logged as a “flight of two”. Fairbanks radar showed "
"nothing at all."
),
},
{
"url": "https://example.org/faa-closing",
"content": (
"The FAA closed the case on 5 March 1987 with an official finding of a "
"“split radar image”. An AARTCC controller said such a split happened "
"“rarely, if ever” in that airspace. The FAA released roughly 1,500 "
"pages of documentation. Forty years on — 40 years — the file is still "
"open, and the estimated object has no accepted explanation."
),
},
]
# --- normalización ----------------------------------------------------------
def test_normalize_thousands_separators():
assert normalize("35,000 FT") == normalize("35000 ft") == "35000 ft"
assert normalize("1.500 paginas") == normalize("1,500 paginas") == "1500 paginas"
# No toca los decimales de verdad: 61.22 es una latitud, no 6122.
assert "61.22" in normalize("61.22")
assert "1.5" in normalize("1.5")
def test_normalize_quote_glyphs_and_dashes():
assert normalize("“FLIGHT OF TWO”") == normalize('"flight of two"')
assert normalize("1,600 2,000") == normalize("1600 - 2000")
assert normalize("4,000") == normalize("-4000")
assert normalize("CONTACT 78 NM · 10 OCLOCK") == "contact 7-8 nm 10 o'clock"
def test_normalize_is_idempotent():
once = normalize("“~1,600 2,000 FT”")
assert normalize(once) == once
# --- extracción -------------------------------------------------------------
def test_extracts_quotes_figures_dates_and_names(spec):
claims = extract_claims(spec)
by_kind = {}
for c in claims:
by_kind.setdefault(c.kind, set()).add(c.text)
assert "TWICE THE SIZE OF AN AIRCRAFT CARRIER" in by_kind["quote"]
assert "SPLIT RADAR IMAGE" in by_kind["quote"]
assert "35,000 FT" in by_kind["figure"]
assert "1500" in by_kind["figure"] # count_to, que sí afirma un dato
assert "17 NOV 1986" in by_kind["date"]
assert "5 MARCH 1987" in by_kind["date"]
assert "CAPT. KENJU TERAUCHI" in by_kind["name"]
assert "ELMENDORF ROCC" in by_kind["name"]
def test_geometry_is_not_a_claim(spec):
"""Latitudes, duraciones, barridos y grados de giro son parámetros de dibujo:
no dicen nada sobre el mundo y no se comprueban."""
paths = " ".join(c.path for c in extract_claims(spec))
for geometry in (".lat", ".lon", ".duration", ".sweeps",
".contact_bearing_deg", ".markers", ".bounds"):
assert geometry not in paths
def test_a_split_quote_is_one_claim(spec):
"""scale_bars.quote son las líneas de UNA cita: se comprueba entera, no a
trozos (el renderizador no envuelve; el caller parte las líneas)."""
quotes = [c.text for c in extract_claims(spec) if c.kind == "quote"]
assert "TWICE THE SIZE OF AN AIRCRAFT CARRIER" in quotes
assert "TWICE THE SIZE OF" not in quotes
# --- comprobación -----------------------------------------------------------
def test_reference_spec_is_fully_grounded(spec):
report = check_grounding(spec, CHUNKS)
assert report.ungrounded == [], \
"sin fundamento: " + "; ".join(f"[{c.kind}] {c.text}" for c in report.ungrounded)
assert report.clean
assert report.total > 25
assert report.chunk_count == 4 and report.url_count == 4
def test_an_injected_figure_is_flagged_and_nothing_else(spec):
corrupted = copy.deepcopy(spec)
corrupted["shots"][7]["props"]["count_to"] = 12000 # eran 1.500 páginas
corrupted["shots"][1]["props"]["subline"] = "41,000 FT · 600 MPH"
report = check_grounding(corrupted, CHUNKS)
flagged = {c.text for c in report.ungrounded}
assert flagged == {"12000", "41,000 FT"}
def test_an_invented_quote_is_flagged(spec):
corrupted = copy.deepcopy(spec)
corrupted["shots"][6]["props"]["quote_a"] = "“RADAR MALFUNCTION”"
report = check_grounding(corrupted, CHUNKS)
assert [c.text for c in report.ungrounded] == ["RADAR MALFUNCTION"]
assert report.ungrounded[0].kind == "quote"
assert report.ungrounded[0].path.startswith("shots.6.document_quote.props.quote_a")
def test_an_invented_agency_is_flagged(spec):
corrupted = copy.deepcopy(spec)
corrupted["shots"][5]["props"]["strips"][2]["label"] = "NORAD CHEYENNE"
report = check_grounding(corrupted, CHUNKS)
assert [c.text for c in report.ungrounded] == ["NORAD CHEYENNE"]
def test_an_invented_date_is_flagged(spec):
corrupted = copy.deepcopy(spec)
corrupted["shots"][1]["props"]["headline"] = "17 NOV 1987"
report = check_grounding(corrupted, CHUNKS)
assert [c.text for c in report.ungrounded] == ["17 NOV 1987"]
def test_dates_match_across_formats(spec):
"""El spec escribe "17 NOV 1986" y la fuente "November 17, 1986". Es la misma
fecha y el comprobador no debe gastarle un aviso al humano."""
report = check_grounding(spec, [CHUNKS[0]])
assert "17 NOV 1986" not in {c.text for c in report.ungrounded}
def test_units_match_their_spelled_out_form(spec):
""""35,000 FT" contra "35,000 feet"."""
report = check_grounding(spec, [CHUNKS[0]])
assert "35,000 FT" not in {c.text for c in report.ungrounded}
def test_a_number_alone_is_not_enough_without_its_unit():
"""1.500 aparece en las fuentes como páginas; 1.500 FT no lo dice nadie."""
spec = {
"version": 1,
"meta": {"id": "x", "title": "x"},
"shots": [{"template": "scale_bars", "duration": 5.0, "props": {
"headline": "H",
"bars": [{"label": "ESTIMATED OBJECT", "value": 1500, "unit": "FT"}]}}],
}
report = check_grounding(spec, [CHUNKS[3]])
assert [c.text for c in report.ungrounded] == ["1500 FT"]
def test_no_chunks_means_nothing_is_supported(spec):
"""Sin material no se apoya nada. Aquí todo cae en `contaminated` porque el
spec de prueba ES el ejemplo del prompt — que es justo el diagnóstico
correcto: ninguna de esas cifras viene de la sesión."""
report = check_grounding(spec, [])
assert report.grounded == []
assert report.unsupported
assert not report.clean
assert report.chunk_count == 0 and report.url_count == 0
# --- fuga del ejemplo del prompt --------------------------------------------
def test_a_figure_copied_from_the_prompt_example_is_diagnosed_as_such(spec):
"""El caso real, medido el 2026-08-01 contra la sesión 153: el modelo
escribió "232 FT" (el largo de un 747) y eso no estaba en ninguno de los 126
chunks — venía del ejemplo del prompt. No es una invención, es una fuga, y
se arregla borrándola, no verificándola."""
sources_without_the_747 = [c for c in CHUNKS if "232" not in c["content"]]
report = check_grounding(spec, sources_without_the_747)
assert "232 FT" in {c.text for c in report.contaminated}
assert "232 FT" not in {c.text for c in report.ungrounded}
def test_an_invention_is_not_confused_with_a_leak(spec):
"""Una cifra que no está ni en las fuentes ni en el ejemplo sigue siendo
una invención."""
corrupted = copy.deepcopy(spec)
corrupted["shots"][1]["props"]["subline"] = "41,000 FT · 600 MPH"
report = check_grounding(corrupted, CHUNKS)
assert [c.text for c in report.ungrounded] == ["41,000 FT"]
assert report.contaminated == []
def test_the_session_wins_over_the_example(spec):
"""Si el dato SÍ está en las fuentes, está fundamentado y punto: que además
aparezca en el ejemplo no lo ensucia."""
report = check_grounding(spec, CHUNKS)
assert report.contaminated == []
assert report.clean
def test_a_leak_shows_up_in_the_report_with_its_own_wording(spec):
report = check_grounding(spec, [c for c in CHUNKS if "232" not in c["content"]])
summary = report.summary()
assert "copiados del EJEMPLO" in summary
assert "232 FT" in summary
assert "fuga, no invención" in summary
def test_both_diagnoses_count_as_unsupported(spec):
corrupted = copy.deepcopy(spec)
corrupted["shots"][1]["props"]["subline"] = "41,000 FT · 600 MPH"
report = check_grounding(corrupted, [c for c in CHUNKS if "232" not in c["content"]])
assert len(report.unsupported) == len(report.ungrounded) + len(report.contaminated)
assert report.total == len(report.grounded) + len(report.unsupported)
assert not report.clean
def test_a_missing_example_file_degrades_to_the_old_behaviour(spec):
"""El contraste con el ejemplo es un diagnóstico extra, no un requisito: sin
fichero, todo lo no encontrado vuelve a ser simplemente 'sin encontrar'."""
report = check_grounding(spec, [], example_haystacks=())
assert report.contaminated == []
assert report.ungrounded
def test_summary_reports_success_out_loud(spec):
report = check_grounding(spec, CHUNKS)
summary = report.summary()
assert "0 sin encontrar" in summary # el éxito NO es silencioso
assert "4 chunks de 4 URLs" in summary
def test_summary_lists_every_ungrounded_string(spec):
corrupted = copy.deepcopy(spec)
corrupted["shots"][7]["props"]["count_to"] = 12000
summary = check_grounding(corrupted, CHUNKS).summary()
assert "⚠️ 1 sin encontrar" in summary
assert '"12000"' in summary
+35
View File
@@ -0,0 +1,35 @@
"""Separación de capas.
`bot/` puede importar de todo; nadie puede importar de `bot/`. El progreso y
los callbacks viajan como callables genéricos justo para no necesitarlo.
"""
import re
from pathlib import Path
SRC = Path(__file__).resolve().parents[1] / "src"
LOWER_LAYERS = ("generator", "scraper", "processor", "db", "seo", "news")
IMPORTS_BOT = re.compile(r"^\s*(from\s+src\.bot|import\s+src\.bot|from\s+\.\.bot)",
re.MULTILINE)
def test_no_lower_layer_imports_from_bot():
offenders = []
for layer in LOWER_LAYERS:
for path in (SRC / layer).rglob("*.py"):
if IMPORTS_BOT.search(path.read_text(encoding="utf-8")):
offenders.append(str(path.relative_to(SRC.parent)))
assert not offenders, f"importan de bot/: {offenders}"
IMPORTS_TELEGRAM = re.compile(r"^\s*(from\s+telegram|import\s+telegram)", re.MULTILINE)
def test_the_short_pipeline_takes_progress_as_a_plain_callable():
"""La comprobación concreta para lo añadido en fase 2: si algún día alguien
mete un `Message` de Telegram aquí, este test lo dice. Nombrar Telegram en
un comentario vale — importarlo, no."""
for module in ("short.py", "shortsmith.py", "shortspec.py", "grounding.py",
"spec_contract.py"):
source = (SRC / "generator" / module).read_text(encoding="utf-8")
assert not IMPORTS_TELEGRAM.search(source), f"{module} importa telegram"
+134
View File
@@ -0,0 +1,134 @@
"""Eval dorada: ¿podría este pipeline haber producido el vídeo que ya sabemos
que está bien?
Corre el generador entero contra una sesión REAL de investigación y compara el
spec resultante con `examples/jal1628.json` — que es el que produjo el primer
Short — de forma ESTRUCTURAL: número de shots, plantillas elegidas, duración
total y claims sin fundamento. Nunca por igualdad de cadenas: el modelo
redactará distinto y eso no es un fallo.
Necesita una sesión de verdad, así que se salta salvo que se le dé todo:
RESEARCHOWL_GOLDEN_DB=/ruta/a/researchowl.db \\
RESEARCHOWL_GOLDEN_SESSION=153 \\
SHORTSMITH_LIVE_URL=http://10.43.86.57:8080 \\
ANTHROPIC_API_KEY=... \\
pytest tests/test_short_golden.py -v -s
Para sacar la sesión del cluster sin arrastrar la DB entera, `make golden-db`.
"""
import json
import os
from pathlib import Path
import pytest
EXAMPLE = Path(__file__).resolve().parents[1] / "src/generator/examples/jal1628.json"
GOLDEN_DB = os.environ.get("RESEARCHOWL_GOLDEN_DB")
GOLDEN_SESSION = os.environ.get("RESEARCHOWL_GOLDEN_SESSION")
LIVE_URL = os.environ.get("SHORTSMITH_LIVE_URL")
pytestmark = pytest.mark.skipif(
not (GOLDEN_DB and GOLDEN_SESSION and LIVE_URL and os.environ.get("ANTHROPIC_API_KEY")),
reason="la eval dorada necesita una sesión real, shortsmith vivo y clave de Claude")
def structure(spec: dict) -> dict:
"""Lo comparable de un spec: forma, no palabras."""
shots = spec.get("shots", [])
return {
"shots": len(shots),
"templates": [s["template"] for s in shots],
"duration": sum(s["duration"] for s in shots),
"distinct_templates": len({s["template"] for s in shots}),
}
@pytest.fixture(scope="module")
def produced():
"""Genera UNA vez (cuesta dinero) y reparte el resultado a los tests."""
import asyncio
from src.config import settings
settings.db_path = GOLDEN_DB
settings.shortsmith_url = LIVE_URL
settings.shortsmith_enabled = True
from src.db.database import ResearchDB, close_db, get_db
from src.generator.short import ShortProducer
from src.processor.processor import ContentProcessor, OllamaClient
async def run():
conn = await get_db()
try:
db = ResearchDB(conn)
producer = ShortProducer(db, ContentProcessor(db, OllamaClient()))
return await producer.produce(int(GOLDEN_SESSION),
lambda text: print(" ", text))
finally:
await close_db()
return asyncio.run(run())
def test_the_pipeline_produces_a_renderable_short(produced):
assert produced.spec is not None, produced.failure
assert produced.failure is None, produced.failure
assert produced.has_video
assert Path(produced.video_path).stat().st_size > 100_000
def test_the_shape_matches_the_reference(produced):
reference = structure(json.loads(EXAMPLE.read_text()))
got = structure(produced.spec)
print(f"\nreferencia: {reference}\nobtenido: {got}")
# El vídeo bueno son 8 shots; ±3 sigue siendo la misma forma narrativa.
assert abs(got["shots"] - reference["shots"]) <= 3
assert got["distinct_templates"] >= 4, "un Short de una sola plantilla es un cartel"
assert 20.0 <= got["duration"] <= 45.0
# Un case_file abre con contexto y cierra con el contador: no se exige la
# misma lista de plantillas, sí que el cierre sea un cierre.
assert got["templates"][-1] == reference["templates"][-1]
def test_no_claim_was_invented(produced):
"""Lo que de verdad decide si esto se puede publicar.
Medido el 2026-08-01 contra la sesión 153, en dos tiradas: 36-37 claims de
38 casan. Lo que se escapa es de dos clases conocidas y ninguna se arregla
endureciendo este assert:
* "232 FT" (el largo de un 747) copiado del ejemplo de la sección 5, que no
está en NINGUNO de los 126 chunks de la sesión — el comprobador lo
etiqueta ya como fuga del ejemplo, no como invención;
* una cita comprimida — "WALNUT SHAPED WIDE RIM" donde la fuente dice
"walnut shaped with a wide rim around its circumference".
El prompt ataca las dos, pero el muestreo del modelo varía entre tiradas, y
un test que gasta $0.05 y depende del muestreo no sirve de puerta. **La
puerta de verdad es el informe de claims en Telegram**, que se manda
siempre. Esto sólo vigila que el fundamento no se desplome.
"""
report = produced.grounding
print("\n" + report.summary())
assert len(report.grounded) >= 30, "el spec dejó de apoyarse en las fuentes"
assert len(report.unsupported) <= 3, \
"sin fundamento: " + "; ".join(f"[{c.kind}] {c.text}" for c in report.unsupported)
def test_it_did_not_take_many_attempts(produced):
"""Métrica del §4: si esto sube de 1.5 de media, lo que hay que arreglar es
el prompt, no el número de reintentos."""
print(f"\nintentos hasta válido: {produced.attempts}; coste ${produced.cost_usd:.4f}")
assert produced.attempts <= 2
def test_it_costs_what_a_blog_costs(produced):
"""Medido, no estimado: 40 chunks de contexto son ~25k tokens de entrada, y
un reintento los paga otra vez. El §9 del spec calculaba $0.003-0.008 con un
contexto mucho más corto; con este, dos intentos salen por ~$0.05. Sigue
siendo lo que cuesta un /generate blog, que era el punto."""
assert produced.cost_usd < 0.08
+326
View File
@@ -0,0 +1,326 @@
"""El pipeline del Short: orden de los pasos y fallbacks.
Todo con dobles: ni Claude ni shortsmith ni SQLite. Lo que se comprueba aquí es
que el fundamento se mira ANTES de renderizar y que ningún camino de fallo se
come el spec.
"""
import json
import pytest
from src.config import settings
from src.generator.short import ShortProducer, ShortResult, ShortsDisabled
from src.generator.shortsmith import JobResult, ShortsmithError, ShortsmithUnavailable
from tests.test_spec_contract import TEMPLATES
SPEC = {
"version": 1,
"meta": {"id": "jal1628", "title": "JAL 1628"},
"shots": [
{"template": "radar_sweep", "duration": 15.0, "props": {"headline": "3 RADARS"}},
{"template": "scale_bars", "duration": 15.0, "props": {
"headline": "REPORTED SCALE",
"bars": [{"label": "BOEING 747", "value": 232, "unit": "FT"}]}},
],
}
CHUNKS = [{
"content": "Three radars tracked the object. A Boeing 747 is 232 ft long.",
"url": "https://faa.example/jal1628",
"title": "FAA file",
"source_type": "web",
}]
class FakeDB:
def __init__(self, article_url=None):
self.article_url = article_url
self.saved: list[tuple] = []
async def get_session(self, session_id):
return {"id": session_id, "topic": "JAL 1628 Alaska 1986"}
async def get_article_url(self, session_id):
return self.article_url
async def save_output(self, session_id, output_type, content):
self.saved.append((session_id, output_type, content))
return len(self.saved)
async def log_api_call(self, *a, **kw):
return None
class FakeProcessor:
def __init__(self, chunks=None):
self.chunks = CHUNKS if chunks is None else chunks
async def rag_chunks(self, session_id, query, top_k=20):
return self.chunks
class FakeClient:
"""shortsmith de mentira. `fail_at` decide dónde se rompe."""
def __init__(self, fail_at=None, job_status="done", warnings=None):
self.fail_at = fail_at
self.job_status = job_status
self.warnings = warnings or []
self.rendered = None
async def templates(self, refresh=False):
if self.fail_at == "templates":
raise ShortsmithUnavailable("no hay nadie al otro lado")
return TEMPLATES
async def render(self, spec):
if self.fail_at == "render":
raise ShortsmithUnavailable("conexión rechazada")
self.rendered = spec
return "job-1"
async def poll(self, job_id, on_progress=None, **kw):
if self.fail_at == "poll":
raise ShortsmithError("job atascado")
if on_progress:
await on_progress(0.5, "running")
return JobResult(job_id, self.job_status, 1.0, self.warnings,
"OOMKilled" if self.job_status == "error" else None)
async def fetch_video(self, job_id):
if self.fail_at == "fetch":
raise ShortsmithError("404 del vídeo")
return b"\x00\x00\x00 ftypisom" + b"\x00" * 2048
def llm_returning(*responses):
queue = list(responses)
async def call(system, prompt):
call.prompts.append(prompt)
return queue.pop(0) if len(queue) > 1 else queue[0]
call.prompts = []
return call
def producer(tmp_path, monkeypatch, *, client=None, llm=None, db=None, processor=None):
monkeypatch.setattr(settings, "shorts_dir", str(tmp_path / "shorts"))
monkeypatch.setattr(settings, "shortsmith_enabled", True)
return ShortProducer(
db or FakeDB(),
processor or FakeProcessor(),
client=client or FakeClient(),
llm_call=llm or llm_returning(json.dumps(SPEC)),
)
# --- camino feliz -----------------------------------------------------------
@pytest.mark.asyncio
async def test_happy_path_writes_the_mp4_to_disk(tmp_path, monkeypatch):
db = FakeDB(article_url="https://www.theexclusionzone.com/jal-1628/")
p = producer(tmp_path, monkeypatch, db=db)
result = await p.produce(153)
assert result.has_video
assert result.video_path.endswith("153.mp4")
assert open(result.video_path, "rb").read()[:12].endswith(b"ftypisom")
assert result.title == "JAL 1628"
assert result.duration_s == 30.0
assert result.article_url.endswith("/jal-1628/")
assert result.failure is None
@pytest.mark.asyncio
async def test_the_spec_is_saved_before_the_render(tmp_path, monkeypatch):
"""Si el render se cae, la parte cara ya está en la DB y /short_spec la
devuelve."""
db = FakeDB()
p = producer(tmp_path, monkeypatch, db=db, client=FakeClient(fail_at="render"))
result = await p.produce(153)
assert db.saved and db.saved[0][1] == "short_en"
assert json.loads(db.saved[0][2])["meta"]["id"] == "jal1628"
assert not result.has_video
@pytest.mark.asyncio
async def test_grounding_runs_before_rendering(tmp_path, monkeypatch):
"""El informe existe aunque el render no llegue a empezar: ese es el orden
del §12 y es lo que hace que la revisión humana llegue igual."""
p = producer(tmp_path, monkeypatch, client=FakeClient(fail_at="render"))
result = await p.produce(153)
assert result.grounding is not None
assert result.grounding.total > 0
@pytest.mark.asyncio
async def test_ungrounded_claims_do_not_block_the_render(tmp_path, monkeypatch):
"""Un dato sin encontrar puede ser una fabricación o un artefacto de
formato. Lo decide una persona: el vídeo se entrega con el aviso al lado."""
invented = json.loads(json.dumps(SPEC))
invented["shots"][0]["props"]["headline"] = "41,000 FT"
p = producer(tmp_path, monkeypatch, llm=llm_returning(json.dumps(invented)))
result = await p.produce(153)
assert result.has_video
assert [c.text for c in result.grounding.ungrounded] == ["41,000 FT"]
@pytest.mark.asyncio
async def test_the_article_url_reaches_the_prompt(tmp_path, monkeypatch):
llm = llm_returning(json.dumps(SPEC))
p = producer(tmp_path, monkeypatch,
db=FakeDB(article_url="https://www.theexclusionzone.com/jal-1628/"),
llm=llm)
await p.produce(153)
assert "https://www.theexclusionzone.com/jal-1628/" in llm.prompts[0]
# --- fallbacks --------------------------------------------------------------
@pytest.mark.asyncio
@pytest.mark.parametrize("fail_at", ["render", "poll", "fetch"])
async def test_every_render_failure_still_returns_the_spec(tmp_path, monkeypatch, fail_at):
p = producer(tmp_path, monkeypatch, client=FakeClient(fail_at=fail_at))
result = await p.produce(153)
assert not result.has_video
assert result.failure
assert json.loads(result.spec_json)["meta"]["id"] == "jal1628"
@pytest.mark.asyncio
async def test_a_job_that_errors_is_reported_with_its_reason(tmp_path, monkeypatch):
p = producer(tmp_path, monkeypatch, client=FakeClient(job_status="error"))
result = await p.produce(153)
assert not result.has_video
assert "OOMKilled" in result.failure
@pytest.mark.asyncio
async def test_an_unwritable_spec_still_returns_the_last_attempt(tmp_path, monkeypatch):
"""Tres intentos fallidos no son motivo para tirar la generación."""
broken = json.loads(json.dumps(SPEC))
broken["meta"]["id"] = "MAYÚSCULAS Y ESPACIOS"
p = producer(tmp_path, monkeypatch, llm=llm_returning(json.dumps(broken)))
result = await p.produce(153)
assert not result.has_video
assert result.attempts == 3
assert result.spec["meta"]["id"] == "MAYÚSCULAS Y ESPACIOS"
assert "no pasó la validación" in result.failure
assert result.grounding is None # no hay spec válido que comprobar
@pytest.mark.asyncio
async def test_a_response_that_is_not_json_at_all_comes_back_raw(tmp_path, monkeypatch):
p = producer(tmp_path, monkeypatch,
llm=llm_returning("Lo siento, no puedo ayudarte con eso."))
result = await p.produce(153)
assert result.spec is None
assert "Lo siento" in result.raw_response
@pytest.mark.asyncio
async def test_render_warnings_travel_with_the_result(tmp_path, monkeypatch):
warnings = [{"template": "data_card", "text": "UNA FILA MUY LARGA",
"requested": 44, "size": 38}]
p = producer(tmp_path, monkeypatch, client=FakeClient(warnings=warnings))
result = await p.produce(153)
assert result.has_video
assert result.render_warnings[0]["template"] == "data_card"
# --- interruptores y precondiciones -----------------------------------------
@pytest.mark.asyncio
async def test_the_kill_switch_says_so_instead_of_crashing(tmp_path, monkeypatch):
p = producer(tmp_path, monkeypatch)
monkeypatch.setattr(settings, "shortsmith_enabled", False)
with pytest.raises(ShortsDisabled):
await p.produce(153)
@pytest.mark.asyncio
async def test_an_unreachable_renderer_is_named_clearly(tmp_path, monkeypatch):
"""Sin contrato no hay prompt que escribir: aquí no hay fallback posible y
el mensaje lo dice."""
p = producer(tmp_path, monkeypatch, client=FakeClient(fail_at="templates"))
with pytest.raises(ShortsmithUnavailable):
await p.produce(153)
@pytest.mark.asyncio
async def test_a_session_without_chunks_says_what_to_run(tmp_path, monkeypatch):
p = producer(tmp_path, monkeypatch, processor=FakeProcessor(chunks=[]))
with pytest.raises(ValueError, match="/process"):
await p.produce(153)
def test_domain_is_drawn_without_protocol_or_www(monkeypatch):
monkeypatch.setattr(settings, "ghost_url_en", "https://www.theexclusionzone.com")
assert ShortProducer(FakeDB(), FakeProcessor(), client=FakeClient())._domain() \
== "THEEXCLUSIONZONE.COM"
def test_short_result_without_a_spec_has_an_empty_json():
assert ShortResult(topic="x").spec_json == ""
# --- limpieza ---------------------------------------------------------------
@pytest.mark.asyncio
async def test_purging_a_session_takes_its_video_with_it(tmp_path, monkeypatch):
"""Un MP4 huérfano en el PVC es negligible contra 5 Gi y es arqueología
dentro de un año."""
import time
import aiosqlite
from src.db import database
from src.db.database import ResearchDB
shorts = tmp_path / "shorts"
shorts.mkdir()
(shorts / "1.mp4").write_bytes(b"viejo")
(shorts / "2.mp4").write_bytes(b"reciente")
monkeypatch.setattr(settings, "shorts_dir", str(shorts))
conn = await aiosqlite.connect(tmp_path / "t.db")
conn.row_factory = aiosqlite.Row
await conn.executescript(database.SCHEMA)
old, now = time.time() - 90 * 86400, time.time()
await conn.execute(
"INSERT INTO research_sessions (id, topic, status, telegram_chat_id,"
" created_at, updated_at) VALUES (1,'viejo','saturated',1,?,?)", (old, old))
await conn.execute(
"INSERT INTO research_sessions (id, topic, status, telegram_chat_id,"
" created_at, updated_at) VALUES (2,'nuevo','saturated',1,?,?)", (now, now))
await conn.commit()
counts = await ResearchDB(conn).purge_old_sessions(30)
await conn.close()
assert counts["shorts"] == 1
assert not (shorts / "1.mp4").exists()
assert (shorts / "2.mp4").exists(), "la sesión reciente conserva su vídeo"
+208
View File
@@ -0,0 +1,208 @@
"""ShortsmithClient — bucle de sondeo, errores y fallbacks.
Todo con un servidor falso; el test contra el servicio vivo es
`test_shortsmith_live.py`, que se salta salvo que se le apunte a uno.
"""
import asyncio
import json
from pathlib import Path
import pytest
from src.generator.shortsmith import (
JobResult, ShortsmithClient, ShortsmithError, ShortsmithRejected,
ShortsmithUnavailable, _templates_cache,
)
EXAMPLE = Path(__file__).resolve().parents[1] / "src/generator/examples/jal1628.json"
@pytest.fixture
def spec():
return json.loads(EXAMPLE.read_text())
class FakeResp:
def __init__(self, status, payload=None, body="", raw=b""):
self.status = status
self._payload = payload
self._body = body
self._raw = raw
async def __aenter__(self):
return self
async def __aexit__(self, *a):
return False
async def json(self):
if self._payload is None:
raise ValueError("no json")
return self._payload
async def text(self):
return self._body
async def read(self):
return self._raw
class FakeSession:
"""Sustituye a aiohttp.ClientSession: sirve respuestas de una cola por ruta."""
def __init__(self, routes):
self.routes = routes
self.calls = []
async def __aenter__(self):
return self
async def __aexit__(self, *a):
return False
def _next(self, method, url):
self.calls.append((method, url))
for pattern, responses in self.routes.items():
if pattern in url:
if isinstance(responses, list):
return responses.pop(0) if len(responses) > 1 else responses[0]
return responses
raise AssertionError(f"ruta no simulada: {method} {url}")
def get(self, url, **kw):
return self._next("GET", url)
def post(self, url, **kw):
return self._next("POST", url)
def patch_session(client, routes):
session = FakeSession(routes)
client._session = lambda total: session
return session
@pytest.mark.asyncio
async def test_templates_cached_per_process():
_templates_cache.clear()
client = ShortsmithClient("http://fake:8080")
session = patch_session(client, {"/templates": FakeResp(200, {"radar_sweep": {}})})
first = await client.templates()
second = await client.templates()
assert first == second == {"radar_sweep": {}}
assert len(session.calls) == 1, "la segunda llamada debe salir de la caché"
# refresh=True vuelve a pedirlo: el renderizador puede haberse actualizado.
patch_session(client, {"/templates": FakeResp(200, {"radar_sweep": {}, "nueva": {}})})
assert "nueva" in await client.templates(refresh=True)
_templates_cache.clear()
@pytest.mark.asyncio
async def test_render_returns_job_id(spec):
client = ShortsmithClient("http://fake:8080")
patch_session(client, {"/render": FakeResp(202, {"job_id": "abc123", "status": "queued"})})
assert await client.render(spec) == "abc123"
@pytest.mark.asyncio
async def test_render_422_propagates_error_paths(spec):
detail = [{
"type": "extra_forbidden",
"loc": ["shots", 0, "radar_sweep", "props", "sweeeps"],
"msg": "Extra inputs are not permitted",
}]
client = ShortsmithClient("http://fake:8080")
patch_session(client, {"/render": FakeResp(422, {"detail": detail})})
with pytest.raises(ShortsmithRejected) as exc:
await client.render(spec)
assert exc.value.errors[0]["loc"][-1] == "sweeeps"
@pytest.mark.asyncio
async def test_poll_queued_then_running_then_done():
client = ShortsmithClient("http://fake:8080")
patch_session(client, {"/jobs/": [
FakeResp(200, {"job_id": "j", "status": "queued", "progress": 0.0}),
FakeResp(200, {"job_id": "j", "status": "running", "progress": 0.4}),
FakeResp(200, {"job_id": "j", "status": "done", "progress": 1.0,
"warnings": [{"template": "data_card", "text": "x"}]}),
]})
seen = []
async def on_progress(fraction, status):
seen.append((fraction, status))
result = await client.poll("j", on_progress=on_progress, interval=0)
assert result.ok and result.status == "done"
assert result.warnings and result.warnings[0]["template"] == "data_card"
assert seen == [(0.0, "queued"), (0.4, "running"), (1.0, "done")]
@pytest.mark.asyncio
async def test_poll_returns_error_status_without_raising():
client = ShortsmithClient("http://fake:8080")
patch_session(client, {"/jobs/": FakeResp(200, {
"job_id": "j", "status": "error", "progress": 0.3,
"error": "interrupted by a restart: the process did not survive this render",
})})
result = await client.poll("j", interval=0)
assert not result.ok
assert "interrupted" in result.error
@pytest.mark.asyncio
async def test_poll_gives_up_on_a_stuck_job():
client = ShortsmithClient("http://fake:8080")
patch_session(client, {"/jobs/": FakeResp(200, {
"job_id": "j", "status": "running", "progress": 0.1})})
with pytest.raises(ShortsmithError, match="atascado"):
await client.poll("j", interval=0, ceiling=0)
@pytest.mark.asyncio
async def test_connection_refused_is_unavailable(spec):
import aiohttp
class Refusing(FakeSession):
def post(self, url, **kw):
raise aiohttp.ClientConnectionError(
"Cannot connect to host shortsmith-svc:8080 [Connection refused]")
client = ShortsmithClient("http://fake:8080")
client._session = lambda total: Refusing({})
with pytest.raises(ShortsmithUnavailable):
await client.render(spec)
@pytest.mark.asyncio
async def test_fetch_video_returns_bytes():
client = ShortsmithClient("http://fake:8080")
patch_session(client, {"/video": FakeResp(200, raw=b"\x00\x00\x00 ftypisom")})
assert (await client.fetch_video("j")).startswith(b"\x00\x00\x00 ftyp")
@pytest.mark.asyncio
async def test_progress_callback_failure_never_kills_the_render():
client = ShortsmithClient("http://fake:8080")
patch_session(client, {"/jobs/": FakeResp(200, {
"job_id": "j", "status": "done", "progress": 1.0})})
async def boom(fraction, status):
raise RuntimeError("Telegram dijo que no")
assert (await client.poll("j", on_progress=boom, interval=0)).ok
def test_jobresult_ok_only_when_done():
assert JobResult("j", "done").ok
assert not JobResult("j", "running").ok
assert not JobResult("j", "error", error="boom").ok
+68
View File
@@ -0,0 +1,68 @@
"""Prueba de fontanería contra un shortsmith VIVO.
Se salta salvo que se le dé una URL alcanzable desde donde corren los tests:
SHORTSMITH_LIVE_URL=http://10.43.86.57:8080 pytest tests/test_shortsmith_live.py -v
(dentro del cluster es `http://shortsmith-svc.shortsmith.svc.cluster.local:8080`;
desde el nodo, la ClusterIP de `kubectl get svc -n shortsmith`).
Renderiza el ejemplo de referencia entero — ~30 s de CPU en el pod — y comprueba
que vuelve un MP4. Es el paso 1 del §12 del spec de fase 2: probar el transporte
antes de generar nada.
"""
import json
import os
from pathlib import Path
import pytest
from src.generator.shortsmith import ShortsmithClient
LIVE_URL = os.environ.get("SHORTSMITH_LIVE_URL")
pytestmark = pytest.mark.skipif(
not LIVE_URL, reason="define SHORTSMITH_LIVE_URL para probar contra el servicio vivo")
EXAMPLE = Path(__file__).resolve().parents[1] / "src/generator/examples/jal1628.json"
@pytest.mark.asyncio
async def test_healthz_and_templates():
client = ShortsmithClient(LIVE_URL)
health = await client.health()
assert health["status"] == "ok"
assert health["templates"] >= 1
templates = await client.templates(refresh=True)
# No se comprueban nombres concretos a propósito: el contrato es de
# shortsmith y añadir plantillas allí no debe romper aquí.
assert templates, "GET /templates devolvió vacío"
for name, schema in templates.items():
assert schema.get("type") == "object", f"{name} no publica un esquema de objeto"
assert "properties" in schema
@pytest.mark.asyncio
async def test_render_the_reference_example_end_to_end(tmp_path):
client = ShortsmithClient(LIVE_URL)
spec = json.loads(EXAMPLE.read_text())
job_id = await client.render(spec)
seen = []
async def on_progress(fraction, status):
seen.append(fraction)
result = await client.poll(job_id, on_progress=on_progress)
assert result.ok, f"el job terminó en {result.status}: {result.error}"
assert seen and max(seen) == 1.0
video = await client.fetch_video(job_id)
# ftyp en los primeros bytes: es un MP4 de verdad, no una página de error.
assert b"ftyp" in video[:32]
assert len(video) > 100_000, f"solo {len(video)} bytes — sospechosamente corto"
out = tmp_path / "jal1628.mp4"
out.write_bytes(video)
print(f"\nrenderizado {len(video)/1e6:.2f} MB en {out}")
+233
View File
@@ -0,0 +1,233 @@
"""Escritura del shot spec: prompt, bucle de reintento y fallback.
El LLM entra como un callable, así que aquí se prueba el bucle, no a Haiku:
respuesta buena, respuesta malformada, typo en una prop, y las tres seguidas.
"""
import json
from pathlib import Path
import pytest
from src.generator.shortspec import (
MAX_ATTEMPTS, ShortSpecWriter, SpecWriteFailed, extract_json,
)
from tests.test_spec_contract import TEMPLATES
EXAMPLE = Path(__file__).resolve().parents[1] / "src/generator/examples/jal1628.json"
GOOD = {
"version": 1,
"meta": {"id": "caso", "title": "Un caso"},
"shots": [
{"template": "radar_sweep", "duration": 15.0, "props": {"headline": "3 RADARS"}},
{"template": "scale_bars", "duration": 15.0, "props": {
"headline": "REPORTED SCALE",
"bars": [{"label": "BOEING 747", "value": 232, "unit": "FT"}]}},
],
}
class FakeLLM:
"""Devuelve respuestas de una cola y guarda los prompts que recibió."""
def __init__(self, *responses):
self.responses = list(responses)
self.prompts: list[str] = []
self.systems: list[str] = []
async def __call__(self, system, prompt):
self.systems.append(system)
self.prompts.append(prompt)
return self.responses.pop(0) if len(self.responses) > 1 else self.responses[0]
def writer(*responses, **kw):
llm = FakeLLM(*responses)
return ShortSpecWriter(llm, TEMPLATES, **kw), llm
# --- parseo -----------------------------------------------------------------
def test_extract_json_survives_markdown_fences():
assert extract_json('```json\n{"a": 1}\n```') == {"a": 1}
assert extract_json('Here you go:\n{"a": 1}\nHope that helps') == {"a": 1}
assert extract_json('{"a": 1}') == {"a": 1}
def test_extract_json_complains_when_there_is_no_object():
with pytest.raises(ValueError):
extract_json("I'm afraid I can't do that")
def test_straight_quotes_inside_a_string_are_repaired():
"""Cómo falla esto en la vida real (sesión de Bélgica, 2026-08-01): el
modelo escribe la cita con comillas rectas, que cierran la cadena JSON antes
de tiempo. Se arregla aquí porque además es lo que se quiere dibujar."""
broken = '{"quote_a": ""CREDIBLE PEOPLE. THEY TOLD WHAT THEY SAW."", "n": 1}'
assert extract_json(broken) == {
"quote_a": "“CREDIBLE PEOPLE. THEY TOLD WHAT THEY SAW.”", "n": 1}
def test_the_repair_leaves_correct_json_alone():
good = {"a": 'texto con “tipográficas” dentro', "b": [1, 2], "c": {"d": "e"}}
assert extract_json(json.dumps(good, ensure_ascii=False)) == good
def test_the_repair_does_not_eat_escaped_quotes():
assert extract_json(r'{"a": "dijo \"hola\" y se fue"}') == {"a": 'dijo "hola" y se fue'}
def test_an_unrepairable_response_reports_the_offending_fragment():
"""El modelo no ve su salida numerada: "line 189 column 22" no le sirve; el
trozo sí."""
with pytest.raises(ValueError) as exc:
extract_json('{"a": 1, "b": [1, 2,,,], "c": 3}')
assert "aquí:" in str(exc.value)
# --- el prompt --------------------------------------------------------------
def test_prompt_carries_the_fetched_contract_not_a_copy():
w, _ = writer("{}")
prompt = w.build_prompt("Caso X", "material", None, "THEEXCLUSIONZONE.COM")
assert "radar_sweep:" in prompt and "contact_bearing_deg" in prompt
assert "1-3 elementos" in prompt # los límites de longitud, del esquema
assert "ink, amber, amber_dark" in prompt # la paleta, también del esquema
def test_prompt_states_the_editorial_constraints():
w, _ = writer("{}")
prompt = w.build_prompt("Caso X", "material", "https://x.test/post", "X.TEST")
assert "20-45 seconds" in prompt
assert "never hex" in prompt
assert "https://x.test/post" in prompt
assert "X.TEST" in prompt
assert "material" in prompt
def test_prompt_includes_the_worked_example_in_full():
w, _ = writer("{}")
prompt = w.build_prompt("Caso X", "material", None, "X.TEST")
example = json.loads(EXAMPLE.read_text())
assert example["meta"]["title"] in prompt
assert "counter_close" in prompt
def test_prompt_says_out_loud_that_there_is_no_article_yet():
w, _ = writer("{}")
assert "No article URL yet" in w.build_prompt("X", "m", None, "X.TEST")
# --- bucle ------------------------------------------------------------------
@pytest.mark.asyncio
async def test_a_good_response_validates_on_the_first_attempt():
w, llm = writer(json.dumps(GOOD))
result = await w.write("Caso X", "material")
assert result.attempts == 1
assert result.spec["meta"]["id"] == "caso"
assert len(llm.prompts) == 1
@pytest.mark.asyncio
async def test_malformed_json_triggers_a_retry():
w, llm = writer("no soy JSON", json.dumps(GOOD))
result = await w.write("Caso X", "material")
assert result.attempts == 2
assert "no es un objeto JSON válido" in result.history[0][0]
assert "previous attempt was rejected" in llm.prompts[1]
@pytest.mark.asyncio
async def test_the_exact_error_paths_are_fed_back_verbatim():
"""La ruta que devuelve la validación es lo más útil que se le puede dar al
modelo: se le pasa tal cual, sin parafrasear."""
bad = json.loads(json.dumps(GOOD))
bad["shots"][0]["props"]["sweeeps"] = 2
w, llm = writer(json.dumps(bad), json.dumps(GOOD))
result = await w.write("Caso X", "material")
assert result.attempts == 2
assert "shots.0.radar_sweep.props.sweeeps" in llm.prompts[1]
assert "sweeps" in llm.prompts[1] # y cuáles sí valen
@pytest.mark.asyncio
async def test_three_failures_raise_but_keep_the_last_attempt():
"""La parte cara es la generación, no el render: el último intento viaja en
la excepción para poder editarlo a mano y reenviarlo."""
bad = json.loads(json.dumps(GOOD))
bad["meta"]["id"] = "Caso Con Espacios"
w, _ = writer(json.dumps(bad))
with pytest.raises(SpecWriteFailed) as exc:
await w.write("Caso X", "material")
assert exc.value.attempts == MAX_ATTEMPTS
assert exc.value.last_spec["meta"]["id"] == "Caso Con Espacios"
assert any("meta.id" in e for e in exc.value.errors)
@pytest.mark.asyncio
async def test_an_off_target_duration_is_commented_once_then_accepted():
"""70 s cumple el contrato pero no el encargo: se comenta y, si el modelo
insiste, se renderiza igual antes que tirar la generación."""
long_spec = json.loads(json.dumps(GOOD))
long_spec["shots"][0]["duration"] = 55.0 # 70 s en total
w, llm = writer(json.dumps(long_spec))
result = await w.write("Caso X", "material")
assert result.attempts == MAX_ATTEMPTS
assert result.notes and "recorta" in result.notes[0]
assert "off-brief" in llm.prompts[1]
@pytest.mark.asyncio
async def test_a_valid_attempt_is_not_thrown_away_by_a_worse_one():
long_spec = json.loads(json.dumps(GOOD))
long_spec["shots"][0]["duration"] = 55.0
w, _ = writer(json.dumps(long_spec), "esto ya no es JSON", "tampoco")
result = await w.write("Caso X", "material")
assert result.spec["shots"][0]["duration"] == 55.0
assert result.notes
@pytest.mark.asyncio
async def test_the_contract_is_refetched_after_a_validation_failure():
"""Si el renderizador se actualizó a mitad de la run, la plantilla nueva
entra en el segundo intento."""
new_template = {"type": "object", "additionalProperties": False,
"required": ["title"],
"properties": {"title": {"type": "string", "minLength": 1}}}
refreshed = {**TEMPLATES, "holo_scan": new_template}
async def refresh():
return refreshed
with_new = json.loads(json.dumps(GOOD))
with_new["shots"][1] = {"template": "holo_scan", "duration": 15.0,
"props": {"title": "X"}}
w, llm = writer(json.dumps(with_new), json.dumps(with_new),
refresh_templates=refresh)
result = await w.write("Caso X", "material")
assert result.attempts == 2
assert "holo_scan" in llm.prompts[1]
assert result.spec["shots"][1]["template"] == "holo_scan"
@pytest.mark.asyncio
async def test_progress_is_reported_only_when_it_retries():
seen = []
async def on_progress(text):
seen.append(text)
w, _ = writer("no JSON", json.dumps(GOOD))
await w.write("Caso X", "material", on_progress=on_progress)
assert len(seen) == 1 and "attempt 2/3" in seen[0]
+231
View File
@@ -0,0 +1,231 @@
"""Validación local del spec contra el esquema publicado por shortsmith.
Los esquemas de abajo son una COPIA REDUCIDA de lo que devuelve
`GET /templates`, sólo para los tests: en producción se piden en vivo. Si
shortsmith cambia el contrato, quien lo nota es `test_shortsmith_live.py`, no
esto.
"""
import copy
import json
from pathlib import Path
import pytest
from src.generator.spec_contract import (
SpecInvalid, describe_templates, editorial_notes, validate_spec,
)
EXAMPLE = Path(__file__).resolve().parents[1] / "src/generator/examples/jal1628.json"
TEMPLATES = {
"radar_sweep": {
"type": "object", "additionalProperties": False,
"required": ["headline"],
"properties": {
"headline": {"type": "string", "minLength": 1},
"subline": {"type": "string", "default": ""},
"contact_bearing_deg": {"type": "number", "minimum": 0,
"exclusiveMaximum": 360, "default": 210.0},
"sweeps": {"type": "number", "exclusiveMinimum": 0, "maximum": 10,
"default": 2.0},
},
},
"scale_bars": {
"type": "object", "additionalProperties": False,
"required": ["headline", "bars"],
"$defs": {"Bar": {
"type": "object", "additionalProperties": False,
"required": ["label", "value"],
"properties": {
"label": {"type": "string", "minLength": 1},
"value": {"type": "number", "exclusiveMinimum": 0},
"unit": {"type": "string", "default": ""},
"color": {"enum": ["ink", "amber", "amber_dark", "muted", "dim", "red"],
"type": "string", "default": "ink"},
"value_label": {"type": "string", "default": ""},
},
}},
"properties": {
"headline": {"type": "string", "minLength": 1},
"bars": {"type": "array", "items": {"$ref": "#/$defs/Bar"},
"minItems": 1, "maxItems": 3},
"quote": {"type": "array", "items": {"type": "string"}, "maxItems": 2},
"attribution": {"type": "string", "default": ""},
},
},
}
def shot(template="radar_sweep", duration=6.0, **props):
base = {"radar_sweep": {"headline": "3 RADARS"},
"scale_bars": {"headline": "ESCALA",
"bars": [{"label": "BOEING 747", "value": 232}]}}[template]
return {"template": template, "duration": duration, "props": {**base, **props}}
def spec_with(*shots, **meta):
return {
"version": 1,
"meta": {"id": "caso", "title": "Un caso", **meta},
"shots": list(shots) or [shot()],
}
def errors_of(spec, templates=None):
with pytest.raises(SpecInvalid) as exc:
validate_spec(spec, templates if templates is not None else TEMPLATES)
return exc.value.errors
# --- lo que pasa ------------------------------------------------------------
def test_a_minimal_valid_spec_passes():
validate_spec(spec_with(shot(duration=25.0)), TEMPLATES)
def test_the_reference_example_passes_against_its_own_templates():
"""El ejemplo de referencia es válido; se comprueba con esquemas laxos para
las plantillas que este fichero no copia (lo estricto lo cubre el test vivo)."""
spec = json.loads(EXAMPLE.read_text())
permissive = {name: {"type": "object"} for name in
{s["template"] for s in spec["shots"]}}
permissive.update(TEMPLATES)
validate_spec(spec, permissive)
# --- rutas de error ---------------------------------------------------------
def test_unknown_prop_is_reported_with_its_full_path():
"""El typo en un nombre de prop es el bug más probable de un spec escrito
por un LLM, y la ruta exacta es lo que se le devuelve para arreglarlo."""
errors = errors_of(spec_with(shot(sweeeps=2)))
assert any(e.startswith("shots.0.radar_sweep.props.sweeeps: campo no permitido")
for e in errors), errors
assert "sweeps" in errors[0], "hay que decirle cuáles SÍ valen"
def test_unknown_template_lists_the_valid_names():
errors = errors_of(spec_with({"template": "radar_swep", "duration": 6.0,
"props": {"headline": "X"}}))
assert errors[0].startswith("shots.0.template:")
assert "radar_sweep" in errors[0] and "scale_bars" in errors[0]
def test_missing_required_prop():
bad = spec_with(shot()); del bad["shots"][0]["props"]["headline"]
assert "shots.0.radar_sweep.props.headline: falta y es obligatorio" in errors_of(bad)
def test_empty_string_where_a_non_empty_one_is_required():
assert any("shots.0.radar_sweep.props.headline" in e
for e in errors_of(spec_with(shot(headline=""))))
def test_numeric_bounds():
errors = errors_of(spec_with(shot(contact_bearing_deg=400)))
assert "shots.0.radar_sweep.props.contact_bearing_deg: 400 debe ser < 360" in errors
def test_list_length_limits_are_enforced():
bars = [{"label": f"B{i}", "value": i + 1} for i in range(4)]
errors = errors_of(spec_with(shot("scale_bars", bars=bars)))
assert "shots.0.scale_bars.props.bars: 4 elementos, el máximo es 3" in errors
def test_colour_must_be_a_palette_name_never_hex():
errors = errors_of(spec_with(shot("scale_bars", bars=[
{"label": "OBJETO", "value": 2000, "color": "#ffbf00"}])))
assert any("color" in e and "amber" in e for e in errors)
def test_nested_paths_survive_lists():
errors = errors_of(spec_with(shot("scale_bars", bars=[
{"label": "BOEING 747", "value": 232},
{"label": "OBJETO", "value": -5}])))
assert "shots.0.scale_bars.props.bars.1.value: -5 debe ser > 0" in errors
def test_every_error_comes_back_at_once():
"""Se devuelven todos: arreglar cinco de una vez sale más barato que cinco vueltas."""
errors = errors_of(spec_with(shot(headline="", sweeeps=1, contact_bearing_deg=999)))
assert len(errors) >= 3
# --- el sobre ---------------------------------------------------------------
def test_meta_id_pattern():
assert any(e.startswith("meta.id:") for e in errors_of(spec_with(id="Caso Roswell")))
def test_resolution_must_be_a_shorts_one():
assert any("no es una resolución admitida" in e
for e in errors_of(spec_with(shot(), width=800, height=600)))
def test_total_duration_ceiling_is_the_contract_not_the_target():
"""45 s es el objetivo editorial; 180 s es el límite duro. Pasarse de 45 no
invalida el spec eso es una nota, no un error."""
long_spec = spec_with(*[shot(duration=10.0) for _ in range(6)]) # 60 s
validate_spec(long_spec, TEMPLATES)
assert editorial_notes(long_spec)
too_long = spec_with(*[shot(duration=30.0) for _ in range(7)]) # 210 s
assert any("pasa del límite" in e for e in errors_of(too_long))
def test_total_duration_floor():
assert any("no llega al mínimo" in e
for e in errors_of(spec_with(shot(duration=2.0))))
def test_silence_window_cannot_run_past_the_end():
bad = spec_with(shot(duration=25.0))
bad["audio"] = {"preset": "sonar", "silence": [[20.0, 40.0]]}
assert any("se sale de la duración total" in e for e in errors_of(bad))
def test_extra_root_key_is_rejected():
bad = spec_with(shot(duration=25.0)); bad["narrative_shape"] = "case_file"
assert any(e.startswith("narrative_shape:") for e in errors_of(bad))
def test_editorial_notes_flag_both_ends():
assert "queda corto" in editorial_notes(spec_with(shot(duration=8.0)))[0]
assert "recorta" in editorial_notes(
spec_with(*[shot(duration=10.0) for _ in range(6)]))[0]
assert editorial_notes(spec_with(shot(duration=30.0))) == []
def test_a_spec_that_is_not_even_a_dict():
with pytest.raises(SpecInvalid):
validate_spec([1, 2, 3], TEMPLATES)
# --- descripción para el prompt ---------------------------------------------
def test_describe_templates_is_driven_by_what_the_service_publishes():
text = describe_templates(TEMPLATES)
assert "radar_sweep:" in text and "scale_bars:" in text
assert "headline: string, no vacío, OBLIGATORIO" in text
assert "1-3 elementos" in text # los límites llegan al prompt
assert "ink, amber, amber_dark, muted, dim, red" in text
assert "label: string, no vacío, OBLIGATORIO" in text # despliega los objetos anidados
def test_a_template_nobody_wrote_here_still_gets_described():
"""La prueba de que el contrato no está copiado: una plantilla inventada,
que este repo no conoce, se describe igual."""
text = describe_templates({**TEMPLATES, "holo_scan": {
"type": "object", "required": ["title"],
"properties": {"title": {"type": "string", "minLength": 1},
"depth_m": {"type": "number", "maximum": 999}}}})
assert "holo_scan:" in text
assert "depth_m: number, ≤ 999" in text
def test_validation_accepts_a_template_nobody_wrote_here():
templates = {**TEMPLATES, "holo_scan": {
"type": "object", "additionalProperties": False, "required": ["title"],
"properties": {"title": {"type": "string", "minLength": 1}}}}
validate_spec(spec_with({"template": "holo_scan", "duration": 30.0,
"props": {"title": "X"}}), templates)