ChemaVXandClaude Opus 5
20c8d03aa7
Build & Deploy ResearchOwl / build-and-push (push) Successful in 9s
Añade /generate short_en y /short_spec. El pipeline genera un shot spec con Haiku, verifica cada cifra, fecha y cita contra los chunks de la sesión, lo renderiza en shortsmith y entrega el MP4 por Telegram junto a un informe de claims. - ShortsmithClient con sondeo y fallback al spec JSON si el render falla - Contrato de plantillas obtenido de GET /templates, no codificado - Comprobación de fundamento determinista, sin LLM - outputs.published_url para enlazar el artículo de Ghost - Normalización de comillas rectas a tipográficas (ver KNOWN-ISSUES.md) Lo que no aparece en los chunks se contrasta contra el ejemplo del prompt: si casa ahí es fuga, no invención, y se informa como tal. El purgado de sesiones se lleva también su MP4. La subida a YouTube queda fuera a propósito: fase 3. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
289 lines
11 KiB
Python
289 lines
11 KiB
Python
"""El comprobador de fundamento, contra el spec de referencia y contra copias
|
||
deliberadamente corrompidas.
|
||
|
||
Los chunks de abajo son material de fuente sintético pero escrito como escribe
|
||
una fuente real: fechas en otro orden que el spec, unidades con la palabra
|
||
entera, comillas tipográficas, números con separador de millares. Si el
|
||
comprobador sólo supiera comparar cadenas idénticas, este fichero lo delataría.
|
||
"""
|
||
import copy
|
||
import json
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
from src.generator.grounding import (
|
||
check_grounding, extract_claims, normalize,
|
||
)
|
||
|
||
EXAMPLE = Path(__file__).resolve().parents[1] / "src/generator/examples/jal1628.json"
|
||
|
||
|
||
@pytest.fixture
|
||
def spec():
|
||
return json.loads(EXAMPLE.read_text())
|
||
|
||
|
||
#: Cada chunk imita una fuente distinta. Entre los cuatro está TODO lo que
|
||
#: afirma examples/jal1628.json, pero casi nunca con las mismas palabras.
|
||
CHUNKS = [
|
||
{
|
||
"url": "https://www.faa.gov/foia/jal1628",
|
||
"content": (
|
||
"On November 17, 1986, Japan Air Lines flight JAL 1628, a Boeing 747 "
|
||
"cargo aircraft, was cruising at 35,000 feet and roughly 600 mph over "
|
||
"Alaska, en route from Fort Yukon toward Anchorage by way of Fairbanks "
|
||
"and Talkeetna. The flight crew reported two lights pacing the aircraft."
|
||
),
|
||
},
|
||
{
|
||
"url": "https://example.org/terauchi-testimony",
|
||
"content": (
|
||
"The pilot in command was Captain Kenju Terauchi, an ex-fighter pilot "
|
||
"with the JASDF, 29 years of flying experience and more than 10,000 "
|
||
"flight hours. Terauchi described the object as “twice the size of an "
|
||
"aircraft carrier”, an estimate that would put it between 1,600 and "
|
||
"2,000 feet across — against the 232 ft length of his own Boeing 747. "
|
||
"The unidentified contact held its relative position through a full "
|
||
"360° turn and a descent of 4,000 ft."
|
||
),
|
||
},
|
||
{
|
||
"url": "https://example.org/radar-records",
|
||
"content": (
|
||
"Three independent sources logged the encounter. The onboard radar "
|
||
"showed a contact 7–8 nm out at the 10 o'clock position. Anchorage "
|
||
"Center recorded primary returns through the turns. The Elmendorf ROCC "
|
||
"tracked what it logged as a “flight of two”. Fairbanks radar showed "
|
||
"nothing at all."
|
||
),
|
||
},
|
||
{
|
||
"url": "https://example.org/faa-closing",
|
||
"content": (
|
||
"The FAA closed the case on 5 March 1987 with an official finding of a "
|
||
"“split radar image”. An AARTCC controller said such a split happened "
|
||
"“rarely, if ever” in that airspace. The FAA released roughly 1,500 "
|
||
"pages of documentation. Forty years on — 40 years — the file is still "
|
||
"open, and the estimated object has no accepted explanation."
|
||
),
|
||
},
|
||
]
|
||
|
||
|
||
# --- normalización ----------------------------------------------------------
|
||
|
||
def test_normalize_thousands_separators():
|
||
assert normalize("35,000 FT") == normalize("35000 ft") == "35000 ft"
|
||
assert normalize("1.500 paginas") == normalize("1,500 paginas") == "1500 paginas"
|
||
# No toca los decimales de verdad: 61.22 es una latitud, no 6122.
|
||
assert "61.22" in normalize("61.22")
|
||
assert "1.5" in normalize("1.5")
|
||
|
||
|
||
def test_normalize_quote_glyphs_and_dashes():
|
||
assert normalize("“FLIGHT OF TWO”") == normalize('"flight of two"')
|
||
assert normalize("1,600 – 2,000") == normalize("1600 - 2000")
|
||
assert normalize("−4,000") == normalize("-4000")
|
||
assert normalize("CONTACT 7–8 NM · 10 O’CLOCK") == "contact 7-8 nm 10 o'clock"
|
||
|
||
|
||
def test_normalize_is_idempotent():
|
||
once = normalize("“~1,600 – 2,000 FT”")
|
||
assert normalize(once) == once
|
||
|
||
|
||
# --- extracción -------------------------------------------------------------
|
||
|
||
def test_extracts_quotes_figures_dates_and_names(spec):
|
||
claims = extract_claims(spec)
|
||
by_kind = {}
|
||
for c in claims:
|
||
by_kind.setdefault(c.kind, set()).add(c.text)
|
||
|
||
assert "TWICE THE SIZE OF AN AIRCRAFT CARRIER" in by_kind["quote"]
|
||
assert "SPLIT RADAR IMAGE" in by_kind["quote"]
|
||
assert "35,000 FT" in by_kind["figure"]
|
||
assert "1500" in by_kind["figure"] # count_to, que sí afirma un dato
|
||
assert "17 NOV 1986" in by_kind["date"]
|
||
assert "5 MARCH 1987" in by_kind["date"]
|
||
assert "CAPT. KENJU TERAUCHI" in by_kind["name"]
|
||
assert "ELMENDORF ROCC" in by_kind["name"]
|
||
|
||
|
||
def test_geometry_is_not_a_claim(spec):
|
||
"""Latitudes, duraciones, barridos y grados de giro son parámetros de dibujo:
|
||
no dicen nada sobre el mundo y no se comprueban."""
|
||
paths = " ".join(c.path for c in extract_claims(spec))
|
||
for geometry in (".lat", ".lon", ".duration", ".sweeps",
|
||
".contact_bearing_deg", ".markers", ".bounds"):
|
||
assert geometry not in paths
|
||
|
||
|
||
def test_a_split_quote_is_one_claim(spec):
|
||
"""scale_bars.quote son las líneas de UNA cita: se comprueba entera, no a
|
||
trozos (el renderizador no envuelve; el caller parte las líneas)."""
|
||
quotes = [c.text for c in extract_claims(spec) if c.kind == "quote"]
|
||
assert "TWICE THE SIZE OF AN AIRCRAFT CARRIER" in quotes
|
||
assert "TWICE THE SIZE OF" not in quotes
|
||
|
||
|
||
# --- comprobación -----------------------------------------------------------
|
||
|
||
def test_reference_spec_is_fully_grounded(spec):
|
||
report = check_grounding(spec, CHUNKS)
|
||
assert report.ungrounded == [], \
|
||
"sin fundamento: " + "; ".join(f"[{c.kind}] {c.text}" for c in report.ungrounded)
|
||
assert report.clean
|
||
assert report.total > 25
|
||
assert report.chunk_count == 4 and report.url_count == 4
|
||
|
||
|
||
def test_an_injected_figure_is_flagged_and_nothing_else(spec):
|
||
corrupted = copy.deepcopy(spec)
|
||
corrupted["shots"][7]["props"]["count_to"] = 12000 # eran 1.500 páginas
|
||
corrupted["shots"][1]["props"]["subline"] = "41,000 FT · 600 MPH"
|
||
|
||
report = check_grounding(corrupted, CHUNKS)
|
||
flagged = {c.text for c in report.ungrounded}
|
||
assert flagged == {"12000", "41,000 FT"}
|
||
|
||
|
||
def test_an_invented_quote_is_flagged(spec):
|
||
corrupted = copy.deepcopy(spec)
|
||
corrupted["shots"][6]["props"]["quote_a"] = "“RADAR MALFUNCTION”"
|
||
|
||
report = check_grounding(corrupted, CHUNKS)
|
||
assert [c.text for c in report.ungrounded] == ["RADAR MALFUNCTION"]
|
||
assert report.ungrounded[0].kind == "quote"
|
||
assert report.ungrounded[0].path.startswith("shots.6.document_quote.props.quote_a")
|
||
|
||
|
||
def test_an_invented_agency_is_flagged(spec):
|
||
corrupted = copy.deepcopy(spec)
|
||
corrupted["shots"][5]["props"]["strips"][2]["label"] = "NORAD CHEYENNE"
|
||
|
||
report = check_grounding(corrupted, CHUNKS)
|
||
assert [c.text for c in report.ungrounded] == ["NORAD CHEYENNE"]
|
||
|
||
|
||
def test_an_invented_date_is_flagged(spec):
|
||
corrupted = copy.deepcopy(spec)
|
||
corrupted["shots"][1]["props"]["headline"] = "17 NOV 1987"
|
||
|
||
report = check_grounding(corrupted, CHUNKS)
|
||
assert [c.text for c in report.ungrounded] == ["17 NOV 1987"]
|
||
|
||
|
||
def test_dates_match_across_formats(spec):
|
||
"""El spec escribe "17 NOV 1986" y la fuente "November 17, 1986". Es la misma
|
||
fecha y el comprobador no debe gastarle un aviso al humano."""
|
||
report = check_grounding(spec, [CHUNKS[0]])
|
||
assert "17 NOV 1986" not in {c.text for c in report.ungrounded}
|
||
|
||
|
||
def test_units_match_their_spelled_out_form(spec):
|
||
""""35,000 FT" contra "35,000 feet"."""
|
||
report = check_grounding(spec, [CHUNKS[0]])
|
||
assert "35,000 FT" not in {c.text for c in report.ungrounded}
|
||
|
||
|
||
def test_a_number_alone_is_not_enough_without_its_unit():
|
||
"""1.500 aparece en las fuentes como páginas; 1.500 FT no lo dice nadie."""
|
||
spec = {
|
||
"version": 1,
|
||
"meta": {"id": "x", "title": "x"},
|
||
"shots": [{"template": "scale_bars", "duration": 5.0, "props": {
|
||
"headline": "H",
|
||
"bars": [{"label": "ESTIMATED OBJECT", "value": 1500, "unit": "FT"}]}}],
|
||
}
|
||
report = check_grounding(spec, [CHUNKS[3]])
|
||
assert [c.text for c in report.ungrounded] == ["1500 FT"]
|
||
|
||
|
||
def test_no_chunks_means_nothing_is_supported(spec):
|
||
"""Sin material no se apoya nada. Aquí todo cae en `contaminated` porque el
|
||
spec de prueba ES el ejemplo del prompt — que es justo el diagnóstico
|
||
correcto: ninguna de esas cifras viene de la sesión."""
|
||
report = check_grounding(spec, [])
|
||
assert report.grounded == []
|
||
assert report.unsupported
|
||
assert not report.clean
|
||
assert report.chunk_count == 0 and report.url_count == 0
|
||
|
||
|
||
# --- fuga del ejemplo del prompt --------------------------------------------
|
||
|
||
def test_a_figure_copied_from_the_prompt_example_is_diagnosed_as_such(spec):
|
||
"""El caso real, medido el 2026-08-01 contra la sesión 153: el modelo
|
||
escribió "232 FT" (el largo de un 747) y eso no estaba en ninguno de los 126
|
||
chunks — venía del ejemplo del prompt. No es una invención, es una fuga, y
|
||
se arregla borrándola, no verificándola."""
|
||
sources_without_the_747 = [c for c in CHUNKS if "232" not in c["content"]]
|
||
|
||
report = check_grounding(spec, sources_without_the_747)
|
||
|
||
assert "232 FT" in {c.text for c in report.contaminated}
|
||
assert "232 FT" not in {c.text for c in report.ungrounded}
|
||
|
||
|
||
def test_an_invention_is_not_confused_with_a_leak(spec):
|
||
"""Una cifra que no está ni en las fuentes ni en el ejemplo sigue siendo
|
||
una invención."""
|
||
corrupted = copy.deepcopy(spec)
|
||
corrupted["shots"][1]["props"]["subline"] = "41,000 FT · 600 MPH"
|
||
|
||
report = check_grounding(corrupted, CHUNKS)
|
||
|
||
assert [c.text for c in report.ungrounded] == ["41,000 FT"]
|
||
assert report.contaminated == []
|
||
|
||
|
||
def test_the_session_wins_over_the_example(spec):
|
||
"""Si el dato SÍ está en las fuentes, está fundamentado y punto: que además
|
||
aparezca en el ejemplo no lo ensucia."""
|
||
report = check_grounding(spec, CHUNKS)
|
||
assert report.contaminated == []
|
||
assert report.clean
|
||
|
||
|
||
def test_a_leak_shows_up_in_the_report_with_its_own_wording(spec):
|
||
report = check_grounding(spec, [c for c in CHUNKS if "232" not in c["content"]])
|
||
summary = report.summary()
|
||
assert "copiados del EJEMPLO" in summary
|
||
assert "232 FT" in summary
|
||
assert "fuga, no invención" in summary
|
||
|
||
|
||
def test_both_diagnoses_count_as_unsupported(spec):
|
||
corrupted = copy.deepcopy(spec)
|
||
corrupted["shots"][1]["props"]["subline"] = "41,000 FT · 600 MPH"
|
||
report = check_grounding(corrupted, [c for c in CHUNKS if "232" not in c["content"]])
|
||
|
||
assert len(report.unsupported) == len(report.ungrounded) + len(report.contaminated)
|
||
assert report.total == len(report.grounded) + len(report.unsupported)
|
||
assert not report.clean
|
||
|
||
|
||
def test_a_missing_example_file_degrades_to_the_old_behaviour(spec):
|
||
"""El contraste con el ejemplo es un diagnóstico extra, no un requisito: sin
|
||
fichero, todo lo no encontrado vuelve a ser simplemente 'sin encontrar'."""
|
||
report = check_grounding(spec, [], example_haystacks=())
|
||
assert report.contaminated == []
|
||
assert report.ungrounded
|
||
|
||
|
||
def test_summary_reports_success_out_loud(spec):
|
||
report = check_grounding(spec, CHUNKS)
|
||
summary = report.summary()
|
||
assert "0 sin encontrar" in summary # el éxito NO es silencioso
|
||
assert "4 chunks de 4 URLs" in summary
|
||
|
||
|
||
def test_summary_lists_every_ungrounded_string(spec):
|
||
corrupted = copy.deepcopy(spec)
|
||
corrupted["shots"][7]["props"]["count_to"] = 12000
|
||
summary = check_grounding(corrupted, CHUNKS).summary()
|
||
assert "⚠️ 1 sin encontrar" in summary
|
||
assert '"12000"' in summary
|