"""El comprobador de fundamento, contra el spec de referencia y contra copias deliberadamente corrompidas. Los chunks de abajo son material de fuente sintético pero escrito como escribe una fuente real: fechas en otro orden que el spec, unidades con la palabra entera, comillas tipográficas, números con separador de millares. Si el comprobador sólo supiera comparar cadenas idénticas, este fichero lo delataría. """ import copy import json from pathlib import Path import pytest from src.generator.grounding import ( check_grounding, extract_claims, normalize, ) EXAMPLE = Path(__file__).resolve().parents[1] / "src/generator/examples/jal1628.json" @pytest.fixture def spec(): return json.loads(EXAMPLE.read_text()) #: Cada chunk imita una fuente distinta. Entre los cuatro está TODO lo que #: afirma examples/jal1628.json, pero casi nunca con las mismas palabras. CHUNKS = [ { "url": "https://www.faa.gov/foia/jal1628", "content": ( "On November 17, 1986, Japan Air Lines flight JAL 1628, a Boeing 747 " "cargo aircraft, was cruising at 35,000 feet and roughly 600 mph over " "Alaska, en route from Fort Yukon toward Anchorage by way of Fairbanks " "and Talkeetna. The flight crew reported two lights pacing the aircraft." ), }, { "url": "https://example.org/terauchi-testimony", "content": ( "The pilot in command was Captain Kenju Terauchi, an ex-fighter pilot " "with the JASDF, 29 years of flying experience and more than 10,000 " "flight hours. Terauchi described the object as “twice the size of an " "aircraft carrier”, an estimate that would put it between 1,600 and " "2,000 feet across — against the 232 ft length of his own Boeing 747. " "The unidentified contact held its relative position through a full " "360° turn and a descent of 4,000 ft." ), }, { "url": "https://example.org/radar-records", "content": ( "Three independent sources logged the encounter. The onboard radar " "showed a contact 7–8 nm out at the 10 o'clock position. Anchorage " "Center recorded primary returns through the turns. The Elmendorf ROCC " "tracked what it logged as a “flight of two”. Fairbanks radar showed " "nothing at all." ), }, { "url": "https://example.org/faa-closing", "content": ( "The FAA closed the case on 5 March 1987 with an official finding of a " "“split radar image”. An AARTCC controller said such a split happened " "“rarely, if ever” in that airspace. The FAA released roughly 1,500 " "pages of documentation. Forty years on — 40 years — the file is still " "open, and the estimated object has no accepted explanation." ), }, ] # --- normalización ---------------------------------------------------------- def test_normalize_thousands_separators(): assert normalize("35,000 FT") == normalize("35000 ft") == "35000 ft" assert normalize("1.500 paginas") == normalize("1,500 paginas") == "1500 paginas" # No toca los decimales de verdad: 61.22 es una latitud, no 6122. assert "61.22" in normalize("61.22") assert "1.5" in normalize("1.5") def test_normalize_quote_glyphs_and_dashes(): assert normalize("“FLIGHT OF TWO”") == normalize('"flight of two"') assert normalize("1,600 – 2,000") == normalize("1600 - 2000") assert normalize("−4,000") == normalize("-4000") assert normalize("CONTACT 7–8 NM · 10 O’CLOCK") == "contact 7-8 nm 10 o'clock" def test_normalize_is_idempotent(): once = normalize("“~1,600 – 2,000 FT”") assert normalize(once) == once # --- extracción ------------------------------------------------------------- def test_extracts_quotes_figures_dates_and_names(spec): claims = extract_claims(spec) by_kind = {} for c in claims: by_kind.setdefault(c.kind, set()).add(c.text) assert "TWICE THE SIZE OF AN AIRCRAFT CARRIER" in by_kind["quote"] assert "SPLIT RADAR IMAGE" in by_kind["quote"] assert "35,000 FT" in by_kind["figure"] assert "1500" in by_kind["figure"] # count_to, que sí afirma un dato assert "17 NOV 1986" in by_kind["date"] assert "5 MARCH 1987" in by_kind["date"] assert "CAPT. KENJU TERAUCHI" in by_kind["name"] assert "ELMENDORF ROCC" in by_kind["name"] def test_geometry_is_not_a_claim(spec): """Latitudes, duraciones, barridos y grados de giro son parámetros de dibujo: no dicen nada sobre el mundo y no se comprueban.""" paths = " ".join(c.path for c in extract_claims(spec)) for geometry in (".lat", ".lon", ".duration", ".sweeps", ".contact_bearing_deg", ".markers", ".bounds"): assert geometry not in paths def test_a_split_quote_is_one_claim(spec): """scale_bars.quote son las líneas de UNA cita: se comprueba entera, no a trozos (esas líneas las parte quien escribe el spec, no el renderizador).""" quotes = [c.text for c in extract_claims(spec) if c.kind == "quote"] assert "TWICE THE SIZE OF AN AIRCRAFT CARRIER" in quotes assert "TWICE THE SIZE OF" not in quotes def test_a_quote_welded_from_two_sources_is_flagged(spec): """El fallo real del Short de Socorro: cada mitad existe, la frase no. El modelo cogió un verbatim del testigo y le soldó la compresión de un resumen posterior, todo dentro de unas comillas y con su atribución debajo. Comprobar las líneas por separado habría dado las dos por buenas y publicado una frase que nadie dijo — por eso la unión es la unidad de comprobación. """ chunks = CHUNKS + [{ "url": "https://example.org/retelling", "content": ("Terauchi called it “twice the size of an aircraft carrier”. " "Later writers described the contact as unlit and silent."), }] haystack = " ".join(c["content"] for c in chunks).upper() assert "TWICE THE SIZE OF" in haystack and "UNLIT AND SILENT" in haystack welded = json.loads(json.dumps(spec)) welded["shots"][3]["props"]["quote"] = ["“TWICE THE SIZE OF", "UNLIT AND SILENT”"] report = check_grounding(welded, chunks) assert [c.text for c in report.ungrounded if c.kind == "quote"] == [ "TWICE THE SIZE OF UNLIT AND SILENT"] # --- comprobación ----------------------------------------------------------- def test_reference_spec_is_fully_grounded(spec): report = check_grounding(spec, CHUNKS) assert report.ungrounded == [], \ "sin fundamento: " + "; ".join(f"[{c.kind}] {c.text}" for c in report.ungrounded) assert report.clean assert report.total > 25 assert report.chunk_count == 4 and report.url_count == 4 def test_an_injected_figure_is_flagged_and_nothing_else(spec): corrupted = copy.deepcopy(spec) corrupted["shots"][7]["props"]["count_to"] = 12000 # eran 1.500 páginas corrupted["shots"][1]["props"]["subline"] = "41,000 FT · 600 MPH" report = check_grounding(corrupted, CHUNKS) flagged = {c.text for c in report.ungrounded} assert flagged == {"12000", "41,000 FT"} def test_an_invented_quote_is_flagged(spec): corrupted = copy.deepcopy(spec) corrupted["shots"][6]["props"]["quote_a"] = "“RADAR MALFUNCTION”" report = check_grounding(corrupted, CHUNKS) assert [c.text for c in report.ungrounded] == ["RADAR MALFUNCTION"] assert report.ungrounded[0].kind == "quote" assert report.ungrounded[0].path.startswith("shots.6.document_quote.props.quote_a") def test_an_invented_agency_is_flagged(spec): corrupted = copy.deepcopy(spec) corrupted["shots"][5]["props"]["strips"][2]["label"] = "NORAD CHEYENNE" report = check_grounding(corrupted, CHUNKS) assert [c.text for c in report.ungrounded] == ["NORAD CHEYENNE"] def test_an_invented_date_is_flagged(spec): corrupted = copy.deepcopy(spec) corrupted["shots"][1]["props"]["headline"] = "17 NOV 1987" report = check_grounding(corrupted, CHUNKS) assert [c.text for c in report.ungrounded] == ["17 NOV 1987"] def test_dates_match_across_formats(spec): """El spec escribe "17 NOV 1986" y la fuente "November 17, 1986". Es la misma fecha y el comprobador no debe gastarle un aviso al humano.""" report = check_grounding(spec, [CHUNKS[0]]) assert "17 NOV 1986" not in {c.text for c in report.ungrounded} def test_units_match_their_spelled_out_form(spec): """"35,000 FT" contra "35,000 feet".""" report = check_grounding(spec, [CHUNKS[0]]) assert "35,000 FT" not in {c.text for c in report.ungrounded} def test_a_number_alone_is_not_enough_without_its_unit(): """1.500 aparece en las fuentes como páginas; 1.500 FT no lo dice nadie.""" spec = { "version": 1, "meta": {"id": "x", "title": "x"}, "shots": [{"template": "scale_bars", "duration": 5.0, "props": { "headline": "H", "bars": [{"label": "ESTIMATED OBJECT", "value": 1500, "unit": "FT"}]}}], } report = check_grounding(spec, [CHUNKS[3]]) assert [c.text for c in report.ungrounded] == ["1500 FT"] def test_no_chunks_means_nothing_is_supported(spec): """Sin material no se apoya nada. Aquí todo cae en `contaminated` porque el spec de prueba ES el ejemplo del prompt — que es justo el diagnóstico correcto: ninguna de esas cifras viene de la sesión.""" report = check_grounding(spec, []) assert report.grounded == [] assert report.unsupported assert not report.clean assert report.chunk_count == 0 and report.url_count == 0 # --- fuga del ejemplo del prompt -------------------------------------------- def test_a_figure_copied_from_the_prompt_example_is_diagnosed_as_such(spec): """El caso real, medido el 2026-08-01 contra la sesión 153: el modelo escribió "232 FT" (el largo de un 747) y eso no estaba en ninguno de los 126 chunks — venía del ejemplo del prompt. No es una invención, es una fuga, y se arregla borrándola, no verificándola.""" sources_without_the_747 = [c for c in CHUNKS if "232" not in c["content"]] report = check_grounding(spec, sources_without_the_747) assert "232 FT" in {c.text for c in report.contaminated} assert "232 FT" not in {c.text for c in report.ungrounded} def test_an_invention_is_not_confused_with_a_leak(spec): """Una cifra que no está ni en las fuentes ni en el ejemplo sigue siendo una invención.""" corrupted = copy.deepcopy(spec) corrupted["shots"][1]["props"]["subline"] = "41,000 FT · 600 MPH" report = check_grounding(corrupted, CHUNKS) assert [c.text for c in report.ungrounded] == ["41,000 FT"] assert report.contaminated == [] def test_the_session_wins_over_the_example(spec): """Si el dato SÍ está en las fuentes, está fundamentado y punto: que además aparezca en el ejemplo no lo ensucia.""" report = check_grounding(spec, CHUNKS) assert report.contaminated == [] assert report.clean def test_a_leak_shows_up_in_the_report_with_its_own_wording(spec): report = check_grounding(spec, [c for c in CHUNKS if "232" not in c["content"]]) summary = report.summary() assert "copiados del EJEMPLO" in summary assert "232 FT" in summary assert "fuga, no invención" in summary def test_both_diagnoses_count_as_unsupported(spec): corrupted = copy.deepcopy(spec) corrupted["shots"][1]["props"]["subline"] = "41,000 FT · 600 MPH" report = check_grounding(corrupted, [c for c in CHUNKS if "232" not in c["content"]]) assert len(report.unsupported) == len(report.ungrounded) + len(report.contaminated) assert report.total == len(report.grounded) + len(report.unsupported) assert not report.clean def test_a_missing_example_file_degrades_to_the_old_behaviour(spec): """El contraste con el ejemplo es un diagnóstico extra, no un requisito: sin fichero, todo lo no encontrado vuelve a ser simplemente 'sin encontrar'.""" report = check_grounding(spec, [], example_haystacks=()) assert report.contaminated == [] assert report.ungrounded def test_summary_reports_success_out_loud(spec): report = check_grounding(spec, CHUNKS) summary = report.summary() assert "0 sin encontrar" in summary # el éxito NO es silencioso assert "4 chunks de 4 URLs" in summary def test_summary_lists_every_ungrounded_string(spec): corrupted = copy.deepcopy(spec) corrupted["shots"][7]["props"]["count_to"] = 12000 summary = check_grounding(corrupted, CHUNKS).summary() assert "⚠️ 1 sin encontrar" in summary assert '"12000"' in summary # --- narración (fase 4b) ---------------------------------------------------- # La narración es prosa que el modelo REDACTA, no una etiqueta que copia: es # el sitio natural donde se cuela una cifra de más. Estos tests existen antes # que el campo, a propósito — el comprobador se escribe sin conocer lo # comprobado (fase 2 §12). def test_a_fabricated_figure_hiding_in_the_narration_is_caught(spec): """El caso que justifica la fase entera: la pantalla dice la verdad y la voz añade una cifra que no está en ninguna fuente.""" narrated = copy.deepcopy(spec) narrated["shots"][0]["narration"] = ( "Three separate radars tracked the object at 41,000 feet.") report = check_grounding(narrated, CHUNKS) assert [c.text for c in report.ungrounded] == ["41,000 feet"] assert report.ungrounded[0].path == "shots.0.narration" def test_a_narration_that_stays_with_the_sources_is_clean(spec): narrated = copy.deepcopy(spec) narrated["shots"][0]["narration"] = ( "On November 17, 1986, the crew was cruising at 35,000 feet over Alaska.") assert check_grounding(narrated, CHUNKS).clean def test_a_quote_invented_for_the_voice_over_is_caught(spec): """Una cita hablada es una cita: o es verbatim o no lo es.""" narrated = copy.deepcopy(spec) narrated["shots"][0]["narration"] = ( 'The captain said it was “the size of two aircraft carriers”.') report = check_grounding(narrated, CHUNKS) assert any("two aircraft carriers" in c.text for c in report.ungrounded) def test_saying_out_loud_what_the_screen_already_shows_is_one_claim(spec): """Deduplicado por (tipo, forma normalizada): la misma cifra dibujada y narrada no infla el informe ni se cuenta dos veces.""" plain = check_grounding(spec, CHUNKS) narrated = copy.deepcopy(spec) narrated["shots"][0]["narration"] = "Three radars, 35,000 feet over Alaska." report = check_grounding(narrated, CHUNKS) assert report.total == plain.total def test_a_leak_from_the_example_is_still_diagnosed_as_a_leak_in_narration(spec): """La narración no se libra del segundo diagnóstico: una cifra del ejemplo del prompt sigue siendo fuga, no invención.""" narrated = copy.deepcopy(spec) narrated["shots"][0]["narration"] = "The aircraft itself measured 232 ft." report = check_grounding(narrated, [{"url": "u", "content": "Nothing useful."}]) assert any(c.text == "232 ft" for c in report.contaminated) def test_narration_contributes_no_name_claims(spec): """Una frase entera no es una etiqueta identificadora. Sacar nombres de la prosa exigiría adivinar por mayúsculas y llenaría el informe de ruido.""" narrated = copy.deepcopy(spec) narrated["shots"][0]["narration"] = "Nobody at Hangar Eighteen ever confirmed it." claims = [c for c in extract_claims(narrated) if c.path == "shots.0.narration"] assert claims == [] def test_a_shot_without_narration_behaves_exactly_as_before(spec): """El campo es opcional: un spec de hoy tiene que dar el mismo informe.""" before = check_grounding(spec, CHUNKS) with_empty = copy.deepcopy(spec) with_empty["shots"][0]["narration"] = "" after = check_grounding(with_empty, CHUNKS) assert after.total == before.total and after.clean == before.clean def test_the_same_figure_spelled_two_ways_is_one_claim(): """La huella de una cifra es su número y su unidad canónica. Sin esto, la voz repitiendo la pantalla duplicaría medio informe.""" spec = {"shots": [{"template": "scale_bars", "props": { "headline": "CRUISE ALTITUDE 35,000 FT"}, "narration": "They were cruising at 35,000 feet."}]} claims = extract_claims(spec) assert len([c for c in claims if c.kind == "figure"]) == 1 def test_the_same_number_with_different_units_stays_two_claims(): """1,600 ft y 1,600 m no son el mismo dato, y confundirlos sería peor que duplicar: escondería una cifra sin comprobar.""" spec = {"shots": [{"template": "x", "props": { "headline": "1,600 FT ACROSS", "footer": "1,600 m of runway"}}]} figures = [c for c in extract_claims(spec) if c.kind == "figure"] assert {c.unit for c in figures} == {"ft", "m"}