ChemaVXandClaude Fable 5
366ded1f59
Build & Deploy ResearchOwl / build-and-push (push) Successful in 8s
El Short de Socorro salió con “LIKE ALUMINUM / SMOOTH, NO WINDOWS” atribuido a Zamora. La primera mitad es su informe literal; la segunda es la compresión de un divulgador resumiéndolo. Ninguna fuente pone esas palabras juntas y Zamora nunca dijo la segunda: es una cita fabricada con material auténtico. El comprobador funcionó — encuentra LIKE ALUMINUM y rechaza la unión — así que el fallo estaba antes, en el prompt. `scale_bars.quote` es una lista de líneas por maquetación, pero al modelo le llega como dos huecos, y los llenó de sitios distintos. Ningún otro campo de cita tiene esa forma: quote_a y quote_b sí son dos citas a propósito, y esas salieron bien. El prompt ahora dice que las líneas se leen unidas, que eso es lo que busca la comprobación y lo que lee el espectador, y que la atribución empeora el pegote en vez de arreglarlo: nombra a quien acabas de poner esas palabras en la boca. El test fija la unión como unidad de comprobación con el caso exacto: dos mitades que existen por separado y una frase que no. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
421 lines
17 KiB
Python
421 lines
17 KiB
Python
"""El comprobador de fundamento, contra el spec de referencia y contra copias
|
||
deliberadamente corrompidas.
|
||
|
||
Los chunks de abajo son material de fuente sintético pero escrito como escribe
|
||
una fuente real: fechas en otro orden que el spec, unidades con la palabra
|
||
entera, comillas tipográficas, números con separador de millares. Si el
|
||
comprobador sólo supiera comparar cadenas idénticas, este fichero lo delataría.
|
||
"""
|
||
import copy
|
||
import json
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
|
||
from src.generator.grounding import (
|
||
check_grounding, extract_claims, normalize,
|
||
)
|
||
|
||
EXAMPLE = Path(__file__).resolve().parents[1] / "src/generator/examples/jal1628.json"
|
||
|
||
|
||
@pytest.fixture
|
||
def spec():
|
||
return json.loads(EXAMPLE.read_text())
|
||
|
||
|
||
#: Cada chunk imita una fuente distinta. Entre los cuatro está TODO lo que
|
||
#: afirma examples/jal1628.json, pero casi nunca con las mismas palabras.
|
||
CHUNKS = [
|
||
{
|
||
"url": "https://www.faa.gov/foia/jal1628",
|
||
"content": (
|
||
"On November 17, 1986, Japan Air Lines flight JAL 1628, a Boeing 747 "
|
||
"cargo aircraft, was cruising at 35,000 feet and roughly 600 mph over "
|
||
"Alaska, en route from Fort Yukon toward Anchorage by way of Fairbanks "
|
||
"and Talkeetna. The flight crew reported two lights pacing the aircraft."
|
||
),
|
||
},
|
||
{
|
||
"url": "https://example.org/terauchi-testimony",
|
||
"content": (
|
||
"The pilot in command was Captain Kenju Terauchi, an ex-fighter pilot "
|
||
"with the JASDF, 29 years of flying experience and more than 10,000 "
|
||
"flight hours. Terauchi described the object as “twice the size of an "
|
||
"aircraft carrier”, an estimate that would put it between 1,600 and "
|
||
"2,000 feet across — against the 232 ft length of his own Boeing 747. "
|
||
"The unidentified contact held its relative position through a full "
|
||
"360° turn and a descent of 4,000 ft."
|
||
),
|
||
},
|
||
{
|
||
"url": "https://example.org/radar-records",
|
||
"content": (
|
||
"Three independent sources logged the encounter. The onboard radar "
|
||
"showed a contact 7–8 nm out at the 10 o'clock position. Anchorage "
|
||
"Center recorded primary returns through the turns. The Elmendorf ROCC "
|
||
"tracked what it logged as a “flight of two”. Fairbanks radar showed "
|
||
"nothing at all."
|
||
),
|
||
},
|
||
{
|
||
"url": "https://example.org/faa-closing",
|
||
"content": (
|
||
"The FAA closed the case on 5 March 1987 with an official finding of a "
|
||
"“split radar image”. An AARTCC controller said such a split happened "
|
||
"“rarely, if ever” in that airspace. The FAA released roughly 1,500 "
|
||
"pages of documentation. Forty years on — 40 years — the file is still "
|
||
"open, and the estimated object has no accepted explanation."
|
||
),
|
||
},
|
||
]
|
||
|
||
|
||
# --- normalización ----------------------------------------------------------
|
||
|
||
def test_normalize_thousands_separators():
|
||
assert normalize("35,000 FT") == normalize("35000 ft") == "35000 ft"
|
||
assert normalize("1.500 paginas") == normalize("1,500 paginas") == "1500 paginas"
|
||
# No toca los decimales de verdad: 61.22 es una latitud, no 6122.
|
||
assert "61.22" in normalize("61.22")
|
||
assert "1.5" in normalize("1.5")
|
||
|
||
|
||
def test_normalize_quote_glyphs_and_dashes():
|
||
assert normalize("“FLIGHT OF TWO”") == normalize('"flight of two"')
|
||
assert normalize("1,600 – 2,000") == normalize("1600 - 2000")
|
||
assert normalize("−4,000") == normalize("-4000")
|
||
assert normalize("CONTACT 7–8 NM · 10 O’CLOCK") == "contact 7-8 nm 10 o'clock"
|
||
|
||
|
||
def test_normalize_is_idempotent():
|
||
once = normalize("“~1,600 – 2,000 FT”")
|
||
assert normalize(once) == once
|
||
|
||
|
||
# --- extracción -------------------------------------------------------------
|
||
|
||
def test_extracts_quotes_figures_dates_and_names(spec):
|
||
claims = extract_claims(spec)
|
||
by_kind = {}
|
||
for c in claims:
|
||
by_kind.setdefault(c.kind, set()).add(c.text)
|
||
|
||
assert "TWICE THE SIZE OF AN AIRCRAFT CARRIER" in by_kind["quote"]
|
||
assert "SPLIT RADAR IMAGE" in by_kind["quote"]
|
||
assert "35,000 FT" in by_kind["figure"]
|
||
assert "1500" in by_kind["figure"] # count_to, que sí afirma un dato
|
||
assert "17 NOV 1986" in by_kind["date"]
|
||
assert "5 MARCH 1987" in by_kind["date"]
|
||
assert "CAPT. KENJU TERAUCHI" in by_kind["name"]
|
||
assert "ELMENDORF ROCC" in by_kind["name"]
|
||
|
||
|
||
def test_geometry_is_not_a_claim(spec):
|
||
"""Latitudes, duraciones, barridos y grados de giro son parámetros de dibujo:
|
||
no dicen nada sobre el mundo y no se comprueban."""
|
||
paths = " ".join(c.path for c in extract_claims(spec))
|
||
for geometry in (".lat", ".lon", ".duration", ".sweeps",
|
||
".contact_bearing_deg", ".markers", ".bounds"):
|
||
assert geometry not in paths
|
||
|
||
|
||
def test_a_split_quote_is_one_claim(spec):
|
||
"""scale_bars.quote son las líneas de UNA cita: se comprueba entera, no a
|
||
trozos (esas líneas las parte quien escribe el spec, no el renderizador)."""
|
||
quotes = [c.text for c in extract_claims(spec) if c.kind == "quote"]
|
||
assert "TWICE THE SIZE OF AN AIRCRAFT CARRIER" in quotes
|
||
assert "TWICE THE SIZE OF" not in quotes
|
||
|
||
|
||
def test_a_quote_welded_from_two_sources_is_flagged(spec):
|
||
"""El fallo real del Short de Socorro: cada mitad existe, la frase no.
|
||
|
||
El modelo cogió un verbatim del testigo y le soldó la compresión de un
|
||
resumen posterior, todo dentro de unas comillas y con su atribución debajo.
|
||
Comprobar las líneas por separado habría dado las dos por buenas y publicado
|
||
una frase que nadie dijo — por eso la unión es la unidad de comprobación.
|
||
"""
|
||
chunks = CHUNKS + [{
|
||
"url": "https://example.org/retelling",
|
||
"content": ("Terauchi called it “twice the size of an aircraft carrier”. "
|
||
"Later writers described the contact as unlit and silent."),
|
||
}]
|
||
haystack = " ".join(c["content"] for c in chunks).upper()
|
||
assert "TWICE THE SIZE OF" in haystack and "UNLIT AND SILENT" in haystack
|
||
|
||
welded = json.loads(json.dumps(spec))
|
||
welded["shots"][3]["props"]["quote"] = ["“TWICE THE SIZE OF", "UNLIT AND SILENT”"]
|
||
|
||
report = check_grounding(welded, chunks)
|
||
|
||
assert [c.text for c in report.ungrounded if c.kind == "quote"] == [
|
||
"TWICE THE SIZE OF UNLIT AND SILENT"]
|
||
|
||
|
||
# --- comprobación -----------------------------------------------------------
|
||
|
||
def test_reference_spec_is_fully_grounded(spec):
|
||
report = check_grounding(spec, CHUNKS)
|
||
assert report.ungrounded == [], \
|
||
"sin fundamento: " + "; ".join(f"[{c.kind}] {c.text}" for c in report.ungrounded)
|
||
assert report.clean
|
||
assert report.total > 25
|
||
assert report.chunk_count == 4 and report.url_count == 4
|
||
|
||
|
||
def test_an_injected_figure_is_flagged_and_nothing_else(spec):
|
||
corrupted = copy.deepcopy(spec)
|
||
corrupted["shots"][7]["props"]["count_to"] = 12000 # eran 1.500 páginas
|
||
corrupted["shots"][1]["props"]["subline"] = "41,000 FT · 600 MPH"
|
||
|
||
report = check_grounding(corrupted, CHUNKS)
|
||
flagged = {c.text for c in report.ungrounded}
|
||
assert flagged == {"12000", "41,000 FT"}
|
||
|
||
|
||
def test_an_invented_quote_is_flagged(spec):
|
||
corrupted = copy.deepcopy(spec)
|
||
corrupted["shots"][6]["props"]["quote_a"] = "“RADAR MALFUNCTION”"
|
||
|
||
report = check_grounding(corrupted, CHUNKS)
|
||
assert [c.text for c in report.ungrounded] == ["RADAR MALFUNCTION"]
|
||
assert report.ungrounded[0].kind == "quote"
|
||
assert report.ungrounded[0].path.startswith("shots.6.document_quote.props.quote_a")
|
||
|
||
|
||
def test_an_invented_agency_is_flagged(spec):
|
||
corrupted = copy.deepcopy(spec)
|
||
corrupted["shots"][5]["props"]["strips"][2]["label"] = "NORAD CHEYENNE"
|
||
|
||
report = check_grounding(corrupted, CHUNKS)
|
||
assert [c.text for c in report.ungrounded] == ["NORAD CHEYENNE"]
|
||
|
||
|
||
def test_an_invented_date_is_flagged(spec):
|
||
corrupted = copy.deepcopy(spec)
|
||
corrupted["shots"][1]["props"]["headline"] = "17 NOV 1987"
|
||
|
||
report = check_grounding(corrupted, CHUNKS)
|
||
assert [c.text for c in report.ungrounded] == ["17 NOV 1987"]
|
||
|
||
|
||
def test_dates_match_across_formats(spec):
|
||
"""El spec escribe "17 NOV 1986" y la fuente "November 17, 1986". Es la misma
|
||
fecha y el comprobador no debe gastarle un aviso al humano."""
|
||
report = check_grounding(spec, [CHUNKS[0]])
|
||
assert "17 NOV 1986" not in {c.text for c in report.ungrounded}
|
||
|
||
|
||
def test_units_match_their_spelled_out_form(spec):
|
||
""""35,000 FT" contra "35,000 feet"."""
|
||
report = check_grounding(spec, [CHUNKS[0]])
|
||
assert "35,000 FT" not in {c.text for c in report.ungrounded}
|
||
|
||
|
||
def test_a_number_alone_is_not_enough_without_its_unit():
|
||
"""1.500 aparece en las fuentes como páginas; 1.500 FT no lo dice nadie."""
|
||
spec = {
|
||
"version": 1,
|
||
"meta": {"id": "x", "title": "x"},
|
||
"shots": [{"template": "scale_bars", "duration": 5.0, "props": {
|
||
"headline": "H",
|
||
"bars": [{"label": "ESTIMATED OBJECT", "value": 1500, "unit": "FT"}]}}],
|
||
}
|
||
report = check_grounding(spec, [CHUNKS[3]])
|
||
assert [c.text for c in report.ungrounded] == ["1500 FT"]
|
||
|
||
|
||
def test_no_chunks_means_nothing_is_supported(spec):
|
||
"""Sin material no se apoya nada. Aquí todo cae en `contaminated` porque el
|
||
spec de prueba ES el ejemplo del prompt — que es justo el diagnóstico
|
||
correcto: ninguna de esas cifras viene de la sesión."""
|
||
report = check_grounding(spec, [])
|
||
assert report.grounded == []
|
||
assert report.unsupported
|
||
assert not report.clean
|
||
assert report.chunk_count == 0 and report.url_count == 0
|
||
|
||
|
||
# --- fuga del ejemplo del prompt --------------------------------------------
|
||
|
||
def test_a_figure_copied_from_the_prompt_example_is_diagnosed_as_such(spec):
|
||
"""El caso real, medido el 2026-08-01 contra la sesión 153: el modelo
|
||
escribió "232 FT" (el largo de un 747) y eso no estaba en ninguno de los 126
|
||
chunks — venía del ejemplo del prompt. No es una invención, es una fuga, y
|
||
se arregla borrándola, no verificándola."""
|
||
sources_without_the_747 = [c for c in CHUNKS if "232" not in c["content"]]
|
||
|
||
report = check_grounding(spec, sources_without_the_747)
|
||
|
||
assert "232 FT" in {c.text for c in report.contaminated}
|
||
assert "232 FT" not in {c.text for c in report.ungrounded}
|
||
|
||
|
||
def test_an_invention_is_not_confused_with_a_leak(spec):
|
||
"""Una cifra que no está ni en las fuentes ni en el ejemplo sigue siendo
|
||
una invención."""
|
||
corrupted = copy.deepcopy(spec)
|
||
corrupted["shots"][1]["props"]["subline"] = "41,000 FT · 600 MPH"
|
||
|
||
report = check_grounding(corrupted, CHUNKS)
|
||
|
||
assert [c.text for c in report.ungrounded] == ["41,000 FT"]
|
||
assert report.contaminated == []
|
||
|
||
|
||
def test_the_session_wins_over_the_example(spec):
|
||
"""Si el dato SÍ está en las fuentes, está fundamentado y punto: que además
|
||
aparezca en el ejemplo no lo ensucia."""
|
||
report = check_grounding(spec, CHUNKS)
|
||
assert report.contaminated == []
|
||
assert report.clean
|
||
|
||
|
||
def test_a_leak_shows_up_in_the_report_with_its_own_wording(spec):
|
||
report = check_grounding(spec, [c for c in CHUNKS if "232" not in c["content"]])
|
||
summary = report.summary()
|
||
assert "copiados del EJEMPLO" in summary
|
||
assert "232 FT" in summary
|
||
assert "fuga, no invención" in summary
|
||
|
||
|
||
def test_both_diagnoses_count_as_unsupported(spec):
|
||
corrupted = copy.deepcopy(spec)
|
||
corrupted["shots"][1]["props"]["subline"] = "41,000 FT · 600 MPH"
|
||
report = check_grounding(corrupted, [c for c in CHUNKS if "232" not in c["content"]])
|
||
|
||
assert len(report.unsupported) == len(report.ungrounded) + len(report.contaminated)
|
||
assert report.total == len(report.grounded) + len(report.unsupported)
|
||
assert not report.clean
|
||
|
||
|
||
def test_a_missing_example_file_degrades_to_the_old_behaviour(spec):
|
||
"""El contraste con el ejemplo es un diagnóstico extra, no un requisito: sin
|
||
fichero, todo lo no encontrado vuelve a ser simplemente 'sin encontrar'."""
|
||
report = check_grounding(spec, [], example_haystacks=())
|
||
assert report.contaminated == []
|
||
assert report.ungrounded
|
||
|
||
|
||
def test_summary_reports_success_out_loud(spec):
|
||
report = check_grounding(spec, CHUNKS)
|
||
summary = report.summary()
|
||
assert "0 sin encontrar" in summary # el éxito NO es silencioso
|
||
assert "4 chunks de 4 URLs" in summary
|
||
|
||
|
||
def test_summary_lists_every_ungrounded_string(spec):
|
||
corrupted = copy.deepcopy(spec)
|
||
corrupted["shots"][7]["props"]["count_to"] = 12000
|
||
summary = check_grounding(corrupted, CHUNKS).summary()
|
||
assert "⚠️ 1 sin encontrar" in summary
|
||
assert '"12000"' in summary
|
||
|
||
|
||
# --- narración (fase 4b) ----------------------------------------------------
|
||
# La narración es prosa que el modelo REDACTA, no una etiqueta que copia: es
|
||
# el sitio natural donde se cuela una cifra de más. Estos tests existen antes
|
||
# que el campo, a propósito — el comprobador se escribe sin conocer lo
|
||
# comprobado (fase 2 §12).
|
||
|
||
|
||
def test_a_fabricated_figure_hiding_in_the_narration_is_caught(spec):
|
||
"""El caso que justifica la fase entera: la pantalla dice la verdad y la
|
||
voz añade una cifra que no está en ninguna fuente."""
|
||
narrated = copy.deepcopy(spec)
|
||
narrated["shots"][0]["narration"] = (
|
||
"Three separate radars tracked the object at 41,000 feet.")
|
||
|
||
report = check_grounding(narrated, CHUNKS)
|
||
|
||
assert [c.text for c in report.ungrounded] == ["41,000 feet"]
|
||
assert report.ungrounded[0].path == "shots.0.narration"
|
||
|
||
|
||
def test_a_narration_that_stays_with_the_sources_is_clean(spec):
|
||
narrated = copy.deepcopy(spec)
|
||
narrated["shots"][0]["narration"] = (
|
||
"On November 17, 1986, the crew was cruising at 35,000 feet over Alaska.")
|
||
|
||
assert check_grounding(narrated, CHUNKS).clean
|
||
|
||
|
||
def test_a_quote_invented_for_the_voice_over_is_caught(spec):
|
||
"""Una cita hablada es una cita: o es verbatim o no lo es."""
|
||
narrated = copy.deepcopy(spec)
|
||
narrated["shots"][0]["narration"] = (
|
||
'The captain said it was “the size of two aircraft carriers”.')
|
||
|
||
report = check_grounding(narrated, CHUNKS)
|
||
|
||
assert any("two aircraft carriers" in c.text for c in report.ungrounded)
|
||
|
||
|
||
def test_saying_out_loud_what_the_screen_already_shows_is_one_claim(spec):
|
||
"""Deduplicado por (tipo, forma normalizada): la misma cifra dibujada y
|
||
narrada no infla el informe ni se cuenta dos veces."""
|
||
plain = check_grounding(spec, CHUNKS)
|
||
narrated = copy.deepcopy(spec)
|
||
narrated["shots"][0]["narration"] = "Three radars, 35,000 feet over Alaska."
|
||
|
||
report = check_grounding(narrated, CHUNKS)
|
||
|
||
assert report.total == plain.total
|
||
|
||
|
||
def test_a_leak_from_the_example_is_still_diagnosed_as_a_leak_in_narration(spec):
|
||
"""La narración no se libra del segundo diagnóstico: una cifra del ejemplo
|
||
del prompt sigue siendo fuga, no invención."""
|
||
narrated = copy.deepcopy(spec)
|
||
narrated["shots"][0]["narration"] = "The aircraft itself measured 232 ft."
|
||
|
||
report = check_grounding(narrated, [{"url": "u", "content": "Nothing useful."}])
|
||
|
||
assert any(c.text == "232 ft" for c in report.contaminated)
|
||
|
||
|
||
def test_narration_contributes_no_name_claims(spec):
|
||
"""Una frase entera no es una etiqueta identificadora. Sacar nombres de la
|
||
prosa exigiría adivinar por mayúsculas y llenaría el informe de ruido."""
|
||
narrated = copy.deepcopy(spec)
|
||
narrated["shots"][0]["narration"] = "Nobody at Hangar Eighteen ever confirmed it."
|
||
|
||
claims = [c for c in extract_claims(narrated) if c.path == "shots.0.narration"]
|
||
|
||
assert claims == []
|
||
|
||
|
||
def test_a_shot_without_narration_behaves_exactly_as_before(spec):
|
||
"""El campo es opcional: un spec de hoy tiene que dar el mismo informe."""
|
||
before = check_grounding(spec, CHUNKS)
|
||
with_empty = copy.deepcopy(spec)
|
||
with_empty["shots"][0]["narration"] = ""
|
||
|
||
after = check_grounding(with_empty, CHUNKS)
|
||
|
||
assert after.total == before.total and after.clean == before.clean
|
||
|
||
|
||
def test_the_same_figure_spelled_two_ways_is_one_claim():
|
||
"""La huella de una cifra es su número y su unidad canónica. Sin esto, la
|
||
voz repitiendo la pantalla duplicaría medio informe."""
|
||
spec = {"shots": [{"template": "scale_bars", "props": {
|
||
"headline": "CRUISE ALTITUDE 35,000 FT"},
|
||
"narration": "They were cruising at 35,000 feet."}]}
|
||
|
||
claims = extract_claims(spec)
|
||
|
||
assert len([c for c in claims if c.kind == "figure"]) == 1
|
||
|
||
|
||
def test_the_same_number_with_different_units_stays_two_claims():
|
||
"""1,600 ft y 1,600 m no son el mismo dato, y confundirlos sería peor que
|
||
duplicar: escondería una cifra sin comprobar."""
|
||
spec = {"shots": [{"template": "x", "props": {
|
||
"headline": "1,600 FT ACROSS", "footer": "1,600 m of runway"}}]}
|
||
|
||
figures = [c for c in extract_claims(spec) if c.kind == "figure"]
|
||
|
||
assert {c.unit for c in figures} == {"ft", "m"}
|