researchowl/src/generator/shortspec.py

"""Escritura del shot spec: prompt, inyección del contrato y bucle de reintento.

Generar un spec no es como generar prosa. La prosa mala se lee y se juzga; un
spec malformado no se puede usar. De ahí las tres mitigaciones, cada una en su
sitio: la validación con reintento vive aquí, el comprobador de fundamento en
`grounding.py`, y la revisión humana en el mensaje de Telegram.

El contrato se INYECTA (`GET /templates` → `describe_templates`), no se copia.
Lo que sí es de este repo son las tres formas narrativas: son decisiones
editoriales del canal, no del renderizador.

Sin dependencias de `bot/`: el LLM entra como un callable y el progreso también.
"""
from __future__ import annotations

import json
import re
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any, Awaitable, Callable, Optional

import structlog

from src.generator.spec_contract import (
    SpecInvalid, describe_templates, editorial_notes, estimated_duration,
    validate_spec, NARRATION_WORDS_PER_SECOND,
    TARGET_MAX_DURATION, TARGET_MIN_DURATION,
)

logger = structlog.get_logger()

#: Tres intentos. Si la media de intentos-hasta-válido sube de 1.5, lo que hay
#: que arreglar es el prompt, no este número.
MAX_ATTEMPTS = 3

#: Reescrituras que se gastan en una nota editorial, no en un fallo de contrato.
#: UNA. Un spec que ya cumple el contrato y sólo se pasa de duración es
#: renderizable: la segunda reescritura no compraba un Short mejor, compraba una
#: generación más. Medido sobre las sesiones 166, 167 y 168 — las tres gastaron
#: los tres intentos por duración y las tres acabaron renderizando un spec que
#: seguía pasándose. Los intentos que quedan son para el contrato, que sí es
#: binario. Ver `_how_to_trim` en `spec_contract`: si la nota no se obedece a la
#: primera, lo que hay que arreglar es la nota.
NOTE_ATTEMPTS = 1

#: Cuánta narración cabe en un Short entero. Comprobación cruzada de la regla
#: de arriba, en la unidad que el modelo escribe: 80 palabras son unos 29 s de
#: voz, y con los respiros y algún plano mudo eso deja el vídeo cerca de 40 s.
#: El ejemplo de referencia habla 74. La sesión 167 habló 97 y salió a 47,5 s.
NARRATION_WORD_BUDGET = 80

#: Lo que mide una línea. No es preferencia de estilo: son las líneas del
#: ejemplo (11, 12, 10, 14, 12, 15 palabras). El tope anterior — "menos de 25" —
#: no describía nada que el canal hubiera publicado, y el modelo escribió líneas
#: de 25 y 27 palabras sin saltarse ninguna regla.
NARRATION_WORDS_PER_LINE = 12
NARRATION_WORDS_PER_LINE_MAX = 18

#: Lo que dura un plano como mucho. También del ejemplo (6,0 s el más largo,
#: 5,6 de media). Sin este tope el modelo declaraba 42 s en seis planos de siete
#: segundos y LUEGO les colgaba la narración encima: el primer borrador salía a
#: 50 s las tres veces que se midió, y hacía falta una reescritura entera para
#: bajarlo.
MAX_SHOT_DURATION = 6.0

EXAMPLE_PATH = Path(__file__).parent / "examples" / "jal1628.json"

__all__ = ["ShortSpecWriter", "SpecResult", "SpecWriteFailed", "NARRATIVE_SHAPES"]


#: Las tres formas que se publican de verdad. Dejar la elección libre produce
#: papilla: el modelo elige UNA y la sigue.
NARRATIVE_SHAPES = """\
case_file      — hook → date/place → witness credentials → escalation →
                 evidence → the official explanation and its problem → close.
                 Fits a single documented encounter (JAL 1628, Belgium, Ariel School).

debunk         — the claim → why it spread → the method → the finding →
                 what it means → close.
                 Fits a claim that dissolves under examination (a mislabelled
                 crater video, a star mistaken for a craft).

document_drop  — what was released → the standout item → context →
                 what is still missing → close.
                 Fits a release of records (PURSUE and similar)."""


SHORT_SYSTEM = """\
You write shot specs for The Exclusion Zone, a documentary channel about UAP \
cases and declassified records. A shot spec is JSON that a renderer turns \
directly into a vertical video: every string you write is drawn on screen \
exactly as you typed it.

You answer with ONE JSON object and nothing else — no prose, no explanation, \
no markdown fences.

You never state a figure, a quote, a date or a name that is not in the research \
material you were given. Not even one you happen to know is true. The channel's \
entire premise is that its numbers come from primary sources."""


PROMPT = """\
Write a shot spec for a Short about: "{topic}"

# 1. Pick one narrative shape and follow its arc

{shapes}

# 2. The contract — these are the templates the renderer accepts

Each shot names a template and supplies its props. Nothing outside this list \
exists, and a prop name that is not listed is a parse error, not a nuance.

{templates}

# 3. Rules

- **Total duration {target_min:.0f}-{target_max:.0f} seconds, and the voice is \
what decides it, not the durations you declare.** The contract allows 180; that \
is a ceiling, not a target. A narrated shot runs as long as its line takes to \
say — the renderer never cuts the voice off, it grows the shot — so the whole \
video is really about {word_budget} words of narration and no more. That is the \
number to hold: **count the words of every `narration` you write, and stop at \
{word_budget}.** Section 3b has the arithmetic behind it.
- Typically 6-9 shots, and **none of them longer than {max_shot:.0f} seconds** \
— that is the example's longest, and its average is 5.6. Give a shot the \
seconds its content needs to be read: a card with four rows needs longer than a \
headline. A {max_shot:.0f}-second shot with a short line on it is not a \
generous shot, it is a shot the viewer has already finished reading.
- Every string is drawn as given. Write them the way they should appear: \
SHORT, UPPERCASE, no trailing punctuation. A headline is 2-5 words.
- **"CABE ~N caracteres dibujados" is a width, and it is the one limit nothing \
will catch for you.** Nothing rejects a longer string: the renderer shrinks the \
type until it fits, so a string at twice its budget is drawn at a fraction of \
its size and ends up the smallest text on a frame it was supposed to dominate. \
Stay at or under N. On a quote that means picking a shorter verbatim span, \
never squeezing the whole sentence in.
- Respect every max length and list-length limit above. They are enforced.
- Colours are palette names ({colors}) — never hex.
- Quotes carry the typographic quote marks: “SPLIT RADAR IMAGE”, with U+201C \
and U+201D. Never a straight " inside a string — that closes the JSON string \
and your whole answer becomes unparseable. This is the single most common way \
this task fails.
- meta.id is a lowercase slug (letters, digits, - and _). meta.title is the \
title a human reads, not a filename.
- version is 1. Keep meta at 1080x1920.
- audio.preset — pick the one whose mood fits the shape you chose:
{presets}

# 3b. Narration — the voice-over

Every shot takes an optional `narration`: one or two spoken sentences, read \
aloud by the renderer and burned in as captions. Write it for the ear.

- **The first line is the whole hook.** Two seconds decide whether anyone \
watches the rest, and the opening shot's narration is those two seconds. Lead \
with the strangest true thing you have, not with a preamble.
- **Keep a line to {words_per_line} words, hard stop at {words_per_line_max}.** \
That is the example's own average, and it is not a style preference: a 25-word \
line is three seconds of your whole budget spent on one shot. Long sentences lose the listener and \
stretch the shot; the renderer will not cut your voice off, it will make the \
shot longer instead, and a Short that drifts past {target_max:.0f} seconds is a \
Short people leave.
- **Do not read the screen aloud.** The captions already show your words and \
the template already shows its own. If the shot draws "35,000 FT", the voice \
says what that altitude meant, not the number again.
- Spoken register, not caption register: normal sentence case, ordinary \
punctuation, whole words. The on-screen props are terse and uppercase; the \
narration is a person talking. Write "seventeenth of November" rather than \
"17 NOV" — the voice reads exactly what you type, and it will say "one seven \
N-O-V" if you make it.
- **Most shots carry one.** The example narrates six of its eight and leaves \
silent exactly the two that draw a quotation, where the voice would only be \
competing with words already on the frame. Chosen silence is an edit; a spec \
with one narrated shot out of eight is not a Short with a voice, it is a Short \
that forgot to speak.
- **Give every narrated shot enough time for its own line, and work it out \
rather than guessing.** The voice reads about {words_per_second:.1f} words a \
second and pauses a quarter second at every full stop, so:

      duration ≥ words ÷ {words_per_second:.1f} + half a second

  A twelve-word line needs five seconds; give that shot 5.0, not 4.0. This is \
the one rule that makes your own arithmetic true: a shot runs for the LONGER of \
its declared duration and its line — never shorter, the voice is never cut off \
— so a shot that declares less than its line silently grows, and the video ends \
up longer than the durations you wrote. Hold this rule and the total you \
declare IS the video's length; break it once and nothing you counted means \
anything.
- As a cross-check, all the narration in the spec together should come to about \
{word_budget} words. The example below speaks 74. A spec that spoke 97 rendered \
at 47.5 seconds and had to be cut.
- Everything in section 4 applies to narration word for word. It is prose you \
compose rather than a label you copy, which makes it the easiest place to \
slip in a figure no source gave you — and it is checked exactly like the rest.
- The closing shot carries the domain, uppercase, no protocol: {domain}

# 4. Grounding — this is the part that matters

Every figure, quote, date, and proper noun in your spec must appear in the \
research material below. An automated check runs against these exact sources \
before anything is rendered, and every string it cannot find is shown to a \
human next to your spec.

If the material does not support a number, do not write the number. A shot with \
one solid fact beats a shot with three plausible ones. This applies to the \
worked example in section 5 as much as to your own knowledge: a figure that is \
only in the example is a figure you cannot use.

Anything you put inside quote marks must be a word-for-word span of the \
material. Copy it; do not compress it. "WALNUT SHAPED WIDE RIM" is not a quote \
when the source says "walnut shaped with a wide rim around its circumference" \
— pick a shorter span that is still verbatim, or drop the quote marks and \
state the fact plainly.

**A quote field that takes a list of lines is ONE span, broken where it has to \
break to fit.** It is not two quotes and not two slots to fill. Read the lines \
back joined with a single space: that sentence is what the check looks for in \
the sources, and what a viewer reads off the frame. Welding a real fragment to \
a phrase from somewhere else produces a sentence nobody ever said — \
"“LIKE ALUMINUM" + "SMOOTH, NO WINDOWS”" is a fabricated quote even though \
every word of it appears in the material, because the witness said the first \
half and a later writer summarising him said the second. Attribution makes it \
worse, not better: the line under the quote names the person you just put \
those words into.

# 5. A worked example — FORMAT ONLY

This is a case_file that produced a good video. Read it for shape: how long a \
shot runs, how a headline is worded, which shots speak and which stay silent, \
how the shots build.

It is not source material. Do not reuse its strings, figures, coordinates, \
quotes or waypoints — not even if it covers the same case you were asked \
about. Every value in your spec comes from section 7 and nowhere else. A \
number copied from here is a fabrication, and the grounding check will find it.

{example}

# 6. The article this Short accompanies

{article}

# 7. Research material — the only facts you may use

{context}

Return the JSON object now."""


PALETTE_FALLBACK = "ink, amber, amber_dark, muted, dim, red"

#: Si el caller no trae la paleta de audio, el prompt solo ofrece lo que
#: cualquier shortsmith renderiza.
PRESETS_FALLBACK = {"sonar": "low drone and sonar pings", "none": "digital silence"}


def _describe_presets(presets: dict[str, str]) -> str:
    return "\n".join(f'  "{name}" — {note}' for name, note in sorted(presets.items()))


@dataclass
class SpecResult:
    spec: dict
    attempts: int
    notes: list[str] = field(default_factory=list)
    #: Errores de cada intento fallido, en orden. Sirve de métrica y de pista
    #: cuando un spec sale a la primera pero raro.
    history: list[list[str]] = field(default_factory=list)


class SpecWriteFailed(Exception):
    """Tres intentos y ninguno válido.

    Lleva el último intento aunque no valga: la parte cara es la generación, no
    el render, y un spec inválido se edita a mano y se reenvía. Nunca se tira.
    """

    def __init__(self, errors: list[str], last_raw: str = "",
                 last_spec: Optional[dict] = None, attempts: int = 0):
        self.errors = errors
        self.last_raw = last_raw
        self.last_spec = last_spec
        self.attempts = attempts
        super().__init__("; ".join(errors[:5]) or "no se pudo escribir el spec")


def _load_example() -> str:
    try:
        return json.dumps(json.loads(EXAMPLE_PATH.read_text(encoding="utf-8")),
                          indent=2, ensure_ascii=False)
    except Exception as e:  # nunca bloquea: el ejemplo mejora el prompt, no lo define
        logger.warning("ejemplo de spec no legible — se sigue sin él", error=str(e))
        return "(no example available)"


def _palette(templates: dict[str, dict]) -> str:
    """Los nombres de color, sacados del propio contrato."""
    found: list[str] = []

    def walk(node: Any):
        if isinstance(node, dict):
            enum = node.get("enum")
            if enum and node.get("type") == "string" and "ink" in enum:
                for name in enum:
                    if name not in found:
                        found.append(name)
            for v in node.values():
                walk(v)
        elif isinstance(node, list):
            for v in node:
                walk(v)

    walk(templates)
    return ", ".join(found) or PALETTE_FALLBACK


#: Lo que puede seguir legítimamente al cierre de una cadena JSON.
_AFTER_STRING = set(',:}] \t\r\n')
#: Antes de una comilla de apertura hay hueco, un guion o el propio inicio.
_BEFORE_OPENING = set(' \t\n([-–—‑:')


def _typographic_inner_quotes(body: str) -> str:
    """Convierte en “ ” las comillas rectas que van DENTRO de una cadena JSON.

    Se recorre el texto sabiendo dónde empieza y acaba cada cadena: una `"` que
    no vaya seguida de `,`, `:`, `}`, `]` o espacio no cierra nada, es una
    comilla del texto. Decidir apertura o cierre por el carácter anterior.

    Sólo se llama tras un fallo de parseo: un JSON correcto no pasa por aquí.
    """
    out: list[str] = []
    in_string = False
    escaped = False
    for i, char in enumerate(body):
        if escaped:
            out.append(char)
            escaped = False
            continue
        if char == "\\":
            out.append(char)
            escaped = in_string
            continue
        if char != '"':
            out.append(char)
            continue

        if not in_string:
            in_string = True
            out.append(char)
            continue

        nxt = next((c for c in body[i + 1:] if not c.isspace()), "")
        if nxt in ",:}]" or nxt == "":
            in_string = False
            out.append(char)
        else:
            previous = out[-1] if out else ""
            out.append("“" if previous in _BEFORE_OPENING or previous == '"' else "”")
    return "".join(out)


def extract_json(text: str) -> dict:
    """El objeto JSON de la respuesta del modelo, con o sin valla de markdown.

    Un error de parseo se cuenta CON el trozo que lo provocó. "Expecting ','
    delimiter: line 189 column 22" no le sirve de nada al modelo, que no ve su
    salida numerada; el fragmento sí — y el fallo típico es una comilla recta
    dentro de una cadena, que cierra la cadena antes de tiempo.
    """
    cleaned = text.strip()
    fenced = re.search(r"```(?:json)?\s*(.+?)```", cleaned, re.DOTALL)
    if fenced:
        cleaned = fenced.group(1).strip()
    start, end = cleaned.find("{"), cleaned.rfind("}")
    if start == -1 or end <= start:
        raise ValueError("la respuesta no contiene ningún objeto JSON")
    body = cleaned[start:end + 1]
    try:
        return json.loads(body)
    except json.JSONDecodeError:
        pass

    # Reparación determinista de LA forma en que esto falla: comillas rectas
    # dentro de una cadena (`"quote_a": ""CREDIBLE PEOPLE""`). Medido el
    # 2026-08-01 contra la sesión de Bélgica: el modelo lo repitió en los tres
    # intentos aunque el prompt lo prohíbe y el error se le devolvía con el
    # fragmento. Arreglarlo aquí es además lo que se quiere dibujar: las citas
    # del canal van con las tipográficas.
    repaired = _typographic_inner_quotes(body)
    try:
        return json.loads(repaired)
    except json.JSONDecodeError as e:
        snippet = repaired[max(0, e.pos - 60):e.pos + 60].replace("\n", " ")
        raise ValueError(
            f"{e.msg} — aquí: …{snippet}… "
            "(si es una comilla recta dentro de una cadena, cierra la cadena: "
            "las citas van con las tipográficas “ ”)") from None


def _format_errors(errors: list[str]) -> str:
    """Las rutas, verbatim. Son más útiles para el modelo que cualquier paráfrasis."""
    listed = "\n".join(f"- {e}" for e in errors)
    return (f"\n\n# Your previous attempt was rejected\n\n{listed}\n\n"
            "Fix exactly these and return the corrected JSON object. "
            "Keep everything else as it was.")


def _format_notes(notes: list[str]) -> str:
    listed = "\n".join(f"- {n}" for n in notes)
    return (f"\n\n# Your previous attempt is valid but off-brief\n\n{listed}\n\n"
            "Return the adjusted JSON object.")


def _off_target(spec: dict) -> float:
    """Segundos fuera de la ventana editorial. 0 = dentro."""
    total = estimated_duration(spec)
    return max(0.0, TARGET_MIN_DURATION - total, total - TARGET_MAX_DURATION)


def _closer_to_target(a: Optional[SpecResult], b: SpecResult) -> SpecResult:
    """De dos specs válidos, el que menos se sale del objetivo.

    Antes se guardaba el PRIMERO válido y punto, con lo que una reescritura que
    obedecía la nota a medias — 53 s en vez de 58 — se tiraba entera y salía el
    largo. El empate se lo lleva el anterior: sin razón para cambiar, no se
    cambia.
    """
    if a is None:
        return b
    return a if _off_target(a.spec) <= _off_target(b.spec) else b


#: (system, prompt) -> texto del modelo.
LLMCall = Callable[[str, str], Awaitable[str]]


class ShortSpecWriter:
    def __init__(self, llm_call: LLMCall, templates: dict[str, dict],
                 refresh_templates: Optional[Callable[[], Awaitable[dict]]] = None,
                 presets: Optional[dict[str, str]] = None):
        self.llm_call = llm_call
        self.templates = templates
        #: Se vuelve a pedir el contrato si una validación falla: el
        #: renderizador puede haberse actualizado a mitad de la run.
        self.refresh_templates = refresh_templates
        self.presets = presets or dict(PRESETS_FALLBACK)

    def build_prompt(self, topic: str, context: str, article_url: Optional[str],
                     domain: str) -> str:
        article = (f"The article is published at {article_url} — the Short points at it."
                   if article_url
                   else "No article URL yet. Use the bare domain on the closing shot.")
        return PROMPT.format(
            topic=topic,
            shapes=NARRATIVE_SHAPES,
            templates=describe_templates(self.templates),
            colors=_palette(self.templates),
            presets=_describe_presets(self.presets),
            domain=domain,
            target_min=TARGET_MIN_DURATION,
            target_max=TARGET_MAX_DURATION,
            words_per_second=NARRATION_WORDS_PER_SECOND,
            word_budget=NARRATION_WORD_BUDGET,
            words_per_line=NARRATION_WORDS_PER_LINE,
            words_per_line_max=NARRATION_WORDS_PER_LINE_MAX,
            max_shot=MAX_SHOT_DURATION,
            example=_load_example(),
            article=article,
            context=context,
        )

    async def write(self, topic: str, context: str, *,
                    article_url: Optional[str] = None,
                    domain: str = "THEEXCLUSIONZONE.COM",
                    on_progress: Optional[Callable[[str], Any]] = None) -> SpecResult:
        base_prompt = self.build_prompt(topic, context, article_url, domain)
        feedback = ""
        history: list[list[str]] = []
        last_raw, last_spec = "", None
        #: Un spec que cumple el contrato pero se pasa de duración. Se guarda
        #: para que un intento posterior peor no lo tire: es renderizable.
        best: Optional[SpecResult] = None
        #: Reescrituras ya gastadas en notas editoriales.
        note_rounds = 0

        for attempt in range(1, MAX_ATTEMPTS + 1):
            if on_progress and attempt > 1:
                await _maybe_await(on_progress(
                    f"🎬 Rewriting the shot spec (attempt {attempt}/{MAX_ATTEMPTS})…"))

            last_raw = await self.llm_call(SHORT_SYSTEM, base_prompt + feedback)

            try:
                spec = extract_json(last_raw)
            except (ValueError, json.JSONDecodeError) as e:
                errors = [f"la respuesta no es un objeto JSON válido: {e}"]
                history.append(errors)
                feedback = _format_errors(errors)
                continue

            last_spec = spec
            try:
                validate_spec(spec, self.templates, presets=self.presets)
            except SpecInvalid as e:
                history.append(e.errors)
                feedback = _format_errors(e.errors)
                # El contrato puede haber cambiado bajo los pies: se refresca
                # una vez antes de volver a intentarlo.
                if self.refresh_templates and attempt == 1:
                    try:
                        self.templates = await self.refresh_templates()
                        base_prompt = self.build_prompt(topic, context, article_url, domain)
                    except Exception as refresh_err:
                        logger.warning("no se pudo refrescar el contrato",
                                       error=str(refresh_err))
                continue

            notes = editorial_notes(spec)
            result = SpecResult(spec=spec, attempts=attempt, notes=notes,
                                history=list(history))
            if not notes:
                logger.info("short spec válido", attempts=attempt,
                            shots=len(spec.get("shots", [])), notes=0)
                return result

            best = _closer_to_target(best, result)
            if note_rounds < NOTE_ATTEMPTS and attempt < MAX_ATTEMPTS:
                # Nota editorial, no violación del contrato: se comenta y, si
                # insiste, se renderiza el intento que menos se pase.
                note_rounds += 1
                history.append(notes)
                feedback = _format_notes(notes)
                continue

            logger.info("short spec válido pero fuera de objetivo",
                        attempts=attempt, shots=len(best.spec.get("shots", [])),
                        off_target=round(_off_target(best.spec), 1))
            best.attempts = attempt
            best.history = history
            return best

        if best is not None:
            # Un intento anterior sí cumplía el contrato. Vale más un Short
            # largo que ningún Short.
            logger.info("short spec: se recupera el intento válido anterior",
                        attempts=MAX_ATTEMPTS, notes=best.notes)
            best.attempts = MAX_ATTEMPTS
            best.history = history
            return best

        logger.warning("short spec inválido tras todos los intentos",
                       attempts=MAX_ATTEMPTS, errors=history[-1] if history else [])
        raise SpecWriteFailed(history[-1] if history else ["sin errores registrados"],
                              last_raw=last_raw, last_spec=last_spec,
                              attempts=MAX_ATTEMPTS)


async def _maybe_await(value):
    import asyncio
    if asyncio.iscoroutine(value):
        return await value
    return value