From 20c8d03aa727d7a92d196f5c76f02b348eec0cc8 Mon Sep 17 00:00:00 2001 From: ChemaVX Date: Sat, 1 Aug 2026 21:55:42 +0000 Subject: [PATCH] =?UTF-8?q?feat(short):=20generaci=C3=B3n=20y=20render=20d?= =?UTF-8?q?e=20Shorts=20v=C3=ADa=20shortsmith?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Añade /generate short_en y /short_spec. El pipeline genera un shot spec con Haiku, verifica cada cifra, fecha y cita contra los chunks de la sesión, lo renderiza en shortsmith y entrega el MP4 por Telegram junto a un informe de claims. - ShortsmithClient con sondeo y fallback al spec JSON si el render falla - Contrato de plantillas obtenido de GET /templates, no codificado - Comprobación de fundamento determinista, sin LLM - outputs.published_url para enlazar el artículo de Ghost - Normalización de comillas rectas a tipográficas (ver KNOWN-ISSUES.md) Lo que no aparece en los chunks se contrasta contra el ejemplo del prompt: si casa ahí es fuga, no invención, y se informa como tal. El purgado de sesiones se lleva también su MP4. La subida a YouTube queda fuera a propósito: fase 3. Co-Authored-By: Claude Opus 5 --- .env.example | 8 + KNOWN-ISSUES.md | 44 +++ Makefile | 47 ++- README.md | 41 +++ docs/shortsmith-phase2-spec.md | 291 +++++++++++++++ requirements-dev.txt | 6 + src/bot/bot.py | 212 ++++++++++- src/config.py | 11 + src/db/database.py | 77 +++- src/generator/examples/PROVENANCE.md | 15 + src/generator/examples/jal1628.json | 137 +++++++ src/generator/generator.py | 31 +- src/generator/grounding.py | 533 +++++++++++++++++++++++++++ src/generator/short.py | 281 ++++++++++++++ src/generator/shortsmith.py | 234 ++++++++++++ src/generator/shortspec.py | 411 +++++++++++++++++++++ src/generator/spec_contract.py | 375 +++++++++++++++++++ src/processor/processor.py | 32 +- tests/test_bot_short_report.py | 64 ++++ tests/test_grounding.py | 288 +++++++++++++++ tests/test_layers.py | 35 ++ tests/test_short_golden.py | 134 +++++++ tests/test_short_producer.py | 326 ++++++++++++++++ tests/test_shortsmith_client.py | 208 +++++++++++ tests/test_shortsmith_live.py | 68 ++++ tests/test_shortspec.py | 233 ++++++++++++ tests/test_spec_contract.py | 231 ++++++++++++ 27 files changed, 4350 insertions(+), 23 deletions(-) create mode 100644 docs/shortsmith-phase2-spec.md create mode 100644 requirements-dev.txt create mode 100644 src/generator/examples/PROVENANCE.md create mode 100644 src/generator/examples/jal1628.json create mode 100644 src/generator/grounding.py create mode 100644 src/generator/short.py create mode 100644 src/generator/shortsmith.py create mode 100644 src/generator/shortspec.py create mode 100644 src/generator/spec_contract.py create mode 100644 tests/test_bot_short_report.py create mode 100644 tests/test_grounding.py create mode 100644 tests/test_layers.py create mode 100644 tests/test_short_golden.py create mode 100644 tests/test_short_producer.py create mode 100644 tests/test_shortsmith_client.py create mode 100644 tests/test_shortsmith_live.py create mode 100644 tests/test_shortspec.py create mode 100644 tests/test_spec_contract.py diff --git a/.env.example b/.env.example index eacdfaf..fe8d312 100644 --- a/.env.example +++ b/.env.example @@ -24,6 +24,14 @@ MAX_PAGES_PER_SEARCH=5 REQUEST_DELAY=1.0 # seconds between requests (be polite) MIN_CONTENT_LENGTH=200 +# Shorts (shortsmith) — el renderizador vive en su propio namespace del cluster. +# Los valores por defecto ya apuntan al Service interno: sólo hace falta tocarlo +# para desarrollo local o para apagarlo. +SHORTSMITH_URL=http://shortsmith-svc.shortsmith.svc.cluster.local:8080 +SHORTSMITH_TIMEOUT=600 +SHORTSMITH_ENABLED=true # false = /generate short_en responde que está apagado +SHORTS_DIR=/data/shorts # los MP4 van a disco, nunca a SQLite + # Processing CHUNK_SIZE=800 CHUNK_OVERLAP=100 diff --git a/KNOWN-ISSUES.md b/KNOWN-ISSUES.md index 7ee2d11..9d3f47a 100644 --- a/KNOWN-ISSUES.md +++ b/KNOWN-ISSUES.md @@ -81,3 +81,47 @@ Mitigations now in place: If a research still dies, the scraped sources survive in the DB: `/process` re-chunks and scores them without re-scraping. + +## Un shot spec escrito por Haiku falla de dos maneras concretas + +Medido el 2026-08-01 generando Shorts de verdad contra sesiones reales (JAL +1628 #153, Bélgica #158). Las dos están mitigadas, pero conviene saber que +existen porque las dos son silenciosas si nadie mira. + +**1. Comillas rectas dentro de una cadena JSON.** El modelo escribe +`"quote_a": ""CREDIBLE PEOPLE. THEY TOLD CLEARLY WHAT THEY SAW.""` y la cadena +se cierra en la segunda comilla: el JSON entero deja de parsear. Se repitió en +los tres intentos aunque el prompt lo prohíbe explícitamente y aunque el error +se le devolvía con el fragmento exacto. **No se arregla insistiendo**: lo +arregla `_typographic_inner_quotes()` en `src/generator/shortspec.py`, que +convierte esas comillas en `“ ”` recorriendo el texto con estado de cadena. +Sólo se ejecuta después de un fallo de parseo, así que un JSON correcto no pasa +por ahí. Además es lo que se quiere dibujar: las citas del canal van con +tipográficas. + +**2. Se copian cifras del ejemplo del prompt.** El `examples/jal1628.json` que +va en el prompt como referencia de formato es también una fuente de datos muy +tentadora. En la primera eval dorada, tres claims del spec generado venían del +ejemplo y no de las fuentes: `232 FT` (largo de un 747) y `40 YEARS` no +aparecían en NINGUNO de los 126 chunks de la sesión, y `RARELY, IF EVER` estaba +en 1 chunk que no entró en el top-40 que vio el modelo. La sección 5 del prompt +lo dice ahora en mayúsculas ("FORMAT ONLY … a number copied from here is a +fabrication") y eso bajó de 3 a 1. + +El ejemplo se queda con cifras reales a propósito — uno sintético enseña peor +la forma —, así que la defensa es estructural: `check_grounding()` contrasta lo +que no encuentra en los chunks **contra el propio ejemplo**, y lo que casa ahí +sale en el informe como `🧪 copiado del EJEMPLO del prompt (fuga, no +invención)`. Son dos diagnósticos y piden dos acciones: una invención hay que +verificarla, una fuga hay que borrarla. + +Corolario: **el informe de claims no es decorativo**. Si algún día se manda el +vídeo sin él, se estará publicando lo que el modelo recuerde del ejemplo. + +## El comprobador de fundamento sólo mira los chunks que vio el modelo + +`check_grounding()` compara contra los mismos ~40 chunks que se metieron en el +prompt, no contra los 126 de la sesión. Es deliberado — la pregunta es "¿lo +sacó de lo que le dimos?" — pero produce falsos positivos cuando el dato existe +en la sesión y no entró en el top-k. Un falso positivo cuesta un vistazo; un +falso negativo cuesta la credibilidad del canal. diff --git a/Makefile b/Makefile index bc5661f..57f4ddb 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,52 @@ VENDORED := src/seo/rules.py HEADER := src/seo/_vendor_header.py MARKER := BEGIN VENDORED seo_rules.py -.PHONY: sync-seo check-seo-sync +.PHONY: sync-seo check-seo-sync test golden-db golden shortsmith-live + +test: ## Suite completa (los tests vivos se saltan solos) + python3 -m pytest tests/ -q + +# --- Shorts: pruebas contra el servicio vivo --------------------------------- +# El renderizador no se expone fuera del cluster, así que desde el nodo se le +# habla por la ClusterIP. Los dos targets de abajo la resuelven solos. + +SHORTSMITH_IP = $(shell kubectl get svc shortsmith-svc -n shortsmith \ + -o jsonpath='{.spec.clusterIP}' 2>/dev/null) +GOLDEN_SESSION ?= 153 +GOLDEN_DB ?= /tmp/researchowl-golden.db + +golden-db: ## Extrae UNA sesión de la DB de producción a un fichero pequeño + @test -n "$(SHORTSMITH_IP)" || echo "aviso: shortsmith-svc no encontrado" + kubectl exec -n researchowl deployment/researchowl -- python3 -c "\ +import sqlite3, os; \ +src = sqlite3.connect('file:/data/researchowl.db?immutable=1', uri=True); \ +out = '/tmp/golden.db'; os.path.exists(out) and os.remove(out); \ +src.execute('ATTACH DATABASE ? AS g', (out,)); \ +src.executescript('''CREATE TABLE g.research_sessions AS SELECT * FROM research_sessions WHERE 0; \ +CREATE TABLE g.sources AS SELECT * FROM sources WHERE 0; \ +CREATE TABLE g.chunks AS SELECT * FROM chunks WHERE 0; \ +CREATE TABLE g.outputs AS SELECT * FROM outputs WHERE 0; \ +CREATE TABLE g.api_usage AS SELECT * FROM api_usage WHERE 0;'''); \ +[src.execute(f'INSERT INTO g.{t} SELECT * FROM {t} WHERE ' + ('id=$(GOLDEN_SESSION)' if t=='research_sessions' else 'session_id=$(GOLDEN_SESSION)')) \ + for t in ('research_sessions','sources','chunks','outputs')]; \ +src.commit(); print('sesión $(GOLDEN_SESSION) ->', os.path.getsize(out), 'bytes')" + kubectl cp -n researchowl \ + $$(kubectl get pod -n researchowl -o name | head -1 | cut -d/ -f2):/tmp/golden.db \ + $(GOLDEN_DB) + @echo "escrito en $(GOLDEN_DB)" + +golden: ## Eval dorada (GASTA dinero: una llamada a Claude + un render) + @test -f $(GOLDEN_DB) || { echo "falta $(GOLDEN_DB) — ejecuta 'make golden-db'"; exit 1; } + RESEARCHOWL_GOLDEN_DB=$(GOLDEN_DB) \ + RESEARCHOWL_GOLDEN_SESSION=$(GOLDEN_SESSION) \ + SHORTSMITH_LIVE_URL=http://$(SHORTSMITH_IP):8080 \ + SHORTS_DIR=/tmp/researchowl-shorts \ + python3 -m pytest tests/test_short_golden.py -v -s + +shortsmith-live: ## Fontanería contra el shortsmith vivo (renderiza el ejemplo) + SHORTSMITH_LIVE_URL=http://$(SHORTSMITH_IP):8080 \ + python3 -m pytest tests/test_shortsmith_live.py -v -s + sync-seo: ## Re-copy canonical seo_rules.py into the vendored file + record hash @test -f "$(CANON)" || { echo "canonical not found at $(CANON)"; exit 1; } diff --git a/README.md b/README.md index 877d071..bac29e4 100644 --- a/README.md +++ b/README.md @@ -39,9 +39,50 @@ OutputGenerator (Ollama) | `/status` | Check progress | | `/finish` | Stop early, proceed to generation | | `/generate podcast\|blog\|report\|thread` | Generate output | +| `/generate short_en` | Vertical Short: shot spec → grounding check → MP4 | +| `/short_spec` | Last shot spec as a JSON file, to hand-edit and re-render | | `/sources` | List all sources found | | `/cancel` | Cancel current research | +## Shorts (`/generate short_en`) + +Claude writes a **shot spec** — typed JSON, not prose — which +[shortsmith](https://git.chemavx.xyz/chemavx/shortsmith) renders into a 1080×1920 +MP4. The bot sends the video and, in a separate message, a **claims report**. + +``` +/research JAL 1628 Alaska 1986 … +/generate blog en → Ghost draft, article URL stored on the output row +/generate short_en → spec → grounding → render → video + claims report + (YouTube upload is deliberately manual) +``` + +Three things make this different from generating text, and each has its own +mitigation: + +- **It is a contract, not prose.** The template schemas are fetched live from + `GET /templates` and never copied here, so a template added to shortsmith is + available immediately. A spec is validated locally against those schemas + before anything renders, and the exact error paths + (`shots.0.radar_sweep.props.sweeeps`) go back to the model verbatim — up to 3 + attempts. +- **It contains figures and quotes.** `grounding.py` extracts every quote, + figure, date and proper noun and checks it against the exact chunks the model + was given. No LLM in that path: normalisation plus substring, deterministic + and free. Whatever is not in the chunks is checked against the worked example + that travels in the prompt, so a figure lifted from it is reported as a + **prompt leak**, not as an invention — different diagnosis, different fix. + Neither ever blocks the render: both are surfaced next to the video and a + human decides. +- **It becomes a published video.** Nothing is uploaded anywhere. The MP4 lands + in Telegram for review, and in `/data/shorts/{session_id}.mp4`. + +Fallbacks hold throughout: if shortsmith is unreachable, the job errors, or the +spec never validates, the spec JSON comes back as a file. The expensive part is +the generation, not the render. + +Full spec of the phase: `docs/shortsmith-phase2-spec.md`. + ## Local Development ```bash diff --git a/docs/shortsmith-phase2-spec.md b/docs/shortsmith-phase2-spec.md new file mode 100644 index 0000000..3d6a7aa --- /dev/null +++ b/docs/shortsmith-phase2-spec.md @@ -0,0 +1,291 @@ +# Phase 2 — researchowl → shortsmith integration + +**Handoff document for Claude Code.** Prerequisite: shortsmith v1 deployed and healthy +(`shortsmith-svc.shortsmith.svc.cluster.local:8080`). + +- **Repo touched:** `git.chemavx.xyz/chemavx/researchowl` only. shortsmith is not modified. +- **Deliverable:** `/generate short_en` produces a rendered MP4 from a research session + and delivers it to Telegram for human review. +- **Explicitly out of scope:** YouTube upload. That is phase 3. See §11. + +--- + +## 1. What changes + +``` +/research + ↓ +/generate blog en → Ghost article published, URL stored + ↓ +/generate short_en → Haiku writes a shot spec (JSON) + → grounding check against source chunks + → POST to shortsmith, poll, fetch MP4 + → Telegram: video + claims report + → human reviews, uploads to YouTube manually +``` + +The Ghost step comes first and is a hard dependency: the Short's description links to +the article, so the article URL must exist before the spec is generated. §6 covers what +happens when it doesn't. + +--- + +## 2. Why this is not like generating prose + +Every other output type in `generator.py` produces text a human reads and judges. A shot +spec is different in three ways, and each needs its own mitigation: + +| Property | Consequence | Mitigation | +|---|---|---| +| It's a typed contract, not prose | Malformed output is unusable, not merely poor | Validation retry loop, §4 | +| It contains figures and quotes | These are exactly what an LLM fabricates | Grounding check, §5 | +| It becomes a published video | An error is public and hard to retract | Human review gate, §8 | + +The grounding check is the one that matters most. The channel's entire premise is that +its numbers come from primary sources. A fabricated radar figure in a 40-second video is +worse than no video. + +--- + +## 3. Prompt construction — fetch the contract, don't hardcode it + +shortsmith exposes `GET /templates`, which returns each template's prop schema. **Fetch +it at generation time and inject it into the prompt.** Do not copy the schemas into +researchowl. + +This means adding a template to shortsmith makes it immediately available to the +generator with no change here. Hardcoding the schemas would create a second source of +truth that silently drifts — the same class of failure as the ffmpeg version delta that +caused the v1 OOM. + +Cache the response for the lifetime of the process; refetch on validation failure, in +case the renderer was updated mid-run. + +### Narrative shapes + +Free template choice produces mush. Give the model three shapes matching the three +article types actually published, and have it pick one: + +| Shape | Arc | Fits | +|---|---|---| +| `case_file` | hook → date/place → witness credentials → escalation → evidence → official explanation and its problem → close | JAL 1628, Belgium, Ariel School | +| `debunk` | the claim → why it spread → the method → the finding → what it means → close | Roswell crater video, Yellow Sea star | +| `document_drop` | what was released → the standout item → context → what's still missing → close | PURSUE releases | + +`examples/jal1628.json` in the shortsmith repo is a worked `case_file`. Include it in the +prompt as a full example — one concrete example is worth more than any amount of +description. + +### Constraints to state explicitly + +- Total duration **20–45 s**. Not the 180 s contract ceiling; that is a hard limit, not a + target. +- Per-template `max_length` limits exist and are enforced. Listing them in the prompt + turns a rejection into a non-event. +- Colours are palette names (`ink`, `amber`, `amber_dark`, `muted`, `dim`, `red`), never + hex. +- Every figure and quote must come from the supplied chunks. No outside knowledge, even + if correct. + +--- + +## 4. Validation retry loop + +```python +for attempt in range(3): + spec = await _generate_spec(prompt, feedback) + try: + validated = await client.validate(spec) # POST /render dry-run or local pydantic + break + except ValidationError as e: + feedback = _format_errors(e) # feed the exact paths back +else: + return _fallback(spec) # §6 +``` + +shortsmith's discriminated union produces precise error locations +(`shots.0.radar_sweep.props.sweeeps`, or `shots.0` with the valid template names listed). +**Feed those paths back verbatim.** They are more useful to the model than any +paraphrase. + +Cap at 3 attempts. Log attempts-to-valid as a metric — if it trends above 1.5, the prompt +needs work, not the retry limit. + +--- + +## 5. Grounding check — the important part + +After the spec validates and **before** rendering, verify every factual string in it +appears in the session's source material. + +```python +def check_grounding(spec, chunks) -> list[Ungrounded]: + """Extract figures and quoted strings from spec props, confirm each + appears in at least one source chunk.""" +``` + +**What to extract from the spec:** + +- Every quoted string (`quote_a`, `quote_b`, `quote`, anything in `“ ”`) +- Every number with a unit or magnitude (`35,000 FT`, `1,500`, `~1,600 – 2,000 FT`, `50 minutes`) +- Every date (`17 NOV 1986`, `5 MARCH 1987`) +- Proper nouns in `label`/`key` positions (`ELMENDORF ROCC`, `CAPT. KENJU TERAUCHI`) + +**Matching:** normalise both sides — case-fold, strip thousands separators, collapse +whitespace, normalise quote glyphs and dashes. Then substring match against chunk text. +No LLM in this path: it must be deterministic and free. + +**On failure:** do not silently drop the shot and do not retry blindly. Return the spec +plus the list of ungrounded strings, and surface them in Telegram (§8). A human decides +whether it is a real fabrication or a formatting artefact. + +Expect false positives at first — `"twice the size of an aircraft carrier"` appears in +the source but a rephrasing would not match. That is the correct bias: a false positive +costs a glance, a false negative costs the channel's credibility. + +This is the automated version of the fact-check table that was written by hand for the +first Short. That table is in `short-01-jal1628-script.md` if you want the shape of the +output. + +--- + +## 6. ShortsmithClient + +Mirror the `GhostPublisher` shape in `generator.py`. Layer rule holds: `generator/` does +not import from `bot/`; progress is reported through a generic callable. + +```python +class ShortsmithClient: + def __init__(self, base_url: str, timeout: float = 600.0) + async def templates(self) -> dict + async def render(self, spec: dict) -> str # -> job_id + async def poll(self, job_id, on_progress=None) -> JobResult + async def fetch_video(self, job_id) -> bytes +``` + +**Polling:** 2 s interval, 10 min ceiling. A 42 s Short renders in ~32 s; the 180 s +ceiling takes ~138 s. Anything past 10 min is a stuck job, not a slow one. + +**Fallbacks always** (repo convention). If shortsmith is unreachable, or the job errors, +or grounding fails hard — **return the spec JSON to Telegram as a file**. The expensive +part is the generation, not the render. Never discard it. + +**Config** (`src/config.py`, Pydantic Settings, env-direct — no secret): + +``` +SHORTSMITH_URL = http://shortsmith-svc.shortsmith.svc.cluster.local:8080 +SHORTSMITH_TIMEOUT = 600 +SHORTSMITH_ENABLED = true +``` + +`SHORTSMITH_ENABLED=false` must make `/generate short_en` reply that the feature is off, +not crash. This is the kill switch if the renderer misbehaves while nobody is watching. + +--- + +## 7. Database + +No migrations (`CREATE TABLE IF NOT EXISTS` convention holds). + +- `outputs` takes `output_type='short_en'`, `content` = the spec JSON as text. +- **New:** the Ghost article URL must be retrievable. Check whether `GhostPublisher` + already persists it; if not, store it on the `outputs` row for the blog post, or add a + `published_url` column to `outputs` (nullable, `ALTER TABLE` guarded by a column check). + The spec generator needs it for the description. +- Store the rendered MP4 **on disk**, not in SQLite. `/data/shorts/{session_id}.mp4`. + Blobs in SQLite will make the WAL pathological. + +--- + +## 8. Telegram flow + +`/generate short_en` — reuse `ProgressReporter`, editing a single message: + +``` +🎬 Writing shot spec… (Haiku, ~5 s) +🔍 Checking claims against sources… +🎞 Rendering… 40% (progress from shortsmith poll) +📤 Uploading… +``` + +Then send the MP4 as a **video message** (not a document, so it plays inline), with a +caption carrying the title and the article URL. + +**Immediately after, send the claims report as a separate message.** This is the review +gate and it must be impossible to miss: + +``` +✅ 11 claims matched to sources +⚠️ 2 not found: + • "roughly 1,600 feet across" + • "NORAD confirmed" + +Sources: 14 chunks from 9 URLs +Cost: $0.004 +``` + +Zero ungrounded claims still sends the report, saying so. A silent success trains the +reader to stop looking. + +Also add `/short_spec` to return the last spec JSON as a file, for hand-editing and +re-rendering without regenerating. + +--- + +## 9. Cost + +One Haiku call over the top-scored chunks. ~$0.003–0.008, plus retries. Rendering is free +(own hardware). A Short costs roughly what a `/generate blog` costs, which for practical +purposes is nothing — the constraint on volume is review time, not money. + +--- + +## 10. Tests + +| Area | Assert | +|---|---| +| Spec generation | Mocked Haiku response validates; malformed response triggers retry with error paths fed back; 3 failures fall through to fallback | +| Grounding | Known-good spec over known chunks yields zero ungrounded; a spec with an injected fabricated figure flags exactly that string; normalisation handles thousands separators, curly quotes, en-dashes | +| Client | Poll loop handles queued→running→done, error status, timeout, connection refused | +| Fallback | Every failure path returns the spec JSON rather than nothing | +| Layer separation | `grep` that `generator/` does not import from `bot/` | + +**Golden eval, worth building once:** run the generator against the stored JAL 1628 +session and compare the output structurally to `examples/jal1628.json` — shape count, +templates chosen, total duration, zero ungrounded claims. Not string equality; the model +will phrase differently. It answers "could this pipeline have produced the video we +already know is good?" + +--- + +## 11. Out of scope — phase 3 + +YouTube upload via Data API v3. Deliberately excluded: it needs OAuth with a stored +refresh token in `researchowl-secrets` (managed imperatively), a new failure surface, and +it removes the human from the loop at exactly the point where the human is most valuable. + +Ship phase 2, publish five or six Shorts by hand, then decide whether the review step is +actually a bottleneck. It probably is not. + +`short_es` for Zona de Exclusión is nearly free once this works — shortsmith draws +whatever strings it is given and does not care about language. Only the prompt and the +narrative shapes need translating. Do it after `short_en` has produced something worth +publishing, not before. + +--- + +## 12. Implementation order + +One change at a time, verified before the next. + +1. `ShortsmithClient` + config + tests, against the live service. No generation yet — + prove the plumbing by POSTing `examples/jal1628.json` and getting the MP4 back. +2. Grounding checker + tests, standalone. Test it against the known-good JAL 1628 spec + and against a deliberately corrupted copy. +3. Spec generation: prompt, `GET /templates` injection, retry loop. +4. Wire `output_type='short_en'` into `generator.py`; article URL retrieval. +5. Telegram `/generate short_en` and `/short_spec`. +6. Golden eval against the JAL 1628 session. + +**Step 2 before step 3 is deliberate.** Build the check before the thing it checks, so +the first generated spec is graded by a checker that was written without knowledge of it. diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..1d6cf3f --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,6 @@ +# Solo para ejecutar la suite en local — NO va en la imagen (el Dockerfile +# instala requirements.txt y nada más; la CI no corre pytest). +# pip install -r requirements-dev.txt && make test +-r requirements.txt +pytest>=8.0 +pytest-asyncio>=0.24 diff --git a/src/bot/bot.py b/src/bot/bot.py index a91a4dd..66905cf 100644 --- a/src/bot/bot.py +++ b/src/bot/bot.py @@ -152,6 +152,8 @@ async def cmd_start(update: Update, ctx: ContextTypes.DEFAULT_TYPE): "`/generate ` — Generate output\n" " Tipos: podcast|blog|report|thread\n" " Extended: podcast_extended|blog_extended|report_extended\n" + "`/generate short_en` — Short vertical (vídeo) + informe de claims\n" + "`/short_spec` — Último shot spec como fichero JSON\n" "`/sources` — List all sources found\n" "`/outputs` — List generated outputs\n" "`/export` — Exportar último output como PDF\n" @@ -285,6 +287,12 @@ async def cmd_generate(update: Update, ctx: ContextTypes.DEFAULT_TYPE): # Telegram, writes a bare draft). Global default still comes from SEO_AUTOFILL. seo_override = "dryrun" if ("dry" in rest or "dryrun" in rest) else None + # El Short no es un output de texto: sale del pipeline de shortsmith y se + # entrega como vídeo + informe de claims. Se desvía antes del type_map. + if output_arg in ("short_en", "short", "corto"): + await cmd_short(update, ctx) + return + type_map = { "podcast": OutputType.PODCAST, "blog": OutputType.BLOG, @@ -301,7 +309,7 @@ async def cmd_generate(update: Update, ctx: ContextTypes.DEFAULT_TYPE): if output_arg not in type_map: await update.message.reply_text( "❌ Invalid output type.\n" - "Use: `/generate podcast|blog|report|thread`", + "Use: `/generate podcast|blog|report|thread|short_en`", parse_mode=ParseMode.MARKDOWN ) return @@ -421,6 +429,177 @@ async def cmd_generate(update: Update, ctx: ContextTypes.DEFAULT_TYPE): await db_conn.close() +async def _session_row(db_conn, chat_id: int): + """La sesión activa del chat si la hay, si no la más reciente.""" + session_id = _active_sessions.get(chat_id) + if session_id: + cursor = await db_conn.execute( + "SELECT * FROM research_sessions WHERE id = ?", (session_id,)) + else: + cursor = await db_conn.execute( + """SELECT * FROM research_sessions WHERE telegram_chat_id = ? + ORDER BY created_at DESC LIMIT 1""", (chat_id,)) + row = await cursor.fetchone() + return dict(row) if row else None + + +def _claims_message(result) -> str: + """El informe de claims: la puerta de revisión humana. + + Se manda SIEMPRE y como mensaje aparte, incluso con cero avisos. Un éxito + silencioso enseña al lector a dejar de mirar. En texto plano a propósito: + lleva citas y comillas del modelo, y un Markdown desbalanceado haría que + Telegram rechazara justo el mensaje que no puede faltar. + """ + lines = [] + if result.grounding: + lines.append(result.grounding.summary()) + else: + lines.append("⚠️ Sin comprobación de fundamento: no se llegó a escribir un spec.") + + if result.render_warnings: + lines.append("") + lines.append(f"✂️ {len(result.render_warnings)} textos recortados al dibujar:") + for w in result.render_warnings[:5]: + lines.append(f" • [{w.get('template', '?')}] {str(w.get('text', ''))[:60]}") + + if result.notes: + lines.append("") + lines.extend(f"📏 {n}" for n in result.notes) + + lines.append("") + if result.duration_s: + lines.append(f"Duración: {result.duration_s:.0f}s · " + f"{len(result.spec.get('shots', []))} shots · " + f"intentos hasta válido: {result.attempts}") + lines.append(f"Coste: ${result.cost_usd:.4f}") + if not result.article_url: + lines.append("⚠️ Esta sesión no tiene URL de artículo: publica antes el blog " + "(`/generate blog en`) para que el Short pueda enlazarlo.") + return "\n".join(lines) + + +async def _send_spec_file(message, result, session_id: int, reason: str): + """Fallback universal: el spec vuelve como fichero pase lo que pase. + + La parte cara es la generación, no el render. Un spec que no se pudo + renderizar se edita a mano y se reenvía; uno que se tira hay que pagarlo + otra vez. + """ + import io + payload = result.spec_json or result.raw_response + if not payload: + await message.reply_text(f"❌ {reason}\n(no hay ni spec que devolver)") + return + suffix = "json" if result.spec else "txt" + await message.reply_document( + document=io.BytesIO(payload.encode("utf-8")), + filename=f"short_{session_id}_spec.{suffix}", + caption=f"⚠️ Sin vídeo — {reason[:800]}", + ) + + +async def cmd_short(update: Update, ctx: ContextTypes.DEFAULT_TYPE): + """`/generate short_en` — spec → fundamento → render → vídeo a revisar. + + La subida a YouTube NO entra aquí: es fase 3, y el humano de en medio es + justo lo más valioso del proceso. + """ + if not is_authorized(update.effective_user.id): + return + + chat_id = update.effective_chat.id + db_conn = await get_db() + db = ResearchDB(db_conn) + + try: + session = await _session_row(db_conn, chat_id) + if not session: + await update.message.reply_text( + "No research sessions found. Start with /research ") + return + session_id = session["id"] + + from src.generator.short import ShortProducer, ShortsDisabled + + reporter = ProgressReporter(update.message) + await reporter.start(f"🎬 Writing shot spec for: {session['topic']}") + + producer = ShortProducer(db, ContentProcessor(db, OllamaClient())) + try: + result = await producer.produce(session_id, reporter.update) + except ShortsDisabled: + await reporter.done( + "🚫 Los Shorts están desactivados (`SHORTSMITH_ENABLED=false`).") + return + + if result.has_video: + await reporter.done("✅ Short renderizado") + caption = f"🎬 {result.title}" + if result.article_url: + caption += f"\n{result.article_url}" + caption += f"\n\n{session['topic']} · {result.duration_s:.0f}s" + with open(result.video_path, "rb") as f: + await update.message.reply_video( + video=f, + filename=f"short_{session_id}.mp4", + caption=caption[:1024], + supports_streaming=True, + write_timeout=180, + ) + else: + await reporter.done("⚠️ Short sin vídeo — te devuelvo el spec") + await _send_spec_file(update.message, result, session_id, + result.failure or "razón desconocida") + + # Informe de claims: SIEMPRE, y en su propio mensaje. + await update.message.reply_text(_claims_message(result)) + + except Exception as e: + logger.error("Short generation failed", error=str(e), exc_info=True) + await update.message.reply_text(f"❌ Short failed: {str(e)[:300]}") + finally: + await db_conn.close() + + +async def cmd_short_spec(update: Update, ctx: ContextTypes.DEFAULT_TYPE): + """Devuelve el último shot spec como fichero, para editarlo a mano y + volver a renderizar sin pagar otra generación.""" + if not is_authorized(update.effective_user.id): + return + + chat_id = update.effective_chat.id + db_conn = await get_db() + db = ResearchDB(db_conn) + + try: + session = await _session_row(db_conn, chat_id) + if not session: + await update.message.reply_text("No sessions found.") + return + + output = await db.get_latest_output(session["id"], OutputType.SHORT_EN) + if not output: + await update.message.reply_text( + "No hay ningún shot spec en esta sesión. Genera uno con " + "`/generate short_en`.", parse_mode=ParseMode.MARKDOWN) + return + + import io + from datetime import datetime + created = datetime.utcfromtimestamp(output["created_at"]).strftime("%Y-%m-%d %H:%M") + await update.message.reply_document( + document=io.BytesIO(output["content"].encode("utf-8")), + filename=f"short_{session['id']}_spec.json", + caption=f"🎬 Shot spec — {session['topic']}\n{created} UTC", + ) + except Exception as e: + logger.error("short_spec failed", error=str(e)) + await update.message.reply_text(f"❌ {str(e)[:200]}") + finally: + await db_conn.close() + + async def cmd_sources(update: Update, ctx: ContextTypes.DEFAULT_TYPE): if not is_authorized(update.effective_user.id): return @@ -1062,7 +1241,15 @@ async def cmd_export(update: Update, ctx: ContextTypes.DEFAULT_TYPE): if chosen: break if not chosen: - chosen = outputs[0] + # Un short_en es JSON, no prosa: maquetarlo en PDF no tiene sentido. + # Se usa /short_spec para eso. + prose = [o for o in outputs if o["output_type"] != OutputType.SHORT_EN] + if not prose: + await update.message.reply_text( + "El único output de esta sesión es un shot spec. " + "Úsalo con `/short_spec`.", parse_mode=ParseMode.MARKDOWN) + return + chosen = prose[0] msg = await update.message.reply_text( f"📄 Generando PDF para `{topic}`…", @@ -1136,7 +1323,8 @@ async def cmd_purge(update: Update, ctx: ContextTypes.DEFAULT_TYPE): f"🗑️ Purged: {result['sessions']} sessions, " f"{result['sources']} sources, " f"{result['chunks']} chunks, " - f"{result['outputs']} outputs" + f"{result['outputs']} outputs, " + f"{result.get('shorts', 0)} vídeos" ) except Exception as e: logger.error("Purge command failed", error=str(e)) @@ -1200,7 +1388,14 @@ async def cmd_publish(update: Update, ctx: ContextTypes.DEFAULT_TYPE): if chosen: break if not chosen: - chosen = outputs[-1] + # Nunca un short_en: su contenido es el shot spec en JSON. + prose = [o for o in outputs if o["output_type"] != OutputType.SHORT_EN] + if not prose: + await update.message.reply_text( + "El único output de esta sesión es un shot spec — eso no se " + "publica en Ghost.") + return + chosen = prose[-1] msg = await update.message.reply_text("📤 Publicando en Ghost como borrador…") @@ -1209,6 +1404,14 @@ async def cmd_publish(update: Update, ctx: ContextTypes.DEFAULT_TYPE): post = result["posts"][0] admin_url = f"{ghost.url}/ghost/#/editor/post/{post['id']}" + # La URL pública queda apuntada en la fila del output: el Short la + # necesita después para enlazar al artículo (best-effort). + if post.get("slug"): + try: + await db.set_output_url(chosen["id"], f"{ghost.url}/{post['slug']}/") + except Exception as e: + logger.warning("No se pudo guardar la URL del artículo", error=str(e)) + await msg.edit_text( f"✅ *Publicado en Ghost como borrador*\n\n" f"📝 Título: `{title}`\n" @@ -1380,6 +1583,7 @@ def create_bot() -> Application: app.add_handler(CommandHandler("status", cmd_status)) app.add_handler(CommandHandler("finish", cmd_finish)) app.add_handler(CommandHandler("generate", cmd_generate)) + app.add_handler(CommandHandler("short_spec", cmd_short_spec)) app.add_handler(CommandHandler("sources", cmd_sources)) app.add_handler(CommandHandler("outputs", cmd_outputs)) app.add_handler(CommandHandler("news", cmd_news)) diff --git a/src/config.py b/src/config.py index 92142bb..27fe613 100644 --- a/src/config.py +++ b/src/config.py @@ -68,6 +68,17 @@ class Settings(BaseSettings): ghost_url_en: str = Field("") ghost_api_key_en: str = Field("") + # shortsmith (renderizador de Shorts) — env directo, sin secreto. + # El bot habla con el Service interno; el renderizador no se expone fuera + # del cluster. shortsmith_enabled=false es el kill switch: /generate short_en + # responde "desactivado" en vez de fallar. + shortsmith_url: str = Field("http://shortsmith-svc.shortsmith.svc.cluster.local:8080") + shortsmith_timeout: float = Field(600.0) + shortsmith_enabled: bool = Field(True) + # MP4 renderizados: en disco, NUNCA en SQLite (blobs en la DB hacen + # patológico el WAL — misma razón que source_contents guarda texto y ya). + shorts_dir: str = Field("/data/shorts") + # SEO autofill — "off" | "on" | "dryrun" (default off). # off = today's exact behavior (bare draft, no second LLM call). # on = adds best-effort meta/OG/Twitter/tags/internal-links to the DRAFT diff --git a/src/db/database.py b/src/db/database.py index 7af66b9..b0edd7c 100644 --- a/src/db/database.py +++ b/src/db/database.py @@ -29,6 +29,9 @@ class OutputType(str, Enum): REPORT_EXTENDED = "report_extended" BLOG_EXTENDED = "blog_extended" PODCAST_EXTENDED = "podcast_extended" + # El contenido de un short_en NO es prosa: es el shot spec (JSON) que + # shortsmith convierte en vídeo. El MP4 vive en disco, nunca en SQLite. + SHORT_EN = "short_en" SCHEMA = """ @@ -79,7 +82,8 @@ CREATE TABLE IF NOT EXISTS outputs ( session_id INTEGER NOT NULL REFERENCES research_sessions(id), output_type TEXT NOT NULL, content TEXT NOT NULL, - created_at REAL NOT NULL + created_at REAL NOT NULL, + published_url TEXT -- URL del artículo publicado (blog -> Ghost) ); CREATE TABLE IF NOT EXISTS source_contents ( @@ -175,12 +179,33 @@ async def _init_shared() -> aiosqlite.Connection: # antes de fallar con "database is locked". await conn.execute("PRAGMA busy_timeout=5000") await conn.executescript(SCHEMA) + await _ensure_columns(conn) await conn.commit() _shared_conn = conn logger.info("Shared DB connection initialized", path=settings.db_path) return _shared_conn +#: Columnas añadidas después de que la tabla existiera en producción. El +#: `CREATE TABLE IF NOT EXISTS` no toca una tabla ya creada, así que una +#: columna nueva necesita su ALTER — guardado por PRAGMA table_info para que +#: sea idempotente. No es una migración: no hay versiones ni orden, sólo +#: "¿existe la columna? si no, créala". +_ADDED_COLUMNS: dict[str, dict[str, str]] = { + "outputs": {"published_url": "TEXT"}, +} + + +async def _ensure_columns(conn: aiosqlite.Connection) -> None: + for table, columns in _ADDED_COLUMNS.items(): + async with conn.execute(f"PRAGMA table_info({table})") as cur: + existing = {row[1] for row in await cur.fetchall()} + for name, decl in columns.items(): + if name not in existing: + await conn.execute(f"ALTER TABLE {table} ADD COLUMN {name} {decl}") + logger.info("Columna añadida", table=table, column=name) + + async def get_db() -> aiosqlite.Connection: conn = await _init_shared() return _SharedConnection(conn) @@ -399,6 +424,40 @@ class ResearchDB: row = await cur.fetchone() return row[0] if row else None + async def set_output_url(self, output_id: int, url: str) -> None: + """Guarda la URL pública del artículo en su fila de `outputs`. + + Hace falta porque el Short enlaza al artículo en su descripción, y hasta + ahora la URL sólo viajaba en el aviso de Telegram: se perdía en cuanto + se cerraba la conversación. + """ + await self.db.execute( + "UPDATE outputs SET published_url = ? WHERE id = ?", (url, output_id)) + await self.db.commit() + + async def get_latest_output(self, session_id: int, + output_type: Optional[str] = None) -> Optional[dict]: + query = "SELECT * FROM outputs WHERE session_id = ?" + params: list = [session_id] + if output_type: + query += " AND output_type = ?" + params.append(output_type) + query += " ORDER BY created_at DESC LIMIT 1" + cursor = await self.db.execute(query, params) + row = await cursor.fetchone() + return dict(row) if row else None + + async def get_article_url(self, session_id: int) -> Optional[str]: + """La URL del artículo publicado más reciente de la sesión, si la hay.""" + cursor = await self.db.execute( + """SELECT published_url FROM outputs + WHERE session_id = ? AND published_url IS NOT NULL AND published_url != '' + ORDER BY created_at DESC LIMIT 1""", + (session_id,) + ) + row = await cursor.fetchone() + return row[0] if row else None + async def get_outputs(self, session_id: int) -> list[dict]: cursor = await self.db.execute( "SELECT * FROM outputs WHERE session_id = ? ORDER BY created_at DESC", @@ -592,9 +651,23 @@ class ResearchDB: ) session_ids = [row[0] for row in await cursor.fetchall()] - counts = {"sessions": 0, "sources": 0, "chunks": 0, "outputs": 0, "api_usage": 0} + counts = {"sessions": 0, "sources": 0, "chunks": 0, "outputs": 0, + "api_usage": 0, "shorts": 0} for sid in session_ids: + # El MP4 del Short vive en disco (los blobs en SQLite hacen + # patológico el WAL), así que su borrado no lo arrastra ninguna FK: + # se hace aquí, que es el único sitio que sabe qué sesiones + # desaparecen. Best-effort — un fichero que no se puede borrar no + # va a impedir purgar la sesión. + try: + video = Path(settings.shorts_dir) / f"{sid}.mp4" + if video.is_file(): + video.unlink() + counts["shorts"] += 1 + except OSError as e: + logger.warning("No se pudo borrar el Short de una sesión purgada", + session_id=sid, error=str(e)) await self.db.execute( "DELETE FROM source_contents WHERE source_id IN (SELECT id FROM sources WHERE session_id = ?)", (sid,) diff --git a/src/generator/examples/PROVENANCE.md b/src/generator/examples/PROVENANCE.md new file mode 100644 index 0000000..e1b54d1 --- /dev/null +++ b/src/generator/examples/PROVENANCE.md @@ -0,0 +1,15 @@ +# Ejemplos vendorizados + +`jal1628.json` es una copia de `examples/jal1628.json` del repo +`git.chemavx.xyz/chemavx/shortsmith` (sha256 +`eb3fe669b714e56db669638d3c02c25aca6df04823dcbc5b1da1a3cb938efda1`, copiado el +2026-08-01). + +Se copia **el ejemplo, no el contrato**. Los esquemas de props se leen en vivo +de `GET /templates` (ver `src/generator/shortsmith.py`): duplicarlos aquí sería +una segunda fuente de verdad. Este fichero es material de prompt — el +`case_file` que ya produjo un vídeo bueno — y el patrón de referencia de la +eval dorada. + +Si el ejemplo cambia en shortsmith, recopiarlo es opcional: que se desincronice +solo empeora un poco el prompt, no rompe nada. diff --git a/src/generator/examples/jal1628.json b/src/generator/examples/jal1628.json new file mode 100644 index 0000000..9ce0190 --- /dev/null +++ b/src/generator/examples/jal1628.json @@ -0,0 +1,137 @@ +{ + "version": 1, + "meta": { + "id": "jal1628", + "title": "JAL 1628: Three Radars, One Object, Zero Explanation", + "width": 1080, + "height": 1920, + "fps": 30, + "theme": "exclusion-zone" + }, + "audio": { + "preset": "sonar", + "silence": [[31.0, 36.0]] + }, + "shots": [ + { + "template": "radar_sweep", + "duration": 4.0, + "props": { + "headline": "3 RADARS", + "subline": "1 UNEXPLAINED RETURN", + "contact_bearing_deg": 210, + "sweeps": 2 + } + }, + { + "template": "track_map", + "duration": 4.0, + "props": { + "headline": "17 NOV 1986", + "subline": "35,000 FT · 600 MPH", + "waypoints": [ + {"label": "FORT YUKON", "lat": 66.57, "lon": -145.27}, + {"label": "FAIRBANKS", "lat": 64.84, "lon": -147.72}, + {"label": "TALKEETNA", "lat": 62.32, "lon": -150.11}, + {"label": "ANCHORAGE", "lat": 61.22, "lon": -149.90} + ], + "bounds": {"lat_min": 60.4, "lat_max": 67.4, "lon_min": -152.0, "lon_max": -143.5} + } + }, + { + "template": "data_card", + "duration": 5.0, + "props": { + "card_title": "FLIGHT CREW", + "rows": [ + {"key": "CAPT. KENJU TERAUCHI", "value": "PILOT IN COMMAND"}, + {"key": "EX-FIGHTER PILOT", "value": "JASDF"}, + {"key": "29 YEARS", "value": "FLYING EXPERIENCE"}, + {"key": "10,000+", "value": "FLIGHT HOURS"} + ], + "footer": "REPORTS TWO LIGHTS PACING THE AIRCRAFT" + } + }, + { + "template": "scale_bars", + "duration": 5.0, + "props": { + "headline": "REPORTED SCALE", + "bars": [ + {"label": "BOEING 747", "value": 232, "unit": "FT", "color": "ink"}, + { + "label": "ESTIMATED OBJECT", + "value": 2000, + "unit": "FT", + "color": "amber", + "value_label": "~1,600 – 2,000 FT" + } + ], + "quote": ["“TWICE THE SIZE OF", "AN AIRCRAFT CARRIER”"], + "attribution": "— CAPT. TERAUCHI, ESTIMATE" + } + }, + { + "template": "orbit_track", + "duration": 6.0, + "props": { + "headline": "EVASIVE MANEUVER", + "subline": "360° TURN · −4,000 FT", + "legend": [ + {"label": "JAL 1628", "color": "ink"}, + {"label": "UNIDENTIFIED CONTACT", "color": "amber"} + ], + "caption": "CONTACT HOLDS RELATIVE POSITION", + "turn_deg": 360 + } + }, + { + "template": "signal_strips", + "duration": 7.0, + "props": { + "headline": "THREE INDEPENDENT SOURCES", + "strips": [ + { + "label": "ONBOARD RADAR", + "sublabel": "CONTACT 7–8 NM · 10 O'CLOCK", + "markers": [0.26, 0.48, 0.63, 0.81] + }, + { + "label": "ANCHORAGE CENTER", + "sublabel": "PRIMARY RETURNS THROUGH TURNS", + "markers": [0.26, 0.48, 0.63, 0.81] + }, + { + "label": "ELMENDORF ROCC", + "sublabel": "TRACKED “FLIGHT OF TWO”", + "markers": [0.26, 0.48, 0.63, 0.81] + } + ], + "footnote": "FAIRBANKS RADAR: NOTHING" + } + }, + { + "template": "document_quote", + "duration": 5.0, + "props": { + "source": "FAA · 5 MARCH 1987", + "label_a": "OFFICIAL FINDING:", + "quote_a": "“SPLIT RADAR IMAGE”", + "label_b": "AARTCC CONTROLLER:", + "quote_b": "“RARELY, IF EVER”", + "tail": "IN THAT AIRSPACE." + } + }, + { + "template": "counter_close", + "duration": 6.0, + "props": { + "count_to": 1500, + "count_label": "PAGES OF FAA DOCUMENTATION", + "lines": ["40 YEARS", "STILL OPEN"], + "url": "THEEXCLUSIONZONE.COM", + "show_mark": true + } + } + ] +} diff --git a/src/generator/generator.py b/src/generator/generator.py index 93dcd29..6b2d0a4 100644 --- a/src/generator/generator.py +++ b/src/generator/generator.py @@ -651,8 +651,27 @@ class OutputGenerator: # never buried inside the long .md document). None on the flag-off path. self.last_publish_notice: str | None = None + async def _remember_article_url(self, ghost: "GhostPublisher", post: dict, + output_id: int | None) -> None: + """Guarda en la fila de `outputs` la URL que tendrá el artículo. + + Se construye desde el slug (`{sitio}/{slug}/`) y no desde el `url` que + devuelve Ghost, porque el post es un DRAFT y ese campo trae la URL de + previsualización. Si Jose cambia el slug al publicar, el enlace habrá + que rehacerlo — el Short lo enseña en la descripción, no en el vídeo. + + Best-effort: nunca bloquea la publicación. + """ + if output_id is None or not post.get("slug"): + return + try: + await self.db.set_output_url(output_id, f"{ghost.url}/{post['slug']}/") + except Exception as e: + logger.warning("No se pudo guardar la URL del artículo", error=str(e)) + async def _publish_blog_to_ghost(self, lang: str, full_output: str, topic: str, - session_id: int, seo_override: str | None) -> str: + session_id: int, seo_override: str | None, + output_id: int | None = None) -> str: """Publish a blog DRAFT to Ghost, gated by the SEO autofill mode. Returns the ghost_notice to APPEND to the returned document (flag-off / @@ -703,6 +722,7 @@ class OutputGenerator: self.last_publish_notice = ( _seo_live_message(ghost, post, seo, inserted_pairs) + collision_note) + await self._remember_article_url(ghost, post, output_id) logger.info("Auto-published blog to Ghost", mode=mode, post_id=post["id"], links=len(inserted_pairs)) return "" @@ -720,6 +740,7 @@ class OutputGenerator: try: result = await ghost.publish_draft(title, full_output) post = result["posts"][0] + await self._remember_article_url(ghost, post, output_id) logger.info("Auto-published blog to Ghost (bare)", post_id=post["id"]) return _bare_ghost_notice(ghost, post) + collision_note except Exception as e: @@ -797,13 +818,13 @@ class OutputGenerator: full_output = header + "\n\n" + output # Save to DB - await self.db.save_output(session_id, output_type, full_output) + output_id = await self.db.save_output(session_id, output_type, full_output) # Auto-publish to Ghost for blog outputs (autofill mode gated inside helper). ghost_notice = "" if output_type in (OutputType.BLOG, OutputType.BLOG_EXTENDED): ghost_notice = await self._publish_blog_to_ghost( - lang, full_output, topic, session_id, seo_override) + lang, full_output, topic, session_id, seo_override, output_id) logger.info("Output generated", type=output_type, length=len(full_output)) return full_output + ghost_notice @@ -960,13 +981,13 @@ class OutputGenerator: header = self._build_header(topic, output_type, session, stats) full_output = header + "\n\n" + full_content - await self.db.save_output(session_id, output_type, full_output) + output_id = await self.db.save_output(session_id, output_type, full_output) # Auto-publish to Ghost for extended blog outputs (autofill mode gated inside). ghost_notice = "" if output_type == OutputType.BLOG_EXTENDED: ghost_notice = await self._publish_blog_to_ghost( - lang, full_output, topic, session_id, seo_override) + lang, full_output, topic, session_id, seo_override, output_id) logger.info("Extended output generated", type=output_type, sections=len(sections), length=len(full_output)) diff --git a/src/generator/grounding.py b/src/generator/grounding.py new file mode 100644 index 0000000..b62a8d9 --- /dev/null +++ b/src/generator/grounding.py @@ -0,0 +1,533 @@ +"""Comprobador de fundamento: ¿cada dato del spec sale de las fuentes? + +Un spec de Short no es prosa que un humano juzgue al leerla: es un contrato +tipado que se convierte en vídeo publicado. La premisa entera del canal es que +sus cifras vienen de fuentes primarias, así que una cifra de radar inventada en +un vídeo de 40 segundos es peor que no publicar vídeo. + +Esto extrae del spec todo lo que afirma un hecho — citas, cifras, fechas y +nombres propios — y confirma que aparece en al menos un chunk de la sesión. + +**Aquí no entra ningún LLM.** Tiene que ser determinista y gratis: si el +comprobador alucinara, no comprobaría nada. Todo es normalización + substring. + +El sesgo es deliberado: se esperan falsos positivos (una cita reformulada no +casa aunque el hecho esté en la fuente). Un falso positivo cuesta un vistazo; +un falso negativo cuesta la credibilidad del canal. +""" +from __future__ import annotations + +import json +import re +import unicodedata +from dataclasses import dataclass, field +from functools import lru_cache +from pathlib import Path +from typing import Any, Iterable, Optional + +__all__ = [ + "Claim", + "GroundingReport", + "check_grounding", + "extract_claims", + "normalize", +] + +#: El ejemplo trabajado que viaja en el prompt. Se contrasta contra él para +#: poder distinguir dos diagnósticos que no piden lo mismo (ver +#: `check_grounding`). +EXAMPLE_PATH = Path(__file__).parent / "examples" / "jal1628.json" + +# --- normalización ---------------------------------------------------------- + +#: Glifos de comilla que hay que unificar antes de comparar: el spec lleva +#: tipográficas («“ ”») y las fuentes scrapeadas, cualquier cosa. +_QUOTE_GLYPHS = "“”„‟«»″ʺ" +_APOSTROPHES = "‘’ʼ′" +_DASHES = "‐‑‒–—―−" +_SEPARATORS = "·•∙|\t   " + +#: Separador de millares dentro de un número: 35,000 y 35.000 -> 35000. El +#: lookahead exige exactamente tres dígitos, así que 1.5 y 61.22 se quedan +#: como están. "1.234" en el sentido decimal se convertiría en 1234, que es un +#: precio asumido: en este dominio los millares son mucho más frecuentes. +_THOUSANDS = re.compile(r"(?<=\d)[.,](?=\d{3}(?!\d))") + + +def normalize(text: str) -> str: + """Forma canónica para comparar los dos lados. Idempotente.""" + if not text: + return "" + out = unicodedata.normalize("NFKC", text) + out = out.translate({ord(c): '"' for c in _QUOTE_GLYPHS}) + out = out.translate({ord(c): "'" for c in _APOSTROPHES}) + out = out.translate({ord(c): "-" for c in _DASHES}) + out = out.translate({ord(c): " " for c in _SEPARATORS}) + out = _THOUSANDS.sub("", out) + out = out.casefold() + return " ".join(out.split()) + + +# --- vocabulario ------------------------------------------------------------ + +#: Unidades reconocidas -> alias equivalentes. Sirven para dos cosas: decidir +#: si un número "lleva unidad" (y por tanto afirma algo) y para casar "35,000 +#: FT" con una fuente que escribe "35,000 feet". +_UNIT_ALIASES: dict[str, set[str]] = { + "ft": {"ft", "feet", "foot", "pies", "pie"}, + "m": {"m", "meter", "meters", "metre", "metres", "metro", "metros"}, + "km": {"km", "kilometer", "kilometers", "kilometre", "kilometres", + "kilometro", "kilometros"}, + "mi": {"mi", "mile", "miles", "milla", "millas"}, + "nm": {"nm", "nmi", "nautical", "naut"}, + "mph": {"mph"}, + "kt": {"kt", "kts", "knot", "knots", "nudo", "nudos"}, + "kph": {"kph", "kmh"}, + "%": {"%", "percent", "pct", "porciento"}, + "deg": {"deg", "degree", "degrees", "grado", "grados", "°"}, + "sec": {"sec", "secs", "second", "seconds", "segundo", "segundos"}, + "min": {"min", "mins", "minute", "minutes", "minuto", "minutos"}, + "hour": {"hour", "hours", "hr", "hrs", "hora", "horas"}, + "day": {"day", "days", "dia", "dias"}, + "week": {"week", "weeks", "semana", "semanas"}, + "month": {"month", "months", "mes", "meses"}, + "year": {"year", "years", "yr", "yrs", "ano", "anos"}, + "page": {"page", "pages", "pagina", "paginas"}, + "kg": {"kg", "kilo", "kilos", "kilogram", "kilograms"}, + "lb": {"lb", "lbs", "pound", "pounds", "libra", "libras"}, + "ton": {"ton", "tons", "tonne", "tonnes", "tonelada", "toneladas"}, + "mhz": {"mhz"}, + "ghz": {"ghz"}, + "km2": {"km2"}, +} +_UNIT_LOOKUP: dict[str, str] = { + alias: canon for canon, aliases in _UNIT_ALIASES.items() for alias in aliases +} + +_MONTHS: dict[str, int] = {} +for _i, _names in enumerate([ + ("january", "jan", "enero", "ene"), + ("february", "feb", "febrero"), + ("march", "mar", "marzo"), + ("april", "apr", "abril", "abr"), + ("may", "mayo"), + ("june", "jun", "junio"), + ("july", "jul", "julio"), + ("august", "aug", "agosto", "ago"), + ("september", "sep", "sept", "septiembre", "setiembre"), + ("october", "oct", "octubre"), + ("november", "nov", "noviembre"), + ("december", "dec", "diciembre", "dic"), +], start=1): + for _n in _names: + _MONTHS[_n] = _i + +#: Palabras función que no identifican nada. Se usan sólo para el respaldo por +#: tokens de los nombres: un nombre cuyos tokens significativos aparecen todos +#: en las fuentes se da por fundamentado aunque la frase entera no case. +_STOPWORDS = { + "the", "a", "an", "and", "or", "of", "in", "on", "at", "to", "for", "with", + "by", "from", "as", "is", "was", "were", "are", "be", "been", "that", + "this", "these", "those", "it", "its", "his", "her", "their", "no", "not", + "el", "la", "los", "las", "un", "una", "unos", "unas", "de", "del", "y", + "o", "en", "con", "por", "para", "que", "se", "su", "sus", "al", "es", + "son", "fue", "fueron", "lo", +} + +#: Claves cuyo valor de texto es una etiqueta identificadora — donde viven los +#: nombres propios ("ELMENDORF ROCC", "CAPT. KENJU TERAUCHI"). La prosa +#: (headline, caption, footnote…) no entra entera: de ella se sacan citas, +#: cifras y fechas, que es lo que afirma hechos. +_NAME_KEYS = {"label", "key", "card_title", "source", "attribution", + "sublabel", "count_label"} + +#: Claves numéricas que AFIRMAN un dato. El resto de números del spec son +#: geometría o tiempo de render (lat, lon, duration, sweeps, turn_deg, +#: contact_bearing_deg, markers, bounds…) y no se comprueban: no dicen nada +#: sobre el mundo. Si una plantilla nueva añade un número que sí afirma algo, +#: se añade aquí. +_FACT_NUMBER_KEYS = {"value", "count_to"} + +#: Listas de texto que forman UNA frase continua al dibujarse (el renderizador +#: no envuelve: el caller parte la cita en líneas). Se unen antes de comprobar. +_JOINED_LIST_KEYS = {"quote"} + + +# --- extracción ------------------------------------------------------------- + +@dataclass(frozen=True) +class Claim: + """Un dato afirmado por el spec, con dónde vive.""" + text: str # tal cual aparece en el spec, para enseñárselo a un humano + kind: str # quote | figure | date | name + path: str # shots.3.scale_bars.props.bars.1.value_label + unit: Optional[str] = None # canónica, cuando la cifra la lleva + _date: Optional[tuple] = None # (dia|None, mes, año) para el respaldo de fechas + + @property + def norm(self) -> str: + return normalize(self.text) + + +_QUOTED = re.compile(r'["“„‟«]([^"“”„‟«»]{3,})' + r'["”„‟»]') +_DATE_DMY = re.compile(r"\b(\d{1,2})\s+([A-Za-zÀ-ž]{3,12})\.?,?\s+(\d{4})\b") +_DATE_MDY = re.compile(r"\b([A-Za-zÀ-ž]{3,12})\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(\d{4})\b") +_DATE_ISO = re.compile(r"\b(\d{4})-(\d{2})-(\d{2})\b") +_DATE_SLASH = re.compile(r"\b(\d{1,2})[/](\d{1,2})[/](\d{2,4})\b") +_YEAR = re.compile(r"\b(1[5-9]\d{2}|20\d{2})\b") +_NUMBER = re.compile( + r"(?P\d{1,3}(?:[.,]\d{3})+(?:\.\d+)?|\d+(?:\.\d+)?)\s*" + r"(?P%|°|[A-Za-zÀ-ž]{1,10})?") +_TOKEN = re.compile(r"[A-Za-zÀ-ž][A-Za-zÀ-ž'.\-]*") + + +def _mask(text: str, start: int, end: int) -> str: + """Tapa un tramo ya extraído para que no lo vuelva a coger otra regla.""" + return text[:start] + " " * (end - start) + text[end:] + + +def _month_number(token: str) -> Optional[int]: + return _MONTHS.get(normalize(token).strip(". ")) + + +def _claims_from_text(text: str, path: str, key: str) -> list[Claim]: + """Todo lo que afirma un hecho dentro de una cadena del spec. + + El orden importa: las fechas se extraen y se tapan antes que los números, + porque si no "17 NOV 1986" produciría además la cifra suelta 1986. + """ + if not text or not text.strip(): + return [] + claims: list[Claim] = [] + rest = text + + # 1. Citas entrecomilladas. Son verbatim por definición: se comprueban tal cual. + for m in _QUOTED.finditer(text): + inner = m.group(1).strip() + if inner: + claims.append(Claim(inner, "quote", path)) + rest = _mask(rest, m.start(), m.end()) + + # 2. Fechas, con sus componentes para el respaldo (día+mes+año en el mismo chunk). + for regex, order in ((_DATE_DMY, "dmy"), (_DATE_MDY, "mdy"), + (_DATE_ISO, "ymd"), (_DATE_SLASH, "dmy_num")): + for m in regex.finditer(rest): + if order == "dmy": + day, month, year = m.group(1), _month_number(m.group(2)), m.group(3) + elif order == "mdy": + month, day, year = _month_number(m.group(1)), m.group(2), m.group(3) + elif order == "ymd": + year, month, day = m.group(1), int(m.group(2)), m.group(3) + else: + day, month, year = m.group(1), int(m.group(2)), m.group(3) + if not 1 <= month <= 12: + continue + if not month: + continue # "5 RADARS 1986" no es una fecha: el token no es un mes + claims.append(Claim(m.group(0).strip(), "date", path, + _date=(int(day), int(month), int(year)))) + rest = _mask(rest, m.start(), m.end()) + + # 3. Cifras: las que llevan unidad reconocida, o las de magnitud (separador + # de millares o >= 1000). "3 RADARS" no afirma una medida y se deja pasar; + # "35,000 FT", "1,500" y "50 minutes" sí. + for m in _NUMBER.finditer(rest): + raw_num, raw_unit = m.group("num"), m.group("unit") + unit = _UNIT_LOOKUP.get(normalize(raw_unit or "")) + had_separator = bool(re.search(r"\d[.,]\d{3}", raw_num)) + try: + magnitude = float(normalize(raw_num)) + except ValueError: + continue + if not unit and not had_separator and magnitude < 1000: + continue + if not unit or not raw_unit: + shown = raw_num + elif raw_unit in "%°": + shown = f"{raw_num}{raw_unit}" # 360°, no 360 ° + else: + shown = f"{raw_num} {raw_unit}" + claims.append(Claim(shown.strip(), "figure", path, unit=unit)) + rest = _mask(rest, m.start(), m.end("num") if not unit else m.end()) + + # 4. Años sueltos que hayan sobrevivido ("40 YEARS" no; "SINCE 1986" sí). + for m in _YEAR.finditer(rest): + claims.append(Claim(m.group(0), "date", path, + _date=(None, None, int(m.group(0))))) + rest = _mask(rest, m.start(), m.end()) + + # 5. Nombres propios: sólo en posiciones de etiqueta, y sólo si queda algo + # que identifique. "10,000+" (sin letras) y "29 YEARS" (cuya única + # palabra es una unidad) ya viajaron como cifra; repetirlos como nombre + # sólo alarga el informe. + if key in _NAME_KEYS and _name_worth_checking(text): + claims.append(Claim(text.strip(), "name", path)) + + return claims + + +def _name_worth_checking(text: str) -> bool: + """¿Queda algún token que identifique a alguien o algo? Las unidades y los + meses no cuentan: ya viajan dentro de la cifra o de la fecha.""" + return any(t not in _UNIT_LOOKUP and t not in _MONTHS + for t in _significant_tokens(text)) + + +def _walk(node: Any, path: str, key: str, claims: list[Claim]) -> None: + if isinstance(node, dict): + for k, v in node.items(): + _walk(v, f"{path}.{k}", k, claims) + elif isinstance(node, list): + if key in _JOINED_LIST_KEYS and all(isinstance(x, str) for x in node): + # Una cita partida en líneas es UNA cita. + _walk(" ".join(node), path, key, claims) + return + for i, v in enumerate(node): + _walk(v, f"{path}.{i}", key, claims) + elif isinstance(node, str): + claims.extend(_claims_from_text(node, path, key)) + elif isinstance(node, bool): + return + elif isinstance(node, (int, float)) and key in _FACT_NUMBER_KEYS: + claims.append(Claim(_pretty_number(node), "figure", path)) + + +def _pretty_number(value: float) -> str: + return str(int(value)) if float(value).is_integer() else str(value) + + +def _attach_units(props: dict, claims: list[Claim], base_path: str) -> list[Claim]: + """Un dict con `value` numérico y `unit` de texto (una barra de escala) dibuja + los dos juntos: "232 FT". Se detecta por forma, no por plantilla.""" + out = [] + for claim in claims: + if claim.kind == "figure" and claim.path == f"{base_path}.value" and props.get("unit"): + out.append(Claim(f"{claim.text} {props['unit']}", "figure", claim.path, + unit=_UNIT_LOOKUP.get(normalize(str(props["unit"]))))) + else: + out.append(claim) + return out + + +def extract_claims(spec: dict) -> list[Claim]: + """Todos los datos afirmados por los shots del spec, sin duplicados. + + `meta` queda fuera a propósito: el título del spec no se dibuja en ningún + fotograma, es el nombre del fichero. + """ + claims: list[Claim] = [] + for i, shot in enumerate(spec.get("shots") or []): + if not isinstance(shot, dict): + continue + template = shot.get("template", "?") + props = shot.get("props") or {} + base = f"shots.{i}.{template}.props" + shot_claims: list[Claim] = [] + _walk(props, base, "props", shot_claims) + # Reconstruye "232 FT" a partir de {value: 232, unit: "FT"}. + for path_prefix, sub in _dicts_with_value_and_unit(props, base): + shot_claims = _attach_units(sub, shot_claims, path_prefix) + claims.extend(shot_claims) + + seen: set[tuple[str, str]] = set() + unique: list[Claim] = [] + for claim in claims: + fingerprint = (claim.kind, claim.norm) + if not claim.norm or fingerprint in seen: + continue + seen.add(fingerprint) + unique.append(claim) + return unique + + +def _dicts_with_value_and_unit(node: Any, path: str) -> Iterable[tuple[str, dict]]: + if isinstance(node, dict): + if isinstance(node.get("value"), (int, float)) and node.get("unit"): + yield path, node + for k, v in node.items(): + yield from _dicts_with_value_and_unit(v, f"{path}.{k}") + elif isinstance(node, list): + for i, v in enumerate(node): + yield from _dicts_with_value_and_unit(v, f"{path}.{i}") + + +# --- comprobación ----------------------------------------------------------- + +@dataclass +class GroundingReport: + grounded: list[Claim] = field(default_factory=list) + ungrounded: list[Claim] = field(default_factory=list) + #: Ni en las fuentes ni inventado: copiado del ejemplo del prompt. + contaminated: list[Claim] = field(default_factory=list) + chunk_count: int = 0 + url_count: int = 0 + + @property + def total(self) -> int: + return len(self.grounded) + len(self.ungrounded) + len(self.contaminated) + + @property + def unsupported(self) -> list[Claim]: + """Todo lo que no se apoya en las fuentes, sea cual sea el diagnóstico.""" + return self.ungrounded + self.contaminated + + @property + def clean(self) -> bool: + return not self.unsupported + + def summary(self) -> str: + """Texto del informe de claims (§8 del spec de fase 2). + + Con cero claims sin fundamento TAMBIÉN se informa: un éxito silencioso + enseña al lector a dejar de mirar. + """ + lines = [f"✅ {len(self.grounded)} claims casados con las fuentes"] + if self.ungrounded: + lines.append(f"⚠️ {len(self.ungrounded)} sin encontrar:") + for claim in self.ungrounded: + lines.append(f" • [{claim.kind}] \"{claim.text}\"") + else: + lines.append("✅ 0 sin encontrar") + if self.contaminated: + lines.append("") + lines.append(f"🧪 {len(self.contaminated)} copiados del EJEMPLO del prompt " + "(fuga, no invención — bórralos o sustitúyelos por datos " + "de esta sesión):") + for claim in self.contaminated: + lines.append(f" • [{claim.kind}] \"{claim.text}\"") + lines.append("") + lines.append(f"Fuentes: {self.chunk_count} chunks de {self.url_count} URLs") + return "\n".join(lines) + + +def _unit_near(haystack: str, number: str, unit: str, window: int = 40) -> bool: + """¿Aparece la unidad (o un alias) cerca de esa cifra en el texto?""" + aliases = _UNIT_ALIASES.get(unit, {unit}) + for m in re.finditer(rf"(? bool: + return re.search(rf"(? bool: + """Respaldo de fechas: "17 NOV 1986" contra una fuente que escribe + "November 17, 1986". Los tres componentes en el mismo chunk bastan.""" + day, month, year = parts + if str(year) not in haystack: + return False + if month is None: + return True + names = [n for n, num in _MONTHS.items() if num == month] + if not any(re.search(rf"\b{n}", haystack) for n in names) and \ + not re.search(rf"(? list[str]: + """Tokens que identifican algo. La puntuación de cola se cae para que + "CAPT." case por substring contra "captain".""" + tokens = (t.strip(".-'") for t in _TOKEN.findall(normalize(text))) + return [t for t in tokens if len(t) >= 3 and t not in _STOPWORDS] + + +def _is_grounded(claim: Claim, haystacks: list[str]) -> bool: + needle = claim.norm + if any(needle in h for h in haystacks): + return True + + if claim.kind == "quote": + return False # una cita o es verbatim o no es una cita + + if claim.kind == "figure": + number = normalize(claim.text.split()[0]) if claim.text else "" + if not number: + return False + if claim.unit: + return any(_unit_near(h, number, claim.unit) for h in haystacks) + return any(_number_present(h, number) for h in haystacks) + + if claim.kind == "date": + return any(_date_present(h, claim._date) for h in haystacks) if claim._date else False + + # name: cada token significativo tiene que aparecer en las fuentes. Casa + # "CAPT. KENJU TERAUCHI" con "Captain Kenju Terauchi" sin dejar pasar un + # "ELMENDORF ROCC" donde ninguna fuente menciona ROCC. + tokens = _significant_tokens(claim.text) + if not tokens: + return False + return all(any(t in h for h in haystacks) for t in tokens) + + +@lru_cache(maxsize=1) +def _example_haystacks() -> tuple[str, ...]: + """El ejemplo del prompt, troceado para poder buscar dentro. + + Se junta lo que el ejemplo DIBUJA (sus claims, con la cita reconstruida) y + sus cadenas sueltas. **Un trozo por campo, no un texto único**: el respaldo + de fechas casa día, mes y año dentro del MISMO pajar, y en un ejemplo con + "17 NOV 1986" y "5 MARCH 1987" pegados, un "17 NOV 1987" inventado parecería + venir de ahí. + """ + try: + example = json.loads(EXAMPLE_PATH.read_text(encoding="utf-8")) + except Exception: + return () + pieces = [c.text for c in extract_claims(example)] + for shot in example.get("shots", []): + _collect_strings(shot.get("props") or {}, pieces) + return tuple(sorted({normalize(p) for p in pieces if normalize(p)})) + + +def _collect_strings(node: Any, out: list[str]) -> None: + if isinstance(node, dict): + for v in node.values(): + _collect_strings(v, out) + elif isinstance(node, list): + for v in node: + _collect_strings(v, out) + elif isinstance(node, str) and node.strip(): + out.append(node) + + +def check_grounding(spec: dict, chunks: list[dict], + example_haystacks: Optional[tuple[str, ...]] = None + ) -> GroundingReport: + """Comprueba el spec contra el material de la sesión. + + `chunks` son filas de la tabla `chunks` (con `content` y, si viene del join + con `sources`, `url`). No se descarta ningún shot ni se reintenta a ciegas: + se devuelve el informe y decide un humano si es una fabricación real o un + artefacto de formato. + + Lo que no aparece en los chunks se contrasta ADEMÁS contra el ejemplo + trabajado que viaja en el prompt. Si casa ahí, no es una invención: es una + fuga del prompt — el modelo copió una cifra del ejemplo en vez de sacarla + de la sesión (medido el 2026-08-01: "232 FT", el largo de un 747, en cero + de 126 chunks). Son dos diagnósticos distintos y piden acciones distintas: + una invención hay que verificarla, una fuga hay que borrarla. + + El ejemplo se queda como está — con cifras reales — a propósito: uno + sintético enseña peor la forma, y la defensa estructural es esto. + """ + haystacks = [normalize(c.get("content") or "") for c in chunks] + haystacks = [h for h in haystacks if h] + urls = {c.get("url") for c in chunks if c.get("url")} + from_example = _example_haystacks() if example_haystacks is None else example_haystacks + + report = GroundingReport(chunk_count=len(chunks), url_count=len(urls)) + for claim in extract_claims(spec): + if _is_grounded(claim, haystacks): + report.grounded.append(claim) + elif from_example and _is_grounded(claim, list(from_example)): + report.contaminated.append(claim) + else: + report.ungrounded.append(claim) + return report diff --git a/src/generator/short.py b/src/generator/short.py new file mode 100644 index 0000000..9c7a53a --- /dev/null +++ b/src/generator/short.py @@ -0,0 +1,281 @@ +"""Producción de un Short: spec → fundamento → render → MP4 en disco. + +Orquesta las tres piezas que ya existen (`shortspec`, `grounding`, +`shortsmith`) y no añade lógica propia salvo el orden, que es deliberado: + + escribir el spec → comprobar los datos → renderizar + +La comprobación va ANTES del render porque el informe de claims es la puerta de +revisión humana, y llega a Telegram junto al vídeo. No bloquea el render: un +dato sin encontrar puede ser una fabricación o un artefacto de formato, y eso +lo decide una persona, no esto. + +**Fallbacks siempre** (convención del repo): si shortsmith no responde, si el +job falla o si el spec no valida, se devuelve el spec igualmente. La parte cara +es la generación, no el render. No se tira nunca. +""" +from __future__ import annotations + +import json +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Callable, Optional + +import structlog + +from src.config import settings +from src.db.database import OutputType, ResearchDB +from src.generator.grounding import GroundingReport, check_grounding +from src.generator.shortsmith import ( + ShortsmithClient, ShortsmithError, ShortsmithRejected, ShortsmithUnavailable, +) +from src.generator.shortspec import ShortSpecWriter, SpecWriteFailed +from src.llm import get_anthropic_client + +logger = structlog.get_logger() + +__all__ = ["ShortProducer", "ShortResult", "ShortsDisabled"] + +#: Cuántos chunks se le dan al modelo. Es también el material contra el que se +#: comprueba el fundamento: se comprueba contra EXACTAMENTE lo que se le pasó. +CONTEXT_CHUNKS = 40 +#: Tope de caracteres del contexto. Un Short son 40 segundos: más material no +#: mejora el guion, sólo la factura. +CONTEXT_BUDGET = 90_000 + + +class ShortsDisabled(Exception): + """SHORTSMITH_ENABLED=false. El interruptor de emergencia del renderizador.""" + + +@dataclass +class ShortResult: + topic: str + spec: Optional[dict] = None + title: str = "" + article_url: Optional[str] = None + grounding: Optional[GroundingReport] = None + video_path: Optional[str] = None + render_warnings: list[dict] = field(default_factory=list) + attempts: int = 0 + notes: list[str] = field(default_factory=list) + #: Por qué no hay vídeo. None = lo hay. + failure: Optional[str] = None + #: La respuesta cruda del modelo cuando ni siquiera llegó a ser JSON. Se + #: conserva para poder mandarla a Telegram y editarla a mano. + raw_response: str = "" + cost_usd: float = 0.0 + duration_s: float = 0.0 + + @property + def spec_json(self) -> str: + return json.dumps(self.spec, indent=2, ensure_ascii=False) if self.spec else "" + + @property + def has_video(self) -> bool: + return bool(self.video_path) + + +class ShortProducer: + def __init__(self, db: ResearchDB, processor, + client: Optional[ShortsmithClient] = None, + llm_call: Optional[Callable] = None): + self.db = db + self.processor = processor + self.client = client or ShortsmithClient() + #: Sustituto del callable a Claude. Sólo lo usan los tests: el bucle de + #: reintento y los fallbacks se prueban sin gastar tokens. + self.llm_override = llm_call + + # --- piezas ------------------------------------------------------------- + + def _llm_call(self, session_id: int): + """Un callable (system, prompt) -> texto que además apunta el gasto.""" + async def call(system: str, prompt: str) -> str: + client = get_anthropic_client() + msg = await client.messages.create( + model=settings.claude_model, + max_tokens=8000, + system=system, + messages=[{"role": "user", "content": prompt}], + ) + try: + await self.db.log_api_call( + session_id, "short_spec", settings.claude_model, + msg.usage.input_tokens, msg.usage.output_tokens) + in_price, out_price = ResearchDB._price_for_model(settings.claude_model) + call.cost += (msg.usage.input_tokens * in_price + + msg.usage.output_tokens * out_price) / 1_000_000 + except Exception as e: + logger.warning("No se pudo apuntar el gasto del spec", error=str(e)) + return msg.content[0].text.strip() + + call.cost = 0.0 + return call + + def _domain(self) -> str: + """El dominio que va dibujado en el shot de cierre, en mayúsculas y sin + protocolo (así lo escribe el ejemplo de referencia).""" + raw = (settings.ghost_url_en or "https://theexclusionzone.com") + return raw.split("://")[-1].strip("/").removeprefix("www.").upper() + + def _context(self, chunks: list[dict]) -> str: + parts, size = [], 0 + for chunk in chunks: + label = f"[{(chunk.get('source_type') or 'web').upper()}] " \ + f"{chunk.get('title') or chunk.get('url') or 'Unknown'}" + piece = f"{label}:\n{chunk['content']}" + if size + len(piece) > CONTEXT_BUDGET: + break + parts.append(piece) + size += len(piece) + return "\n\n---\n\n".join(parts) + + def _video_path(self, session_id: int) -> Path: + directory = Path(settings.shorts_dir) + directory.mkdir(parents=True, exist_ok=True) + return directory / f"{session_id}.mp4" + + # --- pipeline ----------------------------------------------------------- + + async def produce(self, session_id: int, + progress_callback: Optional[Callable[[str], Any]] = None + ) -> ShortResult: + if not settings.shortsmith_enabled: + raise ShortsDisabled( + "SHORTSMITH_ENABLED=false — el renderizador está apagado a propósito") + if not settings.anthropic_api_key and not self.llm_override: + raise ValueError( + "Escribir un shot spec necesita Claude: es JSON con un contrato " + "estricto, no prosa. Configura ANTHROPIC_API_KEY.") + + session = await self.db.get_session(session_id) + if not session: + raise ValueError(f"Session {session_id} not found") + topic = session["topic"] + result = ShortResult(topic=topic) + + # 1. Material. Los mismos chunks alimentan el prompt y el comprobador. + await _report(progress_callback, "🎬 Writing shot spec…") + chunks = await self.processor.rag_chunks( + session_id, f"{topic} key facts figures dates quotes witnesses", + top_k=CONTEXT_CHUNKS) + if not chunks: + raise ValueError("No processed content available. Run /process first.") + context = self._context(chunks) + + result.article_url = await self.db.get_article_url(session_id) + if not result.article_url: + logger.warning("Short sin URL de artículo — se sigue con el dominio pelado", + session_id=session_id) + + # 2. El contrato, en vivo. Sin él no hay prompt que escribir. + templates = await self.client.templates() + + # 3. El spec. + started = time.monotonic() + llm_call = self.llm_override or self._llm_call(session_id) + writer = ShortSpecWriter( + llm_call, templates, + refresh_templates=lambda: self.client.templates(refresh=True)) + try: + written = await writer.write( + topic, context, article_url=result.article_url, + domain=self._domain(), on_progress=progress_callback) + except SpecWriteFailed as e: + result.cost_usd = getattr(llm_call, "cost", 0.0) + result.spec = e.last_spec + result.raw_response = e.last_raw + result.attempts = e.attempts + result.failure = ("El spec no pasó la validación en " + f"{e.attempts} intentos: " + "; ".join(e.errors[:4])) + logger.warning("Short sin vídeo: spec inválido", session_id=session_id, + errors=e.errors[:4]) + return result + + result.spec = written.spec + result.attempts = written.attempts + result.notes = written.notes + result.cost_usd = getattr(llm_call, "cost", 0.0) + result.title = written.spec.get("meta", {}).get("title", topic) + result.duration_s = sum(s.get("duration", 0) for s in written.spec["shots"]) + + # 4. Fundamento, ANTES de renderizar. No descarta ningún shot: informa. + await _report(progress_callback, "🔍 Checking claims against sources…") + result.grounding = check_grounding(written.spec, chunks) + logger.info("Short grounding", session_id=session_id, + matched=len(result.grounding.grounded), + ungrounded=len(result.grounding.ungrounded), + from_example=len(result.grounding.contaminated)) + + # 5. El spec se guarda ANTES del render: si el render falla, la parte + # cara ya está a salvo en la DB y `/short_spec` la puede devolver. + try: + await self.db.save_output(session_id, OutputType.SHORT_EN, result.spec_json) + except Exception as e: + logger.warning("No se pudo guardar el spec en outputs", error=str(e)) + + # 6. Render. + try: + await self._render(result, session_id, progress_callback) + except ShortsmithRejected as e: + # El validador local no replica las reglas de pydantic que cruzan + # campos (los límites de MapBounds, "3 barras no dejan sitio para + # una cita"): las coge el servidor y se cuentan tal cual. + result.failure = ("shortsmith rechazó el spec: " + + "; ".join(_error_line(x) for x in e.errors[:4])) + except ShortsmithUnavailable as e: + result.failure = f"shortsmith no responde: {e}" + except ShortsmithError as e: + result.failure = f"el render falló: {e}" + except OSError as e: + result.failure = f"no se pudo guardar el vídeo: {e}" + + if result.failure: + logger.warning("Short sin vídeo", session_id=session_id, why=result.failure) + logger.info("Short producido", session_id=session_id, + seconds=round(time.monotonic() - started, 1), + video=result.video_path, cost=round(result.cost_usd, 4)) + return result + + async def _render(self, result: ShortResult, session_id: int, + progress_callback: Optional[Callable[[str], Any]]) -> None: + job_id = await self.client.render(result.spec) + + async def on_progress(fraction: float, status: str) -> None: + if status == "queued": + await _report(progress_callback, "🎞 Queued at the renderer…") + else: + await _report(progress_callback, f"🎞 Rendering… {fraction * 100:.0f}%") + + job = await self.client.poll(job_id, on_progress=on_progress) + result.render_warnings = job.warnings + if not job.ok: + result.failure = f"el render terminó en error: {job.error}" + return + + await _report(progress_callback, "📤 Uploading…") + video = await self.client.fetch_video(job_id) + path = self._video_path(session_id) + path.write_bytes(video) + result.video_path = str(path) + + +def _error_line(error: Any) -> str: + """Un error de pydantic del servidor, con su ruta completa.""" + if not isinstance(error, dict): + return str(error) + loc = ".".join(str(p) for p in error.get("loc", [])) + return f"{loc}: {error.get('msg', '')}" if loc else str(error.get("msg", error)) + + +async def _report(callback: Optional[Callable[[str], Any]], text: str) -> None: + if not callback: + return + try: + value = callback(text) + if hasattr(value, "__await__"): + await value + except Exception as e: + logger.warning("Progreso del Short no enviado", error=str(e)) diff --git a/src/generator/shortsmith.py b/src/generator/shortsmith.py new file mode 100644 index 0000000..f988505 --- /dev/null +++ b/src/generator/shortsmith.py @@ -0,0 +1,234 @@ +"""Cliente HTTP de shortsmith — el renderizador de Shorts. + +shortsmith vive en su propio repo y su propio pod (`shortsmith-svc`), y expone +cuatro cosas: el contrato (`GET /templates`), el envío (`POST /render`), el +estado (`GET /jobs/{id}`) y el MP4 (`GET /jobs/{id}/video`). + +Regla de capas (convención del repo): esto vive en `generator/` y NO importa +nada de `bot/`. El progreso sale por un callable genérico. + +El contrato NO se copia aquí. `GET /templates` publica el esquema de props de +cada plantilla y es la única fuente de verdad: añadir una plantilla en +shortsmith la deja disponible al generador sin tocar este repo. Copiar los +esquemas crearía una segunda fuente que se desincroniza en silencio — la misma +clase de fallo que el desfase de versión de ffmpeg que provocó el OOM de v1. +""" +from __future__ import annotations + +import asyncio +import time +from dataclasses import dataclass, field +from typing import Any, Callable, Optional + +import aiohttp +import structlog + +from src.config import settings, SAFE_ACCEPT_ENCODING + +logger = structlog.get_logger() + +#: Cadencia del sondeo. Un Short de 42 s tarda ~32 s en renderizar y el techo +#: de 180 s del contrato tarda ~138 s: 2 s da una barra de progreso viva sin +#: martillear el servicio. +POLL_INTERVAL = 2.0 +#: Techo del sondeo. Más allá de esto el job está atascado, no lento. +POLL_CEILING = 600.0 + +QUEUED, RUNNING, DONE, ERROR = "queued", "running", "done", "error" + +__all__ = [ + "ShortsmithClient", + "ShortsmithError", + "ShortsmithUnavailable", + "ShortsmithRejected", + "JobResult", + "POLL_INTERVAL", + "POLL_CEILING", +] + + +class ShortsmithError(Exception): + """Cualquier fallo hablando con shortsmith.""" + + +class ShortsmithUnavailable(ShortsmithError): + """No se pudo contactar con el servicio (red, DNS, timeout de conexión).""" + + +class ShortsmithRejected(ShortsmithError): + """422: el spec no pasó la validación del servidor. + + `errors` son los errores de pydantic tal cual los devuelve shortsmith, con + su `loc` completo. Se propagan sin parafrasear: las rutas exactas + (`shots.0.radar_sweep.props.sweeeps`) son lo más útil que se le puede dar + al modelo para corregir. + """ + + def __init__(self, errors: list[dict[str, Any]]): + self.errors = errors + super().__init__(f"shortsmith rechazó el spec ({len(errors)} error/es)") + + +@dataclass +class JobResult: + job_id: str + status: str + progress: float = 0.0 + warnings: list[dict[str, Any]] = field(default_factory=list) + error: Optional[str] = None + + @property + def ok(self) -> bool: + return self.status == DONE + + +#: Caché del contrato para la vida del proceso (clave: base_url). Se refresca a +#: petición cuando una validación falla, por si el renderizador se actualizó a +#: mitad de una run. +_templates_cache: dict[str, dict[str, Any]] = {} + + +class ShortsmithClient: + def __init__(self, base_url: str | None = None, timeout: float | None = None): + self.base_url = (base_url or settings.shortsmith_url).rstrip("/") + self.timeout = timeout if timeout is not None else settings.shortsmith_timeout + + # --- transporte --------------------------------------------------------- + + def _session(self, total: float) -> aiohttp.ClientSession: + # Accept-Encoding explícito SIEMPRE: el default de aiohttp anuncia br si + # hay backend instalado y su decode está roto en 3.14 (KNOWN-ISSUES.md). + return aiohttp.ClientSession( + timeout=aiohttp.ClientTimeout(total=total), + headers={"Accept-Encoding": SAFE_ACCEPT_ENCODING}, + ) + + async def health(self) -> dict[str, Any]: + """`GET /healthz`. Sirve de comprobación previa barata.""" + try: + async with self._session(10) as sess: + async with sess.get(f"{self.base_url}/healthz") as resp: + if resp.status != 200: + raise ShortsmithError(f"healthz devolvió {resp.status}") + return await resp.json() + except aiohttp.ClientError as e: + raise ShortsmithUnavailable(f"shortsmith inalcanzable: {e}") from e + except asyncio.TimeoutError as e: + raise ShortsmithUnavailable("shortsmith no respondió a healthz") from e + + async def templates(self, refresh: bool = False) -> dict[str, Any]: + """El contrato: nombre de plantilla -> JSON Schema de sus props.""" + if not refresh and self.base_url in _templates_cache: + return _templates_cache[self.base_url] + try: + async with self._session(30) as sess: + async with sess.get(f"{self.base_url}/templates") as resp: + if resp.status != 200: + body = await resp.text() + raise ShortsmithError( + f"GET /templates devolvió {resp.status}: {body[:200]}") + data = await resp.json() + except aiohttp.ClientError as e: + raise ShortsmithUnavailable(f"shortsmith inalcanzable: {e}") from e + except asyncio.TimeoutError as e: + raise ShortsmithUnavailable("shortsmith no respondió a /templates") from e + _templates_cache[self.base_url] = data + logger.info("shortsmith templates fetched", n=len(data)) + return data + + async def render(self, spec: dict[str, Any]) -> str: + """`POST /render`. Devuelve el job_id. 422 -> ShortsmithRejected.""" + try: + async with self._session(60) as sess: + async with sess.post(f"{self.base_url}/render", json=spec) as resp: + if resp.status == 422: + detail = (await resp.json()).get("detail") + raise ShortsmithRejected( + detail if isinstance(detail, list) else [{"msg": str(detail)}]) + if resp.status not in (200, 202): + body = await resp.text() + raise ShortsmithError( + f"POST /render devolvió {resp.status}: {body[:300]}") + data = await resp.json() + except aiohttp.ClientError as e: + raise ShortsmithUnavailable(f"shortsmith inalcanzable: {e}") from e + except asyncio.TimeoutError as e: + raise ShortsmithUnavailable("shortsmith no respondió a /render") from e + job_id = data.get("job_id") + if not job_id: + raise ShortsmithError(f"/render no devolvió job_id: {str(data)[:200]}") + logger.info("shortsmith job queued", job_id=job_id) + return job_id + + async def job(self, job_id: str) -> JobResult: + try: + async with self._session(30) as sess: + async with sess.get(f"{self.base_url}/jobs/{job_id}") as resp: + if resp.status == 404: + raise ShortsmithError(f"job {job_id} no existe") + if resp.status != 200: + body = await resp.text() + raise ShortsmithError( + f"GET /jobs/{job_id} devolvió {resp.status}: {body[:200]}") + data = await resp.json() + except aiohttp.ClientError as e: + raise ShortsmithUnavailable(f"shortsmith inalcanzable: {e}") from e + except asyncio.TimeoutError as e: + raise ShortsmithUnavailable(f"shortsmith no respondió por el job {job_id}") from e + return JobResult( + job_id=data.get("job_id", job_id), + status=data.get("status", ""), + progress=data.get("progress") or 0.0, + warnings=data.get("warnings") or [], + error=data.get("error"), + ) + + async def poll(self, job_id: str, + on_progress: Optional[Callable[[float, str], Any]] = None, + interval: float = POLL_INTERVAL, + ceiling: float | None = None) -> JobResult: + """Sondea hasta done/error. Devuelve el JobResult final. + + Un job en `error` se DEVUELVE, no se lanza: el caller decide (el spec + sigue valiendo aunque el render falle). Solo el atasco y los fallos de + transporte lanzan. + """ + deadline = time.monotonic() + min( + ceiling if ceiling is not None else POLL_CEILING, self.timeout) + last_reported = -1.0 + while True: + result = await self.job(job_id) + if on_progress and result.progress != last_reported: + last_reported = result.progress + try: + await _maybe_await(on_progress(result.progress, result.status)) + except Exception as e: # el progreso nunca tumba un render + logger.warning("shortsmith progress callback falló", error=str(e)) + if result.status in (DONE, ERROR): + return result + if time.monotonic() >= deadline: + raise ShortsmithError( + f"job {job_id} sigue en '{result.status}' pasados " + f"{min(ceiling if ceiling is not None else POLL_CEILING, self.timeout):.0f}s " + "— está atascado, no lento") + await asyncio.sleep(interval) + + async def fetch_video(self, job_id: str) -> bytes: + try: + async with self._session(self.timeout) as sess: + async with sess.get(f"{self.base_url}/jobs/{job_id}/video") as resp: + if resp.status != 200: + body = await resp.text() + raise ShortsmithError( + f"GET /jobs/{job_id}/video devolvió {resp.status}: {body[:200]}") + return await resp.read() + except aiohttp.ClientError as e: + raise ShortsmithUnavailable(f"shortsmith inalcanzable: {e}") from e + except asyncio.TimeoutError as e: + raise ShortsmithUnavailable(f"descarga del vídeo {job_id} agotó el tiempo") from e + + +async def _maybe_await(value): + if asyncio.iscoroutine(value): + return await value + return value diff --git a/src/generator/shortspec.py b/src/generator/shortspec.py new file mode 100644 index 0000000..2005bf9 --- /dev/null +++ b/src/generator/shortspec.py @@ -0,0 +1,411 @@ +"""Escritura del shot spec: prompt, inyección del contrato y bucle de reintento. + +Generar un spec no es como generar prosa. La prosa mala se lee y se juzga; un +spec malformado no se puede usar. De ahí las tres mitigaciones, cada una en su +sitio: la validación con reintento vive aquí, el comprobador de fundamento en +`grounding.py`, y la revisión humana en el mensaje de Telegram. + +El contrato se INYECTA (`GET /templates` → `describe_templates`), no se copia. +Lo que sí es de este repo son las tres formas narrativas: son decisiones +editoriales del canal, no del renderizador. + +Sin dependencias de `bot/`: el LLM entra como un callable y el progreso también. +""" +from __future__ import annotations + +import json +import re +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Awaitable, Callable, Optional + +import structlog + +from src.generator.spec_contract import ( + SpecInvalid, describe_templates, editorial_notes, validate_spec, + TARGET_MAX_DURATION, TARGET_MIN_DURATION, +) + +logger = structlog.get_logger() + +#: Tres intentos. Si la media de intentos-hasta-válido sube de 1.5, lo que hay +#: que arreglar es el prompt, no este número. +MAX_ATTEMPTS = 3 + +EXAMPLE_PATH = Path(__file__).parent / "examples" / "jal1628.json" + +__all__ = ["ShortSpecWriter", "SpecResult", "SpecWriteFailed", "NARRATIVE_SHAPES"] + + +#: Las tres formas que se publican de verdad. Dejar la elección libre produce +#: papilla: el modelo elige UNA y la sigue. +NARRATIVE_SHAPES = """\ +case_file — hook → date/place → witness credentials → escalation → + evidence → the official explanation and its problem → close. + Fits a single documented encounter (JAL 1628, Belgium, Ariel School). + +debunk — the claim → why it spread → the method → the finding → + what it means → close. + Fits a claim that dissolves under examination (a mislabelled + crater video, a star mistaken for a craft). + +document_drop — what was released → the standout item → context → + what is still missing → close. + Fits a release of records (PURSUE and similar).""" + + +SHORT_SYSTEM = """\ +You write shot specs for The Exclusion Zone, a documentary channel about UAP \ +cases and declassified records. A shot spec is JSON that a renderer turns \ +directly into a vertical video: every string you write is drawn on screen \ +exactly as you typed it. + +You answer with ONE JSON object and nothing else — no prose, no explanation, \ +no markdown fences. + +You never state a figure, a quote, a date or a name that is not in the research \ +material you were given. Not even one you happen to know is true. The channel's \ +entire premise is that its numbers come from primary sources.""" + + +PROMPT = """\ +Write a shot spec for a Short about: "{topic}" + +# 1. Pick one narrative shape and follow its arc + +{shapes} + +# 2. The contract — these are the templates the renderer accepts + +Each shot names a template and supplies its props. Nothing outside this list \ +exists, and a prop name that is not listed is a parse error, not a nuance. + +{templates} + +# 3. Rules + +- Total duration 20-45 seconds. The contract allows 180; that is a ceiling, \ +not a target. Aim for {target_min:.0f}-{target_max:.0f}. +- Typically 6-9 shots. Give a shot the seconds its content needs to be read: \ +a card with four rows needs longer than a headline. +- Every string is drawn as given. Write them the way they should appear: \ +SHORT, UPPERCASE, no trailing punctuation. A headline is 2-5 words. +- Respect every max length and list-length limit above. They are enforced. +- Colours are palette names ({colors}) — never hex. +- Quotes carry the typographic quote marks: “SPLIT RADAR IMAGE”, with U+201C \ +and U+201D. Never a straight " inside a string — that closes the JSON string \ +and your whole answer becomes unparseable. This is the single most common way \ +this task fails. +- meta.id is a lowercase slug (letters, digits, - and _). meta.title is the \ +title a human reads, not a filename. +- version is 1. Keep meta at 1080x1920 and audio preset "sonar". +- The closing shot carries the domain, uppercase, no protocol: {domain} + +# 4. Grounding — this is the part that matters + +Every figure, quote, date, and proper noun in your spec must appear in the \ +research material below. An automated check runs against these exact sources \ +before anything is rendered, and every string it cannot find is shown to a \ +human next to your spec. + +If the material does not support a number, do not write the number. A shot with \ +one solid fact beats a shot with three plausible ones. This applies to the \ +worked example in section 5 as much as to your own knowledge: a figure that is \ +only in the example is a figure you cannot use. + +Anything you put inside quote marks must be a word-for-word span of the \ +material. Copy it; do not compress it. "WALNUT SHAPED WIDE RIM" is not a quote \ +when the source says "walnut shaped with a wide rim around its circumference" \ +— pick a shorter span that is still verbatim, or drop the quote marks and \ +state the fact plainly. + +# 5. A worked example — FORMAT ONLY + +This is a case_file that produced a good video. Read it for shape: how long a \ +shot runs, how a headline is worded, how the shots build. + +It is not source material. Do not reuse its strings, figures, coordinates, \ +quotes or waypoints — not even if it covers the same case you were asked \ +about. Every value in your spec comes from section 7 and nowhere else. A \ +number copied from here is a fabrication, and the grounding check will find it. + +{example} + +# 6. The article this Short accompanies + +{article} + +# 7. Research material — the only facts you may use + +{context} + +Return the JSON object now.""" + + +PALETTE_FALLBACK = "ink, amber, amber_dark, muted, dim, red" + + +@dataclass +class SpecResult: + spec: dict + attempts: int + notes: list[str] = field(default_factory=list) + #: Errores de cada intento fallido, en orden. Sirve de métrica y de pista + #: cuando un spec sale a la primera pero raro. + history: list[list[str]] = field(default_factory=list) + + +class SpecWriteFailed(Exception): + """Tres intentos y ninguno válido. + + Lleva el último intento aunque no valga: la parte cara es la generación, no + el render, y un spec inválido se edita a mano y se reenvía. Nunca se tira. + """ + + def __init__(self, errors: list[str], last_raw: str = "", + last_spec: Optional[dict] = None, attempts: int = 0): + self.errors = errors + self.last_raw = last_raw + self.last_spec = last_spec + self.attempts = attempts + super().__init__("; ".join(errors[:5]) or "no se pudo escribir el spec") + + +def _load_example() -> str: + try: + return json.dumps(json.loads(EXAMPLE_PATH.read_text(encoding="utf-8")), + indent=2, ensure_ascii=False) + except Exception as e: # nunca bloquea: el ejemplo mejora el prompt, no lo define + logger.warning("ejemplo de spec no legible — se sigue sin él", error=str(e)) + return "(no example available)" + + +def _palette(templates: dict[str, dict]) -> str: + """Los nombres de color, sacados del propio contrato.""" + found: list[str] = [] + + def walk(node: Any): + if isinstance(node, dict): + enum = node.get("enum") + if enum and node.get("type") == "string" and "ink" in enum: + for name in enum: + if name not in found: + found.append(name) + for v in node.values(): + walk(v) + elif isinstance(node, list): + for v in node: + walk(v) + + walk(templates) + return ", ".join(found) or PALETTE_FALLBACK + + +#: Lo que puede seguir legítimamente al cierre de una cadena JSON. +_AFTER_STRING = set(',:}] \t\r\n') +#: Antes de una comilla de apertura hay hueco, un guion o el propio inicio. +_BEFORE_OPENING = set(' \t\n([-–—‑:') + + +def _typographic_inner_quotes(body: str) -> str: + """Convierte en “ ” las comillas rectas que van DENTRO de una cadena JSON. + + Se recorre el texto sabiendo dónde empieza y acaba cada cadena: una `"` que + no vaya seguida de `,`, `:`, `}`, `]` o espacio no cierra nada, es una + comilla del texto. Decidir apertura o cierre por el carácter anterior. + + Sólo se llama tras un fallo de parseo: un JSON correcto no pasa por aquí. + """ + out: list[str] = [] + in_string = False + escaped = False + for i, char in enumerate(body): + if escaped: + out.append(char) + escaped = False + continue + if char == "\\": + out.append(char) + escaped = in_string + continue + if char != '"': + out.append(char) + continue + + if not in_string: + in_string = True + out.append(char) + continue + + nxt = next((c for c in body[i + 1:] if not c.isspace()), "") + if nxt in ",:}]" or nxt == "": + in_string = False + out.append(char) + else: + previous = out[-1] if out else "" + out.append("“" if previous in _BEFORE_OPENING or previous == '"' else "”") + return "".join(out) + + +def extract_json(text: str) -> dict: + """El objeto JSON de la respuesta del modelo, con o sin valla de markdown. + + Un error de parseo se cuenta CON el trozo que lo provocó. "Expecting ',' + delimiter: line 189 column 22" no le sirve de nada al modelo, que no ve su + salida numerada; el fragmento sí — y el fallo típico es una comilla recta + dentro de una cadena, que cierra la cadena antes de tiempo. + """ + cleaned = text.strip() + fenced = re.search(r"```(?:json)?\s*(.+?)```", cleaned, re.DOTALL) + if fenced: + cleaned = fenced.group(1).strip() + start, end = cleaned.find("{"), cleaned.rfind("}") + if start == -1 or end <= start: + raise ValueError("la respuesta no contiene ningún objeto JSON") + body = cleaned[start:end + 1] + try: + return json.loads(body) + except json.JSONDecodeError: + pass + + # Reparación determinista de LA forma en que esto falla: comillas rectas + # dentro de una cadena (`"quote_a": ""CREDIBLE PEOPLE""`). Medido el + # 2026-08-01 contra la sesión de Bélgica: el modelo lo repitió en los tres + # intentos aunque el prompt lo prohíbe y el error se le devolvía con el + # fragmento. Arreglarlo aquí es además lo que se quiere dibujar: las citas + # del canal van con las tipográficas. + repaired = _typographic_inner_quotes(body) + try: + return json.loads(repaired) + except json.JSONDecodeError as e: + snippet = repaired[max(0, e.pos - 60):e.pos + 60].replace("\n", " ") + raise ValueError( + f"{e.msg} — aquí: …{snippet}… " + "(si es una comilla recta dentro de una cadena, cierra la cadena: " + "las citas van con las tipográficas “ ”)") from None + + +def _format_errors(errors: list[str]) -> str: + """Las rutas, verbatim. Son más útiles para el modelo que cualquier paráfrasis.""" + listed = "\n".join(f"- {e}" for e in errors) + return (f"\n\n# Your previous attempt was rejected\n\n{listed}\n\n" + "Fix exactly these and return the corrected JSON object. " + "Keep everything else as it was.") + + +def _format_notes(notes: list[str]) -> str: + listed = "\n".join(f"- {n}" for n in notes) + return (f"\n\n# Your previous attempt is valid but off-brief\n\n{listed}\n\n" + "Return the adjusted JSON object.") + + +#: (system, prompt) -> texto del modelo. +LLMCall = Callable[[str, str], Awaitable[str]] + + +class ShortSpecWriter: + def __init__(self, llm_call: LLMCall, templates: dict[str, dict], + refresh_templates: Optional[Callable[[], Awaitable[dict]]] = None): + self.llm_call = llm_call + self.templates = templates + #: Se vuelve a pedir el contrato si una validación falla: el + #: renderizador puede haberse actualizado a mitad de la run. + self.refresh_templates = refresh_templates + + def build_prompt(self, topic: str, context: str, article_url: Optional[str], + domain: str) -> str: + article = (f"The article is published at {article_url} — the Short points at it." + if article_url + else "No article URL yet. Use the bare domain on the closing shot.") + return PROMPT.format( + topic=topic, + shapes=NARRATIVE_SHAPES, + templates=describe_templates(self.templates), + colors=_palette(self.templates), + domain=domain, + target_min=TARGET_MIN_DURATION, + target_max=TARGET_MAX_DURATION, + example=_load_example(), + article=article, + context=context, + ) + + async def write(self, topic: str, context: str, *, + article_url: Optional[str] = None, + domain: str = "THEEXCLUSIONZONE.COM", + on_progress: Optional[Callable[[str], Any]] = None) -> SpecResult: + base_prompt = self.build_prompt(topic, context, article_url, domain) + feedback = "" + history: list[list[str]] = [] + last_raw, last_spec = "", None + #: Un spec que cumple el contrato pero se pasa de duración. Se guarda + #: para que un intento posterior peor no lo tire: es renderizable. + best: Optional[SpecResult] = None + + for attempt in range(1, MAX_ATTEMPTS + 1): + if on_progress and attempt > 1: + await _maybe_await(on_progress( + f"🎬 Rewriting the shot spec (attempt {attempt}/{MAX_ATTEMPTS})…")) + + last_raw = await self.llm_call(SHORT_SYSTEM, base_prompt + feedback) + + try: + spec = extract_json(last_raw) + except (ValueError, json.JSONDecodeError) as e: + errors = [f"la respuesta no es un objeto JSON válido: {e}"] + history.append(errors) + feedback = _format_errors(errors) + continue + + last_spec = spec + try: + validate_spec(spec, self.templates) + except SpecInvalid as e: + history.append(e.errors) + feedback = _format_errors(e.errors) + # El contrato puede haber cambiado bajo los pies: se refresca + # una vez antes de volver a intentarlo. + if self.refresh_templates and attempt == 1: + try: + self.templates = await self.refresh_templates() + base_prompt = self.build_prompt(topic, context, article_url, domain) + except Exception as refresh_err: + logger.warning("no se pudo refrescar el contrato", + error=str(refresh_err)) + continue + + notes = editorial_notes(spec) + result = SpecResult(spec=spec, attempts=attempt, notes=notes, + history=list(history)) + if notes and attempt < MAX_ATTEMPTS: + # Nota editorial, no violación del contrato: se comenta una vez + # y, si insiste, se renderiza igual. + best = best or result + history.append(notes) + feedback = _format_notes(notes) + continue + + logger.info("short spec válido", attempts=attempt, + shots=len(spec.get("shots", [])), notes=len(notes)) + return result + + if best is not None: + # Un intento anterior sí cumplía el contrato. Vale más un Short + # largo que ningún Short. + logger.info("short spec: se recupera el intento válido anterior", + attempts=MAX_ATTEMPTS, notes=best.notes) + best.history = history + return best + + logger.warning("short spec inválido tras todos los intentos", + attempts=MAX_ATTEMPTS, errors=history[-1] if history else []) + raise SpecWriteFailed(history[-1] if history else ["sin errores registrados"], + last_raw=last_raw, last_spec=last_spec, + attempts=MAX_ATTEMPTS) + + +async def _maybe_await(value): + import asyncio + if asyncio.iscoroutine(value): + return await value + return value diff --git a/src/generator/spec_contract.py b/src/generator/spec_contract.py new file mode 100644 index 0000000..3084695 --- /dev/null +++ b/src/generator/spec_contract.py @@ -0,0 +1,375 @@ +"""El contrato del spec, leído — no copiado — de shortsmith. + +Dos cosas, las dos guiadas por lo que publica `GET /templates`: + +* `describe_templates()` — el contrato en prosa compacta, para meterlo en el + prompt. Añadir una plantilla en shortsmith la deja descrita aquí sola. +* `validate_spec()` — validación local ANTES de renderizar, con las mismas + rutas de error que devolvería el servidor + (`shots.0.radar_sweep.props.sweeeps`). Hace falta que sea local porque el + comprobador de fundamento va entre la validación y el render: mandar el spec + a `POST /render` para validarlo ya encolaría el render. + +La mitad de props del contrato NO vive aquí: se valida contra el esquema +recibido. Lo único escrito a mano es el sobre (version/meta/audio/shots), que +es pequeño, estable, y está anotado con la regla equivalente de +`shortsmith/src/shortsmith/spec.py`. Las reglas de pydantic que cruzan campos +(los límites de MapBounds, "3 barras no dejan sitio para una cita") NO se +replican: las coge el 422 del servidor al enviar, y ese camino también está +cubierto. +""" +from __future__ import annotations + +import re +from typing import Any, Optional + +__all__ = [ + "SpecInvalid", + "validate_spec", + "editorial_notes", + "describe_templates", + "TARGET_MIN_DURATION", + "TARGET_MAX_DURATION", +] + +# Límites del sobre — espejo de shortsmith/spec.py. +RESOLUTIONS = {(1080, 1920), (1920, 1080)} +FPS_VALUES = {24, 25, 30, 60} +MIN_SHOT_DURATION = 0.5 +MIN_TOTAL_DURATION = 5.0 +MAX_TOTAL_DURATION = 180.0 # límite duro de YouTube Shorts +MAX_SHOTS = 64 +META_ID = re.compile(r"^[a-z0-9][a-z0-9_-]{0,63}$") + +#: El objetivo editorial, que NO es el techo del contrato. 180 s es lo que el +#: renderizador acepta; 20-45 s es lo que se ve entero. +TARGET_MIN_DURATION = 20.0 +TARGET_MAX_DURATION = 45.0 + + +class SpecInvalid(Exception): + """El spec no cumple el contrato. `errors` son rutas + motivo, verbatim.""" + + def __init__(self, errors: list[str]): + self.errors = errors + super().__init__("; ".join(errors[:5]) or "spec inválido") + + +# --- validación contra el esquema publicado --------------------------------- + +def _resolve(schema: dict, defs: dict) -> dict: + ref = schema.get("$ref") + if not ref: + return schema + name = ref.rsplit("/", 1)[-1] + return defs.get(name, {}) + + +def _type_ok(value: Any, expected: str) -> bool: + if expected == "object": + return isinstance(value, dict) + if expected == "array": + return isinstance(value, list) + if expected == "string": + return isinstance(value, str) + if expected == "integer": + return isinstance(value, int) and not isinstance(value, bool) + if expected == "number": + return isinstance(value, (int, float)) and not isinstance(value, bool) + if expected == "boolean": + return isinstance(value, bool) + if expected == "null": + return value is None + return True + + +def _check(value: Any, schema: dict, path: str, defs: dict) -> list[str]: + """Subconjunto de JSON Schema que emite pydantic. Devuelve rutas de error.""" + schema = _resolve(schema, defs) + if not schema: + return [] + + if "anyOf" in schema: + for branch in schema["anyOf"]: + if not _check(value, branch, path, defs): + return [] + kinds = [_resolve(b, defs).get("type", "?") for b in schema["anyOf"]] + return [f"{path}: no casa con ninguna alternativa ({', '.join(kinds)})"] + + errors: list[str] = [] + expected = schema.get("type") + if expected and not _type_ok(value, expected): + return [f"{path}: se esperaba {expected}, llegó {type(value).__name__}"] + + if "enum" in schema and value not in schema["enum"]: + allowed = ", ".join(repr(v) for v in schema["enum"]) + return [f"{path}: {value!r} no es un valor permitido ({allowed})"] + + if isinstance(value, str): + if len(value) < schema.get("minLength", 0): + errors.append(f"{path}: cadena vacía o más corta que " + f"{schema['minLength']} caracteres") + if "maxLength" in schema and len(value) > schema["maxLength"]: + errors.append(f"{path}: {len(value)} caracteres, el máximo es " + f"{schema['maxLength']}") + + if isinstance(value, (int, float)) and not isinstance(value, bool): + for key, ok, text in ( + ("minimum", lambda v, lim: v >= lim, ">="), + ("maximum", lambda v, lim: v <= lim, "<="), + ("exclusiveMinimum", lambda v, lim: v > lim, ">"), + ("exclusiveMaximum", lambda v, lim: v < lim, "<"), + ): + if key in schema and not ok(value, schema[key]): + errors.append(f"{path}: {value} debe ser {text} {schema[key]}") + + if isinstance(value, list): + if "minItems" in schema and len(value) < schema["minItems"]: + errors.append(f"{path}: {len(value)} elementos, el mínimo es " + f"{schema['minItems']}") + if "maxItems" in schema and len(value) > schema["maxItems"]: + errors.append(f"{path}: {len(value)} elementos, el máximo es " + f"{schema['maxItems']}") + item_schema = schema.get("items") + if item_schema: + for i, item in enumerate(value): + errors.extend(_check(item, item_schema, f"{path}.{i}", defs)) + + if isinstance(value, dict): + properties = schema.get("properties", {}) + for required in schema.get("required", []): + if required not in value: + errors.append(f"{path}.{required}: falta y es obligatorio") + if schema.get("additionalProperties") is False: + for key in value: + if key not in properties: + allowed = ", ".join(sorted(properties)) or "ninguna" + errors.append(f"{path}.{key}: campo no permitido " + f"(las válidas son: {allowed})") + for key, sub in properties.items(): + if key in value: + errors.extend(_check(value[key], sub, f"{path}.{key}", defs)) + + return errors + + +def _check_props(props: Any, schema: dict, path: str) -> list[str]: + return _check(props, schema, path, schema.get("$defs", {})) + + +# --- el sobre --------------------------------------------------------------- + +def _check_meta(meta: Any) -> list[str]: + if not isinstance(meta, dict): + return ["meta: se esperaba un objeto"] + errors = [] + spec_id = meta.get("id") + if not isinstance(spec_id, str) or not META_ID.match(spec_id): + errors.append("meta.id: minúsculas, dígitos, '_' y '-', empezando por " + f"letra o dígito, hasta 64 caracteres (llegó {spec_id!r})") + if not isinstance(meta.get("title"), str) or not meta.get("title"): + errors.append("meta.title: obligatorio y no vacío") + width = meta.get("width", 1080) + height = meta.get("height", 1920) + if (width, height) not in RESOLUTIONS: + allowed = ", ".join(f"{w}x{h}" for w, h in sorted(RESOLUTIONS)) + errors.append(f"meta: {width}x{height} no es una resolución admitida ({allowed})") + if meta.get("fps", 30) not in FPS_VALUES: + errors.append(f"meta.fps: {meta.get('fps')!r} no está entre " + f"{sorted(FPS_VALUES)}") + for key in meta: + if key not in ("id", "title", "width", "height", "fps", "theme"): + errors.append(f"meta.{key}: campo no permitido") + return errors + + +def _check_audio(audio: Any, total: float) -> list[str]: + if audio is None: + return [] + if not isinstance(audio, dict): + return ["audio: se esperaba un objeto"] + errors = [] + if audio.get("preset", "sonar") not in ("sonar", "none"): + errors.append(f"audio.preset: {audio.get('preset')!r} no es 'sonar' ni 'none'") + silence = audio.get("silence", []) + if not isinstance(silence, list): + return errors + ["audio.silence: se esperaba una lista de pares [inicio, fin]"] + if len(silence) > 16: + errors.append(f"audio.silence: {len(silence)} rangos, el máximo es 16") + for i, rango in enumerate(silence): + if not (isinstance(rango, (list, tuple)) and len(rango) == 2 + and all(isinstance(v, (int, float)) for v in rango)): + errors.append(f"audio.silence.{i}: se esperaba [inicio, fin] numérico") + continue + start, end = rango + if start < 0: + errors.append(f"audio.silence.{i}: empieza antes de 0") + if end <= start: + errors.append(f"audio.silence.{i}: el fin no va después del inicio") + if end > total + 1e-9: + errors.append(f"audio.silence.{i}: [{start}, {end}] se sale de la " + f"duración total ({total:.2f}s)") + for key in audio: + if key not in ("preset", "silence"): + errors.append(f"audio.{key}: campo no permitido") + return errors + + +def _total_duration(spec: dict) -> float: + total = 0.0 + for shot in spec.get("shots") or []: + if isinstance(shot, dict) and isinstance(shot.get("duration"), (int, float)): + total += float(shot["duration"]) + return total + + +def validate_spec(spec: Any, templates: dict[str, dict]) -> None: + """Lanza `SpecInvalid` con TODAS las rutas que fallan. + + Se devuelven todos los errores de golpe a propósito: el bucle de reintento + se los da al modelo verbatim y arreglar cinco de una vez sale más barato + que cinco vueltas. + """ + errors: list[str] = [] + if not isinstance(spec, dict): + raise SpecInvalid([f"el spec debe ser un objeto JSON, llegó {type(spec).__name__}"]) + + if spec.get("version") != 1: + errors.append(f"version: debe ser 1 (llegó {spec.get('version')!r})") + for key in spec: + if key not in ("version", "meta", "audio", "shots"): + errors.append(f"{key}: campo no permitido en la raíz " + "(las válidas son: version, meta, audio, shots)") + + errors.extend(_check_meta(spec.get("meta"))) + + shots = spec.get("shots") + if not isinstance(shots, list) or not shots: + errors.append("shots: hace falta al menos un shot") + raise SpecInvalid(errors) + if len(shots) > MAX_SHOTS: + errors.append(f"shots: {len(shots)} shots, el máximo es {MAX_SHOTS}") + + known = ", ".join(sorted(templates)) + for i, shot in enumerate(shots): + path = f"shots.{i}" + if not isinstance(shot, dict): + errors.append(f"{path}: se esperaba un objeto") + continue + template = shot.get("template") + if template not in templates: + errors.append(f"{path}.template: {template!r} no existe " + f"(las plantillas son: {known})") + continue + for key in shot: + if key not in ("template", "duration", "props"): + errors.append(f"{path}.{key}: campo no permitido " + "(las válidas son: template, duration, props)") + duration = shot.get("duration") + if not isinstance(duration, (int, float)) or isinstance(duration, bool): + errors.append(f"{path}.duration: obligatoria y numérica") + elif not MIN_SHOT_DURATION <= duration <= MAX_TOTAL_DURATION: + errors.append(f"{path}.duration: {duration} fuera de " + f"[{MIN_SHOT_DURATION}, {MAX_TOTAL_DURATION}]") + if "props" not in shot: + errors.append(f"{path}.props: falta y es obligatorio") + continue + errors.extend(_check_props(shot["props"], templates[template], + f"{path}.{template}.props")) + + total = _total_duration(spec) + if total < MIN_TOTAL_DURATION: + errors.append(f"shots: la duración total ({total:.2f}s) no llega al " + f"mínimo de {MIN_TOTAL_DURATION}s") + if total > MAX_TOTAL_DURATION: + errors.append(f"shots: la duración total ({total:.2f}s) pasa del límite " + f"de {MAX_TOTAL_DURATION}s") + errors.extend(_check_audio(spec.get("audio"), total)) + + if errors: + raise SpecInvalid(errors) + + +def editorial_notes(spec: dict) -> list[str]: + """Lo que no viola el contrato pero sí el encargo. + + Va aparte de `validate_spec` justo porque no impide renderizar: un Short de + 70 s se ve, sólo que peor. Se le devuelve al modelo como comentario una vez; + si insiste, se renderiza igual antes que tirar la generación a la basura. + """ + notes = [] + total = _total_duration(spec) + if total < TARGET_MIN_DURATION: + notes.append(f"la duración total son {total:.1f}s y el objetivo es " + f"{TARGET_MIN_DURATION:.0f}-{TARGET_MAX_DURATION:.0f}s: " + "queda corto, añade un shot o alarga los que tienes") + elif total > TARGET_MAX_DURATION: + notes.append(f"la duración total son {total:.1f}s y el objetivo es " + f"{TARGET_MIN_DURATION:.0f}-{TARGET_MAX_DURATION:.0f}s: " + "recorta shots o acorta duraciones") + return notes + + +# --- el contrato en prosa, para el prompt ----------------------------------- + +def _describe_field(name: str, schema: dict, required: bool, defs: dict, + indent: str = " ") -> list[str]: + schema = _resolve(schema, defs) + bits: list[str] = [] + + if "anyOf" in schema: + inner = [b for b in schema["anyOf"] if _resolve(b, defs).get("type") != "null"] + if inner: + return _describe_field(name, inner[0], required, defs, indent) + \ + [f"{indent} (opcional, admite null)"] + + kind = schema.get("type", "?") + if "enum" in schema: + bits.append("uno de: " + ", ".join(str(v) for v in schema["enum"])) + elif kind == "array": + item = _resolve(schema.get("items", {}), defs) + bits.append("lista") + if "minItems" in schema or "maxItems" in schema: + bits.append(f"{schema.get('minItems', 0)}-{schema.get('maxItems', '∞')} elementos") + else: + bits.append(kind) + + if schema.get("minLength"): + bits.append("no vacío") + if "maxLength" in schema: + bits.append(f"máx {schema['maxLength']} caracteres") + for key, text in (("minimum", "≥"), ("maximum", "≤"), + ("exclusiveMinimum", ">"), ("exclusiveMaximum", "<")): + if key in schema: + bits.append(f"{text} {schema[key]}") + bits.append("OBLIGATORIO" if required else f"opcional (por defecto {schema.get('default')!r})") + + lines = [f"{indent}{name}: {', '.join(bits)}"] + + # Los objetos (sueltos o dentro de una lista) se despliegan: si no, el + # modelo ve "waypoints: lista" y no sabe que cada uno lleva label/lat/lon. + nested = _resolve(schema.get("items", {}), defs) if kind == "array" else schema + if nested.get("type") == "object" and nested.get("properties"): + nested_required = set(nested.get("required", [])) + for sub, sub_schema in nested["properties"].items(): + lines.extend(_describe_field(sub, sub_schema, sub in nested_required, + defs, indent + " ")) + return lines + + +def describe_templates(templates: dict[str, dict]) -> str: + """El contrato tal cual lo publica el servicio, en prosa compacta. + + Se describe lo recibido, sin lista de plantillas escrita a mano: una + plantilla nueva en shortsmith aparece aquí sin tocar este repo. + """ + blocks = [] + for name in sorted(templates): + schema = templates[name] or {} + defs = schema.get("$defs", {}) + required = set(schema.get("required", [])) + lines = [f"{name}:"] + for field, field_schema in schema.get("properties", {}).items(): + lines.extend(_describe_field(field, field_schema, field in required, defs)) + blocks.append("\n".join(lines)) + return "\n\n".join(blocks) diff --git a/src/processor/processor.py b/src/processor/processor.py index 7375d87..56d2c19 100644 --- a/src/processor/processor.py +++ b/src/processor/processor.py @@ -442,6 +442,24 @@ class ContentProcessor: """ Retrieve most relevant chunks for a query using embeddings + keyword fallback """ + top_chunks = await self.rag_chunks(session_id, query, top_k) + + # Build context + context_parts = [] + for chunk in top_chunks: + source_label = f"[{chunk.get('source_type', 'web').upper()}] {chunk.get('title', 'Unknown')}" + context_parts.append(f"{source_label}:\n{chunk['content']}") + + return "\n\n---\n\n".join(context_parts) + + async def rag_chunks(self, session_id: int, query: str, + top_k: int = 20) -> list[dict]: + """Los chunks en sí, no el contexto ya montado. + + Mismo ranking que `rag_query` (que ahora llama aquí). Hace falta para el + comprobador de fundamento: comprueba contra EXACTAMENTE el material que + se le pasó al modelo, y para eso necesita las filas, con su `url`. + """ # Get query embedding query_embedding = await self.ollama.embed(query) @@ -462,15 +480,7 @@ class ContentProcessor: scored.append((sim * 0.7 + chunk["quality_score"] * 0.3, chunk)) scored.sort(key=lambda x: x[0], reverse=True) - top_chunks = [c for _, c in scored[:top_k]] - else: - # Fallback: just use quality score - top_chunks = chunks[:top_k] + return [c for _, c in scored[:top_k]] - # Build context - context_parts = [] - for chunk in top_chunks: - source_label = f"[{chunk.get('source_type', 'web').upper()}] {chunk.get('title', 'Unknown')}" - context_parts.append(f"{source_label}:\n{chunk['content']}") - - return "\n\n---\n\n".join(context_parts) + # Fallback: just use quality score + return chunks[:top_k] diff --git a/tests/test_bot_short_report.py b/tests/test_bot_short_report.py new file mode 100644 index 0000000..4afefa6 --- /dev/null +++ b/tests/test_bot_short_report.py @@ -0,0 +1,64 @@ +"""El mensaje de revisión del Short. + +Es la puerta humana: si este mensaje no sale, o sale sin los avisos, se está +publicando lo que el modelo recuerde en vez de lo que dicen las fuentes. Por eso +tiene test propio aparte del pipeline. +""" +import json + +from src.bot.bot import _claims_message +from src.generator.grounding import check_grounding +from src.generator.short import ShortResult + +SPEC = { + "version": 1, + "meta": {"id": "x", "title": "X"}, + "shots": [{"template": "scale_bars", "duration": 30.0, "props": { + "headline": "REPORTED SCALE", + "bars": [{"label": "BOEING 747", "value": 232, "unit": "FT"}]}}], +} +CHUNKS = [{"content": "A Boeing 747 is 232 ft long.", "url": "https://a.test/1"}] + + +def result_with(**kw): + base = dict(topic="Caso X", spec=SPEC, title="X", attempts=1, + cost_usd=0.0042, duration_s=30.0, + article_url="https://www.theexclusionzone.com/caso-x/", + grounding=check_grounding(SPEC, CHUNKS)) + base.update(kw) + return ShortResult(**base) + + +def test_a_clean_report_still_says_so(): + """Un éxito silencioso enseña al lector a dejar de mirar.""" + text = _claims_message(result_with()) + assert "0 sin encontrar" in text + assert "1 chunks de 1 URLs" in text + assert "Coste: $0.0042" in text + + +def test_ungrounded_claims_are_listed_one_by_one(): + invented = json.loads(json.dumps(SPEC)) + invented["shots"][0]["props"]["headline"] = "41,000 FT" + text = _claims_message(result_with(spec=invented, + grounding=check_grounding(invented, CHUNKS))) + assert "1 sin encontrar" in text + assert "41,000 FT" in text + + +def test_a_session_without_an_article_url_says_what_to_run(): + text = _claims_message(result_with(article_url=None)) + assert "/generate blog en" in text + + +def test_render_warnings_reach_the_human(): + text = _claims_message(result_with(render_warnings=[ + {"template": "data_card", "text": "UNA FILA DEMASIADO LARGA", + "requested": 44, "size": 38}])) + assert "recortados" in text and "data_card" in text + + +def test_there_is_a_report_even_when_there_was_no_spec(): + text = _claims_message(ShortResult(topic="Caso X")) + assert "Sin comprobación de fundamento" in text + assert "Coste:" in text diff --git a/tests/test_grounding.py b/tests/test_grounding.py new file mode 100644 index 0000000..4a9cdb1 --- /dev/null +++ b/tests/test_grounding.py @@ -0,0 +1,288 @@ +"""El comprobador de fundamento, contra el spec de referencia y contra copias +deliberadamente corrompidas. + +Los chunks de abajo son material de fuente sintético pero escrito como escribe +una fuente real: fechas en otro orden que el spec, unidades con la palabra +entera, comillas tipográficas, números con separador de millares. Si el +comprobador sólo supiera comparar cadenas idénticas, este fichero lo delataría. +""" +import copy +import json +from pathlib import Path + +import pytest + +from src.generator.grounding import ( + check_grounding, extract_claims, normalize, +) + +EXAMPLE = Path(__file__).resolve().parents[1] / "src/generator/examples/jal1628.json" + + +@pytest.fixture +def spec(): + return json.loads(EXAMPLE.read_text()) + + +#: Cada chunk imita una fuente distinta. Entre los cuatro está TODO lo que +#: afirma examples/jal1628.json, pero casi nunca con las mismas palabras. +CHUNKS = [ + { + "url": "https://www.faa.gov/foia/jal1628", + "content": ( + "On November 17, 1986, Japan Air Lines flight JAL 1628, a Boeing 747 " + "cargo aircraft, was cruising at 35,000 feet and roughly 600 mph over " + "Alaska, en route from Fort Yukon toward Anchorage by way of Fairbanks " + "and Talkeetna. The flight crew reported two lights pacing the aircraft." + ), + }, + { + "url": "https://example.org/terauchi-testimony", + "content": ( + "The pilot in command was Captain Kenju Terauchi, an ex-fighter pilot " + "with the JASDF, 29 years of flying experience and more than 10,000 " + "flight hours. Terauchi described the object as “twice the size of an " + "aircraft carrier”, an estimate that would put it between 1,600 and " + "2,000 feet across — against the 232 ft length of his own Boeing 747. " + "The unidentified contact held its relative position through a full " + "360° turn and a descent of 4,000 ft." + ), + }, + { + "url": "https://example.org/radar-records", + "content": ( + "Three independent sources logged the encounter. The onboard radar " + "showed a contact 7–8 nm out at the 10 o'clock position. Anchorage " + "Center recorded primary returns through the turns. The Elmendorf ROCC " + "tracked what it logged as a “flight of two”. Fairbanks radar showed " + "nothing at all." + ), + }, + { + "url": "https://example.org/faa-closing", + "content": ( + "The FAA closed the case on 5 March 1987 with an official finding of a " + "“split radar image”. An AARTCC controller said such a split happened " + "“rarely, if ever” in that airspace. The FAA released roughly 1,500 " + "pages of documentation. Forty years on — 40 years — the file is still " + "open, and the estimated object has no accepted explanation." + ), + }, +] + + +# --- normalización ---------------------------------------------------------- + +def test_normalize_thousands_separators(): + assert normalize("35,000 FT") == normalize("35000 ft") == "35000 ft" + assert normalize("1.500 paginas") == normalize("1,500 paginas") == "1500 paginas" + # No toca los decimales de verdad: 61.22 es una latitud, no 6122. + assert "61.22" in normalize("61.22") + assert "1.5" in normalize("1.5") + + +def test_normalize_quote_glyphs_and_dashes(): + assert normalize("“FLIGHT OF TWO”") == normalize('"flight of two"') + assert normalize("1,600 – 2,000") == normalize("1600 - 2000") + assert normalize("−4,000") == normalize("-4000") + assert normalize("CONTACT 7–8 NM · 10 O’CLOCK") == "contact 7-8 nm 10 o'clock" + + +def test_normalize_is_idempotent(): + once = normalize("“~1,600 – 2,000 FT”") + assert normalize(once) == once + + +# --- extracción ------------------------------------------------------------- + +def test_extracts_quotes_figures_dates_and_names(spec): + claims = extract_claims(spec) + by_kind = {} + for c in claims: + by_kind.setdefault(c.kind, set()).add(c.text) + + assert "TWICE THE SIZE OF AN AIRCRAFT CARRIER" in by_kind["quote"] + assert "SPLIT RADAR IMAGE" in by_kind["quote"] + assert "35,000 FT" in by_kind["figure"] + assert "1500" in by_kind["figure"] # count_to, que sí afirma un dato + assert "17 NOV 1986" in by_kind["date"] + assert "5 MARCH 1987" in by_kind["date"] + assert "CAPT. KENJU TERAUCHI" in by_kind["name"] + assert "ELMENDORF ROCC" in by_kind["name"] + + +def test_geometry_is_not_a_claim(spec): + """Latitudes, duraciones, barridos y grados de giro son parámetros de dibujo: + no dicen nada sobre el mundo y no se comprueban.""" + paths = " ".join(c.path for c in extract_claims(spec)) + for geometry in (".lat", ".lon", ".duration", ".sweeps", + ".contact_bearing_deg", ".markers", ".bounds"): + assert geometry not in paths + + +def test_a_split_quote_is_one_claim(spec): + """scale_bars.quote son las líneas de UNA cita: se comprueba entera, no a + trozos (el renderizador no envuelve; el caller parte las líneas).""" + quotes = [c.text for c in extract_claims(spec) if c.kind == "quote"] + assert "TWICE THE SIZE OF AN AIRCRAFT CARRIER" in quotes + assert "TWICE THE SIZE OF" not in quotes + + +# --- comprobación ----------------------------------------------------------- + +def test_reference_spec_is_fully_grounded(spec): + report = check_grounding(spec, CHUNKS) + assert report.ungrounded == [], \ + "sin fundamento: " + "; ".join(f"[{c.kind}] {c.text}" for c in report.ungrounded) + assert report.clean + assert report.total > 25 + assert report.chunk_count == 4 and report.url_count == 4 + + +def test_an_injected_figure_is_flagged_and_nothing_else(spec): + corrupted = copy.deepcopy(spec) + corrupted["shots"][7]["props"]["count_to"] = 12000 # eran 1.500 páginas + corrupted["shots"][1]["props"]["subline"] = "41,000 FT · 600 MPH" + + report = check_grounding(corrupted, CHUNKS) + flagged = {c.text for c in report.ungrounded} + assert flagged == {"12000", "41,000 FT"} + + +def test_an_invented_quote_is_flagged(spec): + corrupted = copy.deepcopy(spec) + corrupted["shots"][6]["props"]["quote_a"] = "“RADAR MALFUNCTION”" + + report = check_grounding(corrupted, CHUNKS) + assert [c.text for c in report.ungrounded] == ["RADAR MALFUNCTION"] + assert report.ungrounded[0].kind == "quote" + assert report.ungrounded[0].path.startswith("shots.6.document_quote.props.quote_a") + + +def test_an_invented_agency_is_flagged(spec): + corrupted = copy.deepcopy(spec) + corrupted["shots"][5]["props"]["strips"][2]["label"] = "NORAD CHEYENNE" + + report = check_grounding(corrupted, CHUNKS) + assert [c.text for c in report.ungrounded] == ["NORAD CHEYENNE"] + + +def test_an_invented_date_is_flagged(spec): + corrupted = copy.deepcopy(spec) + corrupted["shots"][1]["props"]["headline"] = "17 NOV 1987" + + report = check_grounding(corrupted, CHUNKS) + assert [c.text for c in report.ungrounded] == ["17 NOV 1987"] + + +def test_dates_match_across_formats(spec): + """El spec escribe "17 NOV 1986" y la fuente "November 17, 1986". Es la misma + fecha y el comprobador no debe gastarle un aviso al humano.""" + report = check_grounding(spec, [CHUNKS[0]]) + assert "17 NOV 1986" not in {c.text for c in report.ungrounded} + + +def test_units_match_their_spelled_out_form(spec): + """"35,000 FT" contra "35,000 feet".""" + report = check_grounding(spec, [CHUNKS[0]]) + assert "35,000 FT" not in {c.text for c in report.ungrounded} + + +def test_a_number_alone_is_not_enough_without_its_unit(): + """1.500 aparece en las fuentes como páginas; 1.500 FT no lo dice nadie.""" + spec = { + "version": 1, + "meta": {"id": "x", "title": "x"}, + "shots": [{"template": "scale_bars", "duration": 5.0, "props": { + "headline": "H", + "bars": [{"label": "ESTIMATED OBJECT", "value": 1500, "unit": "FT"}]}}], + } + report = check_grounding(spec, [CHUNKS[3]]) + assert [c.text for c in report.ungrounded] == ["1500 FT"] + + +def test_no_chunks_means_nothing_is_supported(spec): + """Sin material no se apoya nada. Aquí todo cae en `contaminated` porque el + spec de prueba ES el ejemplo del prompt — que es justo el diagnóstico + correcto: ninguna de esas cifras viene de la sesión.""" + report = check_grounding(spec, []) + assert report.grounded == [] + assert report.unsupported + assert not report.clean + assert report.chunk_count == 0 and report.url_count == 0 + + +# --- fuga del ejemplo del prompt -------------------------------------------- + +def test_a_figure_copied_from_the_prompt_example_is_diagnosed_as_such(spec): + """El caso real, medido el 2026-08-01 contra la sesión 153: el modelo + escribió "232 FT" (el largo de un 747) y eso no estaba en ninguno de los 126 + chunks — venía del ejemplo del prompt. No es una invención, es una fuga, y + se arregla borrándola, no verificándola.""" + sources_without_the_747 = [c for c in CHUNKS if "232" not in c["content"]] + + report = check_grounding(spec, sources_without_the_747) + + assert "232 FT" in {c.text for c in report.contaminated} + assert "232 FT" not in {c.text for c in report.ungrounded} + + +def test_an_invention_is_not_confused_with_a_leak(spec): + """Una cifra que no está ni en las fuentes ni en el ejemplo sigue siendo + una invención.""" + corrupted = copy.deepcopy(spec) + corrupted["shots"][1]["props"]["subline"] = "41,000 FT · 600 MPH" + + report = check_grounding(corrupted, CHUNKS) + + assert [c.text for c in report.ungrounded] == ["41,000 FT"] + assert report.contaminated == [] + + +def test_the_session_wins_over_the_example(spec): + """Si el dato SÍ está en las fuentes, está fundamentado y punto: que además + aparezca en el ejemplo no lo ensucia.""" + report = check_grounding(spec, CHUNKS) + assert report.contaminated == [] + assert report.clean + + +def test_a_leak_shows_up_in_the_report_with_its_own_wording(spec): + report = check_grounding(spec, [c for c in CHUNKS if "232" not in c["content"]]) + summary = report.summary() + assert "copiados del EJEMPLO" in summary + assert "232 FT" in summary + assert "fuga, no invención" in summary + + +def test_both_diagnoses_count_as_unsupported(spec): + corrupted = copy.deepcopy(spec) + corrupted["shots"][1]["props"]["subline"] = "41,000 FT · 600 MPH" + report = check_grounding(corrupted, [c for c in CHUNKS if "232" not in c["content"]]) + + assert len(report.unsupported) == len(report.ungrounded) + len(report.contaminated) + assert report.total == len(report.grounded) + len(report.unsupported) + assert not report.clean + + +def test_a_missing_example_file_degrades_to_the_old_behaviour(spec): + """El contraste con el ejemplo es un diagnóstico extra, no un requisito: sin + fichero, todo lo no encontrado vuelve a ser simplemente 'sin encontrar'.""" + report = check_grounding(spec, [], example_haystacks=()) + assert report.contaminated == [] + assert report.ungrounded + + +def test_summary_reports_success_out_loud(spec): + report = check_grounding(spec, CHUNKS) + summary = report.summary() + assert "0 sin encontrar" in summary # el éxito NO es silencioso + assert "4 chunks de 4 URLs" in summary + + +def test_summary_lists_every_ungrounded_string(spec): + corrupted = copy.deepcopy(spec) + corrupted["shots"][7]["props"]["count_to"] = 12000 + summary = check_grounding(corrupted, CHUNKS).summary() + assert "⚠️ 1 sin encontrar" in summary + assert '"12000"' in summary diff --git a/tests/test_layers.py b/tests/test_layers.py new file mode 100644 index 0000000..89f350b --- /dev/null +++ b/tests/test_layers.py @@ -0,0 +1,35 @@ +"""Separación de capas. + +`bot/` puede importar de todo; nadie puede importar de `bot/`. El progreso y +los callbacks viajan como callables genéricos justo para no necesitarlo. +""" +import re +from pathlib import Path + +SRC = Path(__file__).resolve().parents[1] / "src" +LOWER_LAYERS = ("generator", "scraper", "processor", "db", "seo", "news") + +IMPORTS_BOT = re.compile(r"^\s*(from\s+src\.bot|import\s+src\.bot|from\s+\.\.bot)", + re.MULTILINE) + + +def test_no_lower_layer_imports_from_bot(): + offenders = [] + for layer in LOWER_LAYERS: + for path in (SRC / layer).rglob("*.py"): + if IMPORTS_BOT.search(path.read_text(encoding="utf-8")): + offenders.append(str(path.relative_to(SRC.parent))) + assert not offenders, f"importan de bot/: {offenders}" + + +IMPORTS_TELEGRAM = re.compile(r"^\s*(from\s+telegram|import\s+telegram)", re.MULTILINE) + + +def test_the_short_pipeline_takes_progress_as_a_plain_callable(): + """La comprobación concreta para lo añadido en fase 2: si algún día alguien + mete un `Message` de Telegram aquí, este test lo dice. Nombrar Telegram en + un comentario vale — importarlo, no.""" + for module in ("short.py", "shortsmith.py", "shortspec.py", "grounding.py", + "spec_contract.py"): + source = (SRC / "generator" / module).read_text(encoding="utf-8") + assert not IMPORTS_TELEGRAM.search(source), f"{module} importa telegram" diff --git a/tests/test_short_golden.py b/tests/test_short_golden.py new file mode 100644 index 0000000..13e1da5 --- /dev/null +++ b/tests/test_short_golden.py @@ -0,0 +1,134 @@ +"""Eval dorada: ¿podría este pipeline haber producido el vídeo que ya sabemos +que está bien? + +Corre el generador entero contra una sesión REAL de investigación y compara el +spec resultante con `examples/jal1628.json` — que es el que produjo el primer +Short — de forma ESTRUCTURAL: número de shots, plantillas elegidas, duración +total y claims sin fundamento. Nunca por igualdad de cadenas: el modelo +redactará distinto y eso no es un fallo. + +Necesita una sesión de verdad, así que se salta salvo que se le dé todo: + + RESEARCHOWL_GOLDEN_DB=/ruta/a/researchowl.db \\ + RESEARCHOWL_GOLDEN_SESSION=153 \\ + SHORTSMITH_LIVE_URL=http://10.43.86.57:8080 \\ + ANTHROPIC_API_KEY=... \\ + pytest tests/test_short_golden.py -v -s + +Para sacar la sesión del cluster sin arrastrar la DB entera, `make golden-db`. +""" +import json +import os +from pathlib import Path + +import pytest + +EXAMPLE = Path(__file__).resolve().parents[1] / "src/generator/examples/jal1628.json" + +GOLDEN_DB = os.environ.get("RESEARCHOWL_GOLDEN_DB") +GOLDEN_SESSION = os.environ.get("RESEARCHOWL_GOLDEN_SESSION") +LIVE_URL = os.environ.get("SHORTSMITH_LIVE_URL") + +pytestmark = pytest.mark.skipif( + not (GOLDEN_DB and GOLDEN_SESSION and LIVE_URL and os.environ.get("ANTHROPIC_API_KEY")), + reason="la eval dorada necesita una sesión real, shortsmith vivo y clave de Claude") + + +def structure(spec: dict) -> dict: + """Lo comparable de un spec: forma, no palabras.""" + shots = spec.get("shots", []) + return { + "shots": len(shots), + "templates": [s["template"] for s in shots], + "duration": sum(s["duration"] for s in shots), + "distinct_templates": len({s["template"] for s in shots}), + } + + +@pytest.fixture(scope="module") +def produced(): + """Genera UNA vez (cuesta dinero) y reparte el resultado a los tests.""" + import asyncio + + from src.config import settings + settings.db_path = GOLDEN_DB + settings.shortsmith_url = LIVE_URL + settings.shortsmith_enabled = True + + from src.db.database import ResearchDB, close_db, get_db + from src.generator.short import ShortProducer + from src.processor.processor import ContentProcessor, OllamaClient + + async def run(): + conn = await get_db() + try: + db = ResearchDB(conn) + producer = ShortProducer(db, ContentProcessor(db, OllamaClient())) + return await producer.produce(int(GOLDEN_SESSION), + lambda text: print(" ", text)) + finally: + await close_db() + + return asyncio.run(run()) + + +def test_the_pipeline_produces_a_renderable_short(produced): + assert produced.spec is not None, produced.failure + assert produced.failure is None, produced.failure + assert produced.has_video + assert Path(produced.video_path).stat().st_size > 100_000 + + +def test_the_shape_matches_the_reference(produced): + reference = structure(json.loads(EXAMPLE.read_text())) + got = structure(produced.spec) + print(f"\nreferencia: {reference}\nobtenido: {got}") + + # El vídeo bueno son 8 shots; ±3 sigue siendo la misma forma narrativa. + assert abs(got["shots"] - reference["shots"]) <= 3 + assert got["distinct_templates"] >= 4, "un Short de una sola plantilla es un cartel" + assert 20.0 <= got["duration"] <= 45.0 + # Un case_file abre con contexto y cierra con el contador: no se exige la + # misma lista de plantillas, sí que el cierre sea un cierre. + assert got["templates"][-1] == reference["templates"][-1] + + +def test_no_claim_was_invented(produced): + """Lo que de verdad decide si esto se puede publicar. + + Medido el 2026-08-01 contra la sesión 153, en dos tiradas: 36-37 claims de + 38 casan. Lo que se escapa es de dos clases conocidas y ninguna se arregla + endureciendo este assert: + + * "232 FT" (el largo de un 747) copiado del ejemplo de la sección 5, que no + está en NINGUNO de los 126 chunks de la sesión — el comprobador lo + etiqueta ya como fuga del ejemplo, no como invención; + * una cita comprimida — "WALNUT SHAPED WIDE RIM" donde la fuente dice + "walnut shaped with a wide rim around its circumference". + + El prompt ataca las dos, pero el muestreo del modelo varía entre tiradas, y + un test que gasta $0.05 y depende del muestreo no sirve de puerta. **La + puerta de verdad es el informe de claims en Telegram**, que se manda + siempre. Esto sólo vigila que el fundamento no se desplome. + """ + report = produced.grounding + print("\n" + report.summary()) + + assert len(report.grounded) >= 30, "el spec dejó de apoyarse en las fuentes" + assert len(report.unsupported) <= 3, \ + "sin fundamento: " + "; ".join(f"[{c.kind}] {c.text}" for c in report.unsupported) + + +def test_it_did_not_take_many_attempts(produced): + """Métrica del §4: si esto sube de 1.5 de media, lo que hay que arreglar es + el prompt, no el número de reintentos.""" + print(f"\nintentos hasta válido: {produced.attempts}; coste ${produced.cost_usd:.4f}") + assert produced.attempts <= 2 + + +def test_it_costs_what_a_blog_costs(produced): + """Medido, no estimado: 40 chunks de contexto son ~25k tokens de entrada, y + un reintento los paga otra vez. El §9 del spec calculaba $0.003-0.008 con un + contexto mucho más corto; con este, dos intentos salen por ~$0.05. Sigue + siendo lo que cuesta un /generate blog, que era el punto.""" + assert produced.cost_usd < 0.08 diff --git a/tests/test_short_producer.py b/tests/test_short_producer.py new file mode 100644 index 0000000..173e030 --- /dev/null +++ b/tests/test_short_producer.py @@ -0,0 +1,326 @@ +"""El pipeline del Short: orden de los pasos y fallbacks. + +Todo con dobles: ni Claude ni shortsmith ni SQLite. Lo que se comprueba aquí es +que el fundamento se mira ANTES de renderizar y que ningún camino de fallo se +come el spec. +""" +import json + +import pytest + +from src.config import settings +from src.generator.short import ShortProducer, ShortResult, ShortsDisabled +from src.generator.shortsmith import JobResult, ShortsmithError, ShortsmithUnavailable +from tests.test_spec_contract import TEMPLATES + +SPEC = { + "version": 1, + "meta": {"id": "jal1628", "title": "JAL 1628"}, + "shots": [ + {"template": "radar_sweep", "duration": 15.0, "props": {"headline": "3 RADARS"}}, + {"template": "scale_bars", "duration": 15.0, "props": { + "headline": "REPORTED SCALE", + "bars": [{"label": "BOEING 747", "value": 232, "unit": "FT"}]}}, + ], +} + +CHUNKS = [{ + "content": "Three radars tracked the object. A Boeing 747 is 232 ft long.", + "url": "https://faa.example/jal1628", + "title": "FAA file", + "source_type": "web", +}] + + +class FakeDB: + def __init__(self, article_url=None): + self.article_url = article_url + self.saved: list[tuple] = [] + + async def get_session(self, session_id): + return {"id": session_id, "topic": "JAL 1628 Alaska 1986"} + + async def get_article_url(self, session_id): + return self.article_url + + async def save_output(self, session_id, output_type, content): + self.saved.append((session_id, output_type, content)) + return len(self.saved) + + async def log_api_call(self, *a, **kw): + return None + + +class FakeProcessor: + def __init__(self, chunks=None): + self.chunks = CHUNKS if chunks is None else chunks + + async def rag_chunks(self, session_id, query, top_k=20): + return self.chunks + + +class FakeClient: + """shortsmith de mentira. `fail_at` decide dónde se rompe.""" + + def __init__(self, fail_at=None, job_status="done", warnings=None): + self.fail_at = fail_at + self.job_status = job_status + self.warnings = warnings or [] + self.rendered = None + + async def templates(self, refresh=False): + if self.fail_at == "templates": + raise ShortsmithUnavailable("no hay nadie al otro lado") + return TEMPLATES + + async def render(self, spec): + if self.fail_at == "render": + raise ShortsmithUnavailable("conexión rechazada") + self.rendered = spec + return "job-1" + + async def poll(self, job_id, on_progress=None, **kw): + if self.fail_at == "poll": + raise ShortsmithError("job atascado") + if on_progress: + await on_progress(0.5, "running") + return JobResult(job_id, self.job_status, 1.0, self.warnings, + "OOMKilled" if self.job_status == "error" else None) + + async def fetch_video(self, job_id): + if self.fail_at == "fetch": + raise ShortsmithError("404 del vídeo") + return b"\x00\x00\x00 ftypisom" + b"\x00" * 2048 + + +def llm_returning(*responses): + queue = list(responses) + + async def call(system, prompt): + call.prompts.append(prompt) + return queue.pop(0) if len(queue) > 1 else queue[0] + + call.prompts = [] + return call + + +def producer(tmp_path, monkeypatch, *, client=None, llm=None, db=None, processor=None): + monkeypatch.setattr(settings, "shorts_dir", str(tmp_path / "shorts")) + monkeypatch.setattr(settings, "shortsmith_enabled", True) + return ShortProducer( + db or FakeDB(), + processor or FakeProcessor(), + client=client or FakeClient(), + llm_call=llm or llm_returning(json.dumps(SPEC)), + ) + + +# --- camino feliz ----------------------------------------------------------- + +@pytest.mark.asyncio +async def test_happy_path_writes_the_mp4_to_disk(tmp_path, monkeypatch): + db = FakeDB(article_url="https://www.theexclusionzone.com/jal-1628/") + p = producer(tmp_path, monkeypatch, db=db) + + result = await p.produce(153) + + assert result.has_video + assert result.video_path.endswith("153.mp4") + assert open(result.video_path, "rb").read()[:12].endswith(b"ftypisom") + assert result.title == "JAL 1628" + assert result.duration_s == 30.0 + assert result.article_url.endswith("/jal-1628/") + assert result.failure is None + + +@pytest.mark.asyncio +async def test_the_spec_is_saved_before_the_render(tmp_path, monkeypatch): + """Si el render se cae, la parte cara ya está en la DB y /short_spec la + devuelve.""" + db = FakeDB() + p = producer(tmp_path, monkeypatch, db=db, client=FakeClient(fail_at="render")) + + result = await p.produce(153) + + assert db.saved and db.saved[0][1] == "short_en" + assert json.loads(db.saved[0][2])["meta"]["id"] == "jal1628" + assert not result.has_video + + +@pytest.mark.asyncio +async def test_grounding_runs_before_rendering(tmp_path, monkeypatch): + """El informe existe aunque el render no llegue a empezar: ese es el orden + del §12 y es lo que hace que la revisión humana llegue igual.""" + p = producer(tmp_path, monkeypatch, client=FakeClient(fail_at="render")) + + result = await p.produce(153) + + assert result.grounding is not None + assert result.grounding.total > 0 + + +@pytest.mark.asyncio +async def test_ungrounded_claims_do_not_block_the_render(tmp_path, monkeypatch): + """Un dato sin encontrar puede ser una fabricación o un artefacto de + formato. Lo decide una persona: el vídeo se entrega con el aviso al lado.""" + invented = json.loads(json.dumps(SPEC)) + invented["shots"][0]["props"]["headline"] = "41,000 FT" + p = producer(tmp_path, monkeypatch, llm=llm_returning(json.dumps(invented))) + + result = await p.produce(153) + + assert result.has_video + assert [c.text for c in result.grounding.ungrounded] == ["41,000 FT"] + + +@pytest.mark.asyncio +async def test_the_article_url_reaches_the_prompt(tmp_path, monkeypatch): + llm = llm_returning(json.dumps(SPEC)) + p = producer(tmp_path, monkeypatch, + db=FakeDB(article_url="https://www.theexclusionzone.com/jal-1628/"), + llm=llm) + + await p.produce(153) + + assert "https://www.theexclusionzone.com/jal-1628/" in llm.prompts[0] + + +# --- fallbacks -------------------------------------------------------------- + +@pytest.mark.asyncio +@pytest.mark.parametrize("fail_at", ["render", "poll", "fetch"]) +async def test_every_render_failure_still_returns_the_spec(tmp_path, monkeypatch, fail_at): + p = producer(tmp_path, monkeypatch, client=FakeClient(fail_at=fail_at)) + + result = await p.produce(153) + + assert not result.has_video + assert result.failure + assert json.loads(result.spec_json)["meta"]["id"] == "jal1628" + + +@pytest.mark.asyncio +async def test_a_job_that_errors_is_reported_with_its_reason(tmp_path, monkeypatch): + p = producer(tmp_path, monkeypatch, client=FakeClient(job_status="error")) + + result = await p.produce(153) + + assert not result.has_video + assert "OOMKilled" in result.failure + + +@pytest.mark.asyncio +async def test_an_unwritable_spec_still_returns_the_last_attempt(tmp_path, monkeypatch): + """Tres intentos fallidos no son motivo para tirar la generación.""" + broken = json.loads(json.dumps(SPEC)) + broken["meta"]["id"] = "MAYÚSCULAS Y ESPACIOS" + p = producer(tmp_path, monkeypatch, llm=llm_returning(json.dumps(broken))) + + result = await p.produce(153) + + assert not result.has_video + assert result.attempts == 3 + assert result.spec["meta"]["id"] == "MAYÚSCULAS Y ESPACIOS" + assert "no pasó la validación" in result.failure + assert result.grounding is None # no hay spec válido que comprobar + + +@pytest.mark.asyncio +async def test_a_response_that_is_not_json_at_all_comes_back_raw(tmp_path, monkeypatch): + p = producer(tmp_path, monkeypatch, + llm=llm_returning("Lo siento, no puedo ayudarte con eso.")) + + result = await p.produce(153) + + assert result.spec is None + assert "Lo siento" in result.raw_response + + +@pytest.mark.asyncio +async def test_render_warnings_travel_with_the_result(tmp_path, monkeypatch): + warnings = [{"template": "data_card", "text": "UNA FILA MUY LARGA", + "requested": 44, "size": 38}] + p = producer(tmp_path, monkeypatch, client=FakeClient(warnings=warnings)) + + result = await p.produce(153) + + assert result.has_video + assert result.render_warnings[0]["template"] == "data_card" + + +# --- interruptores y precondiciones ----------------------------------------- + +@pytest.mark.asyncio +async def test_the_kill_switch_says_so_instead_of_crashing(tmp_path, monkeypatch): + p = producer(tmp_path, monkeypatch) + monkeypatch.setattr(settings, "shortsmith_enabled", False) + + with pytest.raises(ShortsDisabled): + await p.produce(153) + + +@pytest.mark.asyncio +async def test_an_unreachable_renderer_is_named_clearly(tmp_path, monkeypatch): + """Sin contrato no hay prompt que escribir: aquí no hay fallback posible y + el mensaje lo dice.""" + p = producer(tmp_path, monkeypatch, client=FakeClient(fail_at="templates")) + + with pytest.raises(ShortsmithUnavailable): + await p.produce(153) + + +@pytest.mark.asyncio +async def test_a_session_without_chunks_says_what_to_run(tmp_path, monkeypatch): + p = producer(tmp_path, monkeypatch, processor=FakeProcessor(chunks=[])) + + with pytest.raises(ValueError, match="/process"): + await p.produce(153) + + +def test_domain_is_drawn_without_protocol_or_www(monkeypatch): + monkeypatch.setattr(settings, "ghost_url_en", "https://www.theexclusionzone.com") + assert ShortProducer(FakeDB(), FakeProcessor(), client=FakeClient())._domain() \ + == "THEEXCLUSIONZONE.COM" + + +def test_short_result_without_a_spec_has_an_empty_json(): + assert ShortResult(topic="x").spec_json == "" + + +# --- limpieza --------------------------------------------------------------- + +@pytest.mark.asyncio +async def test_purging_a_session_takes_its_video_with_it(tmp_path, monkeypatch): + """Un MP4 huérfano en el PVC es negligible contra 5 Gi y es arqueología + dentro de un año.""" + import time + + import aiosqlite + + from src.db import database + from src.db.database import ResearchDB + + shorts = tmp_path / "shorts" + shorts.mkdir() + (shorts / "1.mp4").write_bytes(b"viejo") + (shorts / "2.mp4").write_bytes(b"reciente") + monkeypatch.setattr(settings, "shorts_dir", str(shorts)) + + conn = await aiosqlite.connect(tmp_path / "t.db") + conn.row_factory = aiosqlite.Row + await conn.executescript(database.SCHEMA) + old, now = time.time() - 90 * 86400, time.time() + await conn.execute( + "INSERT INTO research_sessions (id, topic, status, telegram_chat_id," + " created_at, updated_at) VALUES (1,'viejo','saturated',1,?,?)", (old, old)) + await conn.execute( + "INSERT INTO research_sessions (id, topic, status, telegram_chat_id," + " created_at, updated_at) VALUES (2,'nuevo','saturated',1,?,?)", (now, now)) + await conn.commit() + + counts = await ResearchDB(conn).purge_old_sessions(30) + await conn.close() + + assert counts["shorts"] == 1 + assert not (shorts / "1.mp4").exists() + assert (shorts / "2.mp4").exists(), "la sesión reciente conserva su vídeo" diff --git a/tests/test_shortsmith_client.py b/tests/test_shortsmith_client.py new file mode 100644 index 0000000..7c4dc77 --- /dev/null +++ b/tests/test_shortsmith_client.py @@ -0,0 +1,208 @@ +"""ShortsmithClient — bucle de sondeo, errores y fallbacks. + +Todo con un servidor falso; el test contra el servicio vivo es +`test_shortsmith_live.py`, que se salta salvo que se le apunte a uno. +""" +import asyncio +import json +from pathlib import Path + +import pytest + +from src.generator.shortsmith import ( + JobResult, ShortsmithClient, ShortsmithError, ShortsmithRejected, + ShortsmithUnavailable, _templates_cache, +) + +EXAMPLE = Path(__file__).resolve().parents[1] / "src/generator/examples/jal1628.json" + + +@pytest.fixture +def spec(): + return json.loads(EXAMPLE.read_text()) + + +class FakeResp: + def __init__(self, status, payload=None, body="", raw=b""): + self.status = status + self._payload = payload + self._body = body + self._raw = raw + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return False + + async def json(self): + if self._payload is None: + raise ValueError("no json") + return self._payload + + async def text(self): + return self._body + + async def read(self): + return self._raw + + +class FakeSession: + """Sustituye a aiohttp.ClientSession: sirve respuestas de una cola por ruta.""" + + def __init__(self, routes): + self.routes = routes + self.calls = [] + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return False + + def _next(self, method, url): + self.calls.append((method, url)) + for pattern, responses in self.routes.items(): + if pattern in url: + if isinstance(responses, list): + return responses.pop(0) if len(responses) > 1 else responses[0] + return responses + raise AssertionError(f"ruta no simulada: {method} {url}") + + def get(self, url, **kw): + return self._next("GET", url) + + def post(self, url, **kw): + return self._next("POST", url) + + +def patch_session(client, routes): + session = FakeSession(routes) + client._session = lambda total: session + return session + + +@pytest.mark.asyncio +async def test_templates_cached_per_process(): + _templates_cache.clear() + client = ShortsmithClient("http://fake:8080") + session = patch_session(client, {"/templates": FakeResp(200, {"radar_sweep": {}})}) + + first = await client.templates() + second = await client.templates() + + assert first == second == {"radar_sweep": {}} + assert len(session.calls) == 1, "la segunda llamada debe salir de la caché" + + # refresh=True vuelve a pedirlo: el renderizador puede haberse actualizado. + patch_session(client, {"/templates": FakeResp(200, {"radar_sweep": {}, "nueva": {}})}) + assert "nueva" in await client.templates(refresh=True) + _templates_cache.clear() + + +@pytest.mark.asyncio +async def test_render_returns_job_id(spec): + client = ShortsmithClient("http://fake:8080") + patch_session(client, {"/render": FakeResp(202, {"job_id": "abc123", "status": "queued"})}) + assert await client.render(spec) == "abc123" + + +@pytest.mark.asyncio +async def test_render_422_propagates_error_paths(spec): + detail = [{ + "type": "extra_forbidden", + "loc": ["shots", 0, "radar_sweep", "props", "sweeeps"], + "msg": "Extra inputs are not permitted", + }] + client = ShortsmithClient("http://fake:8080") + patch_session(client, {"/render": FakeResp(422, {"detail": detail})}) + + with pytest.raises(ShortsmithRejected) as exc: + await client.render(spec) + assert exc.value.errors[0]["loc"][-1] == "sweeeps" + + +@pytest.mark.asyncio +async def test_poll_queued_then_running_then_done(): + client = ShortsmithClient("http://fake:8080") + patch_session(client, {"/jobs/": [ + FakeResp(200, {"job_id": "j", "status": "queued", "progress": 0.0}), + FakeResp(200, {"job_id": "j", "status": "running", "progress": 0.4}), + FakeResp(200, {"job_id": "j", "status": "done", "progress": 1.0, + "warnings": [{"template": "data_card", "text": "x"}]}), + ]}) + + seen = [] + + async def on_progress(fraction, status): + seen.append((fraction, status)) + + result = await client.poll("j", on_progress=on_progress, interval=0) + + assert result.ok and result.status == "done" + assert result.warnings and result.warnings[0]["template"] == "data_card" + assert seen == [(0.0, "queued"), (0.4, "running"), (1.0, "done")] + + +@pytest.mark.asyncio +async def test_poll_returns_error_status_without_raising(): + client = ShortsmithClient("http://fake:8080") + patch_session(client, {"/jobs/": FakeResp(200, { + "job_id": "j", "status": "error", "progress": 0.3, + "error": "interrupted by a restart: the process did not survive this render", + })}) + + result = await client.poll("j", interval=0) + assert not result.ok + assert "interrupted" in result.error + + +@pytest.mark.asyncio +async def test_poll_gives_up_on_a_stuck_job(): + client = ShortsmithClient("http://fake:8080") + patch_session(client, {"/jobs/": FakeResp(200, { + "job_id": "j", "status": "running", "progress": 0.1})}) + + with pytest.raises(ShortsmithError, match="atascado"): + await client.poll("j", interval=0, ceiling=0) + + +@pytest.mark.asyncio +async def test_connection_refused_is_unavailable(spec): + import aiohttp + + class Refusing(FakeSession): + def post(self, url, **kw): + raise aiohttp.ClientConnectionError( + "Cannot connect to host shortsmith-svc:8080 [Connection refused]") + + client = ShortsmithClient("http://fake:8080") + client._session = lambda total: Refusing({}) + + with pytest.raises(ShortsmithUnavailable): + await client.render(spec) + + +@pytest.mark.asyncio +async def test_fetch_video_returns_bytes(): + client = ShortsmithClient("http://fake:8080") + patch_session(client, {"/video": FakeResp(200, raw=b"\x00\x00\x00 ftypisom")}) + assert (await client.fetch_video("j")).startswith(b"\x00\x00\x00 ftyp") + + +@pytest.mark.asyncio +async def test_progress_callback_failure_never_kills_the_render(): + client = ShortsmithClient("http://fake:8080") + patch_session(client, {"/jobs/": FakeResp(200, { + "job_id": "j", "status": "done", "progress": 1.0})}) + + async def boom(fraction, status): + raise RuntimeError("Telegram dijo que no") + + assert (await client.poll("j", on_progress=boom, interval=0)).ok + + +def test_jobresult_ok_only_when_done(): + assert JobResult("j", "done").ok + assert not JobResult("j", "running").ok + assert not JobResult("j", "error", error="boom").ok diff --git a/tests/test_shortsmith_live.py b/tests/test_shortsmith_live.py new file mode 100644 index 0000000..6a7ec6b --- /dev/null +++ b/tests/test_shortsmith_live.py @@ -0,0 +1,68 @@ +"""Prueba de fontanería contra un shortsmith VIVO. + +Se salta salvo que se le dé una URL alcanzable desde donde corren los tests: + + SHORTSMITH_LIVE_URL=http://10.43.86.57:8080 pytest tests/test_shortsmith_live.py -v + +(dentro del cluster es `http://shortsmith-svc.shortsmith.svc.cluster.local:8080`; +desde el nodo, la ClusterIP de `kubectl get svc -n shortsmith`). + +Renderiza el ejemplo de referencia entero — ~30 s de CPU en el pod — y comprueba +que vuelve un MP4. Es el paso 1 del §12 del spec de fase 2: probar el transporte +antes de generar nada. +""" +import json +import os +from pathlib import Path + +import pytest + +from src.generator.shortsmith import ShortsmithClient + +LIVE_URL = os.environ.get("SHORTSMITH_LIVE_URL") +pytestmark = pytest.mark.skipif( + not LIVE_URL, reason="define SHORTSMITH_LIVE_URL para probar contra el servicio vivo") + +EXAMPLE = Path(__file__).resolve().parents[1] / "src/generator/examples/jal1628.json" + + +@pytest.mark.asyncio +async def test_healthz_and_templates(): + client = ShortsmithClient(LIVE_URL) + + health = await client.health() + assert health["status"] == "ok" + assert health["templates"] >= 1 + + templates = await client.templates(refresh=True) + # No se comprueban nombres concretos a propósito: el contrato es de + # shortsmith y añadir plantillas allí no debe romper aquí. + assert templates, "GET /templates devolvió vacío" + for name, schema in templates.items(): + assert schema.get("type") == "object", f"{name} no publica un esquema de objeto" + assert "properties" in schema + + +@pytest.mark.asyncio +async def test_render_the_reference_example_end_to_end(tmp_path): + client = ShortsmithClient(LIVE_URL) + spec = json.loads(EXAMPLE.read_text()) + + job_id = await client.render(spec) + seen = [] + + async def on_progress(fraction, status): + seen.append(fraction) + + result = await client.poll(job_id, on_progress=on_progress) + assert result.ok, f"el job terminó en {result.status}: {result.error}" + assert seen and max(seen) == 1.0 + + video = await client.fetch_video(job_id) + # ftyp en los primeros bytes: es un MP4 de verdad, no una página de error. + assert b"ftyp" in video[:32] + assert len(video) > 100_000, f"solo {len(video)} bytes — sospechosamente corto" + + out = tmp_path / "jal1628.mp4" + out.write_bytes(video) + print(f"\nrenderizado {len(video)/1e6:.2f} MB en {out}") diff --git a/tests/test_shortspec.py b/tests/test_shortspec.py new file mode 100644 index 0000000..b7a0f27 --- /dev/null +++ b/tests/test_shortspec.py @@ -0,0 +1,233 @@ +"""Escritura del shot spec: prompt, bucle de reintento y fallback. + +El LLM entra como un callable, así que aquí se prueba el bucle, no a Haiku: +respuesta buena, respuesta malformada, typo en una prop, y las tres seguidas. +""" +import json +from pathlib import Path + +import pytest + +from src.generator.shortspec import ( + MAX_ATTEMPTS, ShortSpecWriter, SpecWriteFailed, extract_json, +) +from tests.test_spec_contract import TEMPLATES + +EXAMPLE = Path(__file__).resolve().parents[1] / "src/generator/examples/jal1628.json" + +GOOD = { + "version": 1, + "meta": {"id": "caso", "title": "Un caso"}, + "shots": [ + {"template": "radar_sweep", "duration": 15.0, "props": {"headline": "3 RADARS"}}, + {"template": "scale_bars", "duration": 15.0, "props": { + "headline": "REPORTED SCALE", + "bars": [{"label": "BOEING 747", "value": 232, "unit": "FT"}]}}, + ], +} + + +class FakeLLM: + """Devuelve respuestas de una cola y guarda los prompts que recibió.""" + + def __init__(self, *responses): + self.responses = list(responses) + self.prompts: list[str] = [] + self.systems: list[str] = [] + + async def __call__(self, system, prompt): + self.systems.append(system) + self.prompts.append(prompt) + return self.responses.pop(0) if len(self.responses) > 1 else self.responses[0] + + +def writer(*responses, **kw): + llm = FakeLLM(*responses) + return ShortSpecWriter(llm, TEMPLATES, **kw), llm + + +# --- parseo ----------------------------------------------------------------- + +def test_extract_json_survives_markdown_fences(): + assert extract_json('```json\n{"a": 1}\n```') == {"a": 1} + assert extract_json('Here you go:\n{"a": 1}\nHope that helps') == {"a": 1} + assert extract_json('{"a": 1}') == {"a": 1} + + +def test_extract_json_complains_when_there_is_no_object(): + with pytest.raises(ValueError): + extract_json("I'm afraid I can't do that") + + +def test_straight_quotes_inside_a_string_are_repaired(): + """Cómo falla esto en la vida real (sesión de Bélgica, 2026-08-01): el + modelo escribe la cita con comillas rectas, que cierran la cadena JSON antes + de tiempo. Se arregla aquí porque además es lo que se quiere dibujar.""" + broken = '{"quote_a": ""CREDIBLE PEOPLE. THEY TOLD WHAT THEY SAW."", "n": 1}' + assert extract_json(broken) == { + "quote_a": "“CREDIBLE PEOPLE. THEY TOLD WHAT THEY SAW.”", "n": 1} + + +def test_the_repair_leaves_correct_json_alone(): + good = {"a": 'texto con “tipográficas” dentro', "b": [1, 2], "c": {"d": "e"}} + assert extract_json(json.dumps(good, ensure_ascii=False)) == good + + +def test_the_repair_does_not_eat_escaped_quotes(): + assert extract_json(r'{"a": "dijo \"hola\" y se fue"}') == {"a": 'dijo "hola" y se fue'} + + +def test_an_unrepairable_response_reports_the_offending_fragment(): + """El modelo no ve su salida numerada: "line 189 column 22" no le sirve; el + trozo sí.""" + with pytest.raises(ValueError) as exc: + extract_json('{"a": 1, "b": [1, 2,,,], "c": 3}') + assert "aquí:" in str(exc.value) + + +# --- el prompt -------------------------------------------------------------- + +def test_prompt_carries_the_fetched_contract_not_a_copy(): + w, _ = writer("{}") + prompt = w.build_prompt("Caso X", "material", None, "THEEXCLUSIONZONE.COM") + assert "radar_sweep:" in prompt and "contact_bearing_deg" in prompt + assert "1-3 elementos" in prompt # los límites de longitud, del esquema + assert "ink, amber, amber_dark" in prompt # la paleta, también del esquema + + +def test_prompt_states_the_editorial_constraints(): + w, _ = writer("{}") + prompt = w.build_prompt("Caso X", "material", "https://x.test/post", "X.TEST") + assert "20-45 seconds" in prompt + assert "never hex" in prompt + assert "https://x.test/post" in prompt + assert "X.TEST" in prompt + assert "material" in prompt + + +def test_prompt_includes_the_worked_example_in_full(): + w, _ = writer("{}") + prompt = w.build_prompt("Caso X", "material", None, "X.TEST") + example = json.loads(EXAMPLE.read_text()) + assert example["meta"]["title"] in prompt + assert "counter_close" in prompt + + +def test_prompt_says_out_loud_that_there_is_no_article_yet(): + w, _ = writer("{}") + assert "No article URL yet" in w.build_prompt("X", "m", None, "X.TEST") + + +# --- bucle ------------------------------------------------------------------ + +@pytest.mark.asyncio +async def test_a_good_response_validates_on_the_first_attempt(): + w, llm = writer(json.dumps(GOOD)) + result = await w.write("Caso X", "material") + assert result.attempts == 1 + assert result.spec["meta"]["id"] == "caso" + assert len(llm.prompts) == 1 + + +@pytest.mark.asyncio +async def test_malformed_json_triggers_a_retry(): + w, llm = writer("no soy JSON", json.dumps(GOOD)) + result = await w.write("Caso X", "material") + assert result.attempts == 2 + assert "no es un objeto JSON válido" in result.history[0][0] + assert "previous attempt was rejected" in llm.prompts[1] + + +@pytest.mark.asyncio +async def test_the_exact_error_paths_are_fed_back_verbatim(): + """La ruta que devuelve la validación es lo más útil que se le puede dar al + modelo: se le pasa tal cual, sin parafrasear.""" + bad = json.loads(json.dumps(GOOD)) + bad["shots"][0]["props"]["sweeeps"] = 2 + w, llm = writer(json.dumps(bad), json.dumps(GOOD)) + + result = await w.write("Caso X", "material") + + assert result.attempts == 2 + assert "shots.0.radar_sweep.props.sweeeps" in llm.prompts[1] + assert "sweeps" in llm.prompts[1] # y cuáles sí valen + + +@pytest.mark.asyncio +async def test_three_failures_raise_but_keep_the_last_attempt(): + """La parte cara es la generación, no el render: el último intento viaja en + la excepción para poder editarlo a mano y reenviarlo.""" + bad = json.loads(json.dumps(GOOD)) + bad["meta"]["id"] = "Caso Con Espacios" + w, _ = writer(json.dumps(bad)) + + with pytest.raises(SpecWriteFailed) as exc: + await w.write("Caso X", "material") + + assert exc.value.attempts == MAX_ATTEMPTS + assert exc.value.last_spec["meta"]["id"] == "Caso Con Espacios" + assert any("meta.id" in e for e in exc.value.errors) + + +@pytest.mark.asyncio +async def test_an_off_target_duration_is_commented_once_then_accepted(): + """70 s cumple el contrato pero no el encargo: se comenta y, si el modelo + insiste, se renderiza igual antes que tirar la generación.""" + long_spec = json.loads(json.dumps(GOOD)) + long_spec["shots"][0]["duration"] = 55.0 # 70 s en total + w, llm = writer(json.dumps(long_spec)) + + result = await w.write("Caso X", "material") + + assert result.attempts == MAX_ATTEMPTS + assert result.notes and "recorta" in result.notes[0] + assert "off-brief" in llm.prompts[1] + + +@pytest.mark.asyncio +async def test_a_valid_attempt_is_not_thrown_away_by_a_worse_one(): + long_spec = json.loads(json.dumps(GOOD)) + long_spec["shots"][0]["duration"] = 55.0 + w, _ = writer(json.dumps(long_spec), "esto ya no es JSON", "tampoco") + + result = await w.write("Caso X", "material") + + assert result.spec["shots"][0]["duration"] == 55.0 + assert result.notes + + +@pytest.mark.asyncio +async def test_the_contract_is_refetched_after_a_validation_failure(): + """Si el renderizador se actualizó a mitad de la run, la plantilla nueva + entra en el segundo intento.""" + new_template = {"type": "object", "additionalProperties": False, + "required": ["title"], + "properties": {"title": {"type": "string", "minLength": 1}}} + refreshed = {**TEMPLATES, "holo_scan": new_template} + + async def refresh(): + return refreshed + + with_new = json.loads(json.dumps(GOOD)) + with_new["shots"][1] = {"template": "holo_scan", "duration": 15.0, + "props": {"title": "X"}} + w, llm = writer(json.dumps(with_new), json.dumps(with_new), + refresh_templates=refresh) + + result = await w.write("Caso X", "material") + + assert result.attempts == 2 + assert "holo_scan" in llm.prompts[1] + assert result.spec["shots"][1]["template"] == "holo_scan" + + +@pytest.mark.asyncio +async def test_progress_is_reported_only_when_it_retries(): + seen = [] + + async def on_progress(text): + seen.append(text) + + w, _ = writer("no JSON", json.dumps(GOOD)) + await w.write("Caso X", "material", on_progress=on_progress) + assert len(seen) == 1 and "attempt 2/3" in seen[0] diff --git a/tests/test_spec_contract.py b/tests/test_spec_contract.py new file mode 100644 index 0000000..7abafb1 --- /dev/null +++ b/tests/test_spec_contract.py @@ -0,0 +1,231 @@ +"""Validación local del spec contra el esquema publicado por shortsmith. + +Los esquemas de abajo son una COPIA REDUCIDA de lo que devuelve +`GET /templates`, sólo para los tests: en producción se piden en vivo. Si +shortsmith cambia el contrato, quien lo nota es `test_shortsmith_live.py`, no +esto. +""" +import copy +import json +from pathlib import Path + +import pytest + +from src.generator.spec_contract import ( + SpecInvalid, describe_templates, editorial_notes, validate_spec, +) + +EXAMPLE = Path(__file__).resolve().parents[1] / "src/generator/examples/jal1628.json" + +TEMPLATES = { + "radar_sweep": { + "type": "object", "additionalProperties": False, + "required": ["headline"], + "properties": { + "headline": {"type": "string", "minLength": 1}, + "subline": {"type": "string", "default": ""}, + "contact_bearing_deg": {"type": "number", "minimum": 0, + "exclusiveMaximum": 360, "default": 210.0}, + "sweeps": {"type": "number", "exclusiveMinimum": 0, "maximum": 10, + "default": 2.0}, + }, + }, + "scale_bars": { + "type": "object", "additionalProperties": False, + "required": ["headline", "bars"], + "$defs": {"Bar": { + "type": "object", "additionalProperties": False, + "required": ["label", "value"], + "properties": { + "label": {"type": "string", "minLength": 1}, + "value": {"type": "number", "exclusiveMinimum": 0}, + "unit": {"type": "string", "default": ""}, + "color": {"enum": ["ink", "amber", "amber_dark", "muted", "dim", "red"], + "type": "string", "default": "ink"}, + "value_label": {"type": "string", "default": ""}, + }, + }}, + "properties": { + "headline": {"type": "string", "minLength": 1}, + "bars": {"type": "array", "items": {"$ref": "#/$defs/Bar"}, + "minItems": 1, "maxItems": 3}, + "quote": {"type": "array", "items": {"type": "string"}, "maxItems": 2}, + "attribution": {"type": "string", "default": ""}, + }, + }, +} + + +def shot(template="radar_sweep", duration=6.0, **props): + base = {"radar_sweep": {"headline": "3 RADARS"}, + "scale_bars": {"headline": "ESCALA", + "bars": [{"label": "BOEING 747", "value": 232}]}}[template] + return {"template": template, "duration": duration, "props": {**base, **props}} + + +def spec_with(*shots, **meta): + return { + "version": 1, + "meta": {"id": "caso", "title": "Un caso", **meta}, + "shots": list(shots) or [shot()], + } + + +def errors_of(spec, templates=None): + with pytest.raises(SpecInvalid) as exc: + validate_spec(spec, templates if templates is not None else TEMPLATES) + return exc.value.errors + + +# --- lo que pasa ------------------------------------------------------------ + +def test_a_minimal_valid_spec_passes(): + validate_spec(spec_with(shot(duration=25.0)), TEMPLATES) + + +def test_the_reference_example_passes_against_its_own_templates(): + """El ejemplo de referencia es válido; se comprueba con esquemas laxos para + las plantillas que este fichero no copia (lo estricto lo cubre el test vivo).""" + spec = json.loads(EXAMPLE.read_text()) + permissive = {name: {"type": "object"} for name in + {s["template"] for s in spec["shots"]}} + permissive.update(TEMPLATES) + validate_spec(spec, permissive) + + +# --- rutas de error --------------------------------------------------------- + +def test_unknown_prop_is_reported_with_its_full_path(): + """El typo en un nombre de prop es el bug más probable de un spec escrito + por un LLM, y la ruta exacta es lo que se le devuelve para arreglarlo.""" + errors = errors_of(spec_with(shot(sweeeps=2))) + assert any(e.startswith("shots.0.radar_sweep.props.sweeeps: campo no permitido") + for e in errors), errors + assert "sweeps" in errors[0], "hay que decirle cuáles SÍ valen" + + +def test_unknown_template_lists_the_valid_names(): + errors = errors_of(spec_with({"template": "radar_swep", "duration": 6.0, + "props": {"headline": "X"}})) + assert errors[0].startswith("shots.0.template:") + assert "radar_sweep" in errors[0] and "scale_bars" in errors[0] + + +def test_missing_required_prop(): + bad = spec_with(shot()); del bad["shots"][0]["props"]["headline"] + assert "shots.0.radar_sweep.props.headline: falta y es obligatorio" in errors_of(bad) + + +def test_empty_string_where_a_non_empty_one_is_required(): + assert any("shots.0.radar_sweep.props.headline" in e + for e in errors_of(spec_with(shot(headline="")))) + + +def test_numeric_bounds(): + errors = errors_of(spec_with(shot(contact_bearing_deg=400))) + assert "shots.0.radar_sweep.props.contact_bearing_deg: 400 debe ser < 360" in errors + + +def test_list_length_limits_are_enforced(): + bars = [{"label": f"B{i}", "value": i + 1} for i in range(4)] + errors = errors_of(spec_with(shot("scale_bars", bars=bars))) + assert "shots.0.scale_bars.props.bars: 4 elementos, el máximo es 3" in errors + + +def test_colour_must_be_a_palette_name_never_hex(): + errors = errors_of(spec_with(shot("scale_bars", bars=[ + {"label": "OBJETO", "value": 2000, "color": "#ffbf00"}]))) + assert any("color" in e and "amber" in e for e in errors) + + +def test_nested_paths_survive_lists(): + errors = errors_of(spec_with(shot("scale_bars", bars=[ + {"label": "BOEING 747", "value": 232}, + {"label": "OBJETO", "value": -5}]))) + assert "shots.0.scale_bars.props.bars.1.value: -5 debe ser > 0" in errors + + +def test_every_error_comes_back_at_once(): + """Se devuelven todos: arreglar cinco de una vez sale más barato que cinco vueltas.""" + errors = errors_of(spec_with(shot(headline="", sweeeps=1, contact_bearing_deg=999))) + assert len(errors) >= 3 + + +# --- el sobre --------------------------------------------------------------- + +def test_meta_id_pattern(): + assert any(e.startswith("meta.id:") for e in errors_of(spec_with(id="Caso Roswell"))) + + +def test_resolution_must_be_a_shorts_one(): + assert any("no es una resolución admitida" in e + for e in errors_of(spec_with(shot(), width=800, height=600))) + + +def test_total_duration_ceiling_is_the_contract_not_the_target(): + """45 s es el objetivo editorial; 180 s es el límite duro. Pasarse de 45 no + invalida el spec — eso es una nota, no un error.""" + long_spec = spec_with(*[shot(duration=10.0) for _ in range(6)]) # 60 s + validate_spec(long_spec, TEMPLATES) + assert editorial_notes(long_spec) + + too_long = spec_with(*[shot(duration=30.0) for _ in range(7)]) # 210 s + assert any("pasa del límite" in e for e in errors_of(too_long)) + + +def test_total_duration_floor(): + assert any("no llega al mínimo" in e + for e in errors_of(spec_with(shot(duration=2.0)))) + + +def test_silence_window_cannot_run_past_the_end(): + bad = spec_with(shot(duration=25.0)) + bad["audio"] = {"preset": "sonar", "silence": [[20.0, 40.0]]} + assert any("se sale de la duración total" in e for e in errors_of(bad)) + + +def test_extra_root_key_is_rejected(): + bad = spec_with(shot(duration=25.0)); bad["narrative_shape"] = "case_file" + assert any(e.startswith("narrative_shape:") for e in errors_of(bad)) + + +def test_editorial_notes_flag_both_ends(): + assert "queda corto" in editorial_notes(spec_with(shot(duration=8.0)))[0] + assert "recorta" in editorial_notes( + spec_with(*[shot(duration=10.0) for _ in range(6)]))[0] + assert editorial_notes(spec_with(shot(duration=30.0))) == [] + + +def test_a_spec_that_is_not_even_a_dict(): + with pytest.raises(SpecInvalid): + validate_spec([1, 2, 3], TEMPLATES) + + +# --- descripción para el prompt --------------------------------------------- + +def test_describe_templates_is_driven_by_what_the_service_publishes(): + text = describe_templates(TEMPLATES) + assert "radar_sweep:" in text and "scale_bars:" in text + assert "headline: string, no vacío, OBLIGATORIO" in text + assert "1-3 elementos" in text # los límites llegan al prompt + assert "ink, amber, amber_dark, muted, dim, red" in text + assert "label: string, no vacío, OBLIGATORIO" in text # despliega los objetos anidados + + +def test_a_template_nobody_wrote_here_still_gets_described(): + """La prueba de que el contrato no está copiado: una plantilla inventada, + que este repo no conoce, se describe igual.""" + text = describe_templates({**TEMPLATES, "holo_scan": { + "type": "object", "required": ["title"], + "properties": {"title": {"type": "string", "minLength": 1}, + "depth_m": {"type": "number", "maximum": 999}}}}) + assert "holo_scan:" in text + assert "depth_m: number, ≤ 999" in text + + +def test_validation_accepts_a_template_nobody_wrote_here(): + templates = {**TEMPLATES, "holo_scan": { + "type": "object", "additionalProperties": False, "required": ["title"], + "properties": {"title": {"type": "string", "minLength": 1}}}} + validate_spec(spec_with({"template": "holo_scan", "duration": 30.0, + "props": {"title": "X"}}), templates)