From 972bd2f883ebe3ebc8291a894b238326e74440aa Mon Sep 17 00:00:00 2001 From: ChemaVX Date: Mon, 15 Jun 2026 14:53:00 +0000 Subject: [PATCH] =?UTF-8?q?fix(rag):=20chunking=20real=20por=20l=C3=ADneas?= =?UTF-8?q?=20+=20embedding=20de=20chunk=20completo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit simple_chunk: - parte por \n+ (no solo \n\n): Wikipedia/trafilatura usan \n simple, lo que colapsaba cada fuente en un único chunk gigante - subdivide párrafos que superan chunk_size - el overlap arrastra un tail de N palabras en vez del párrafo completo (evita chunks inflados a ~2x cuando los párrafos son grandes) processor: embedding sobre el chunk completo (antes truncaba a 1000 chars, el vector solo representaba el principio del chunk → ranking RAG pobre) Co-Authored-By: Claude Opus 4.8 --- src/processor/processor.py | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/src/processor/processor.py b/src/processor/processor.py index 98ab1e6..c7deef8 100644 --- a/src/processor/processor.py +++ b/src/processor/processor.py @@ -70,9 +70,28 @@ class OllamaClient: def simple_chunk(text: str, chunk_size: int = 800, overlap: int = 100) -> list[str]: """ Split text into overlapping chunks by approximate word count. - Respects paragraph boundaries when possible. + Respects paragraph/line boundaries when possible. + + Acepta párrafos separados por uno o más saltos de línea (Wikipedia y + trafilatura usan '\n' simple, lo que antes dejaba el documento entero como + un único 'párrafo' → un solo chunk gigante). Además subdivide por palabras + cualquier párrafo que por sí solo supere chunk_size. """ - paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()] + raw_paragraphs = [p.strip() for p in re.split(r"\n+", text) if p.strip()] + + # Subdivide párrafos sobredimensionados en piezas de (chunk_size - overlap) + # palabras; así, al reinyectar 'overlap' palabras de solapamiento, ningún + # chunk resultante supera chunk_size. + piece_size = max(1, chunk_size - max(0, overlap)) + paragraphs: list[str] = [] + for para in raw_paragraphs: + words = para.split() + if len(words) <= chunk_size: + paragraphs.append(para) + else: + for i in range(0, len(words), piece_size): + paragraphs.append(" ".join(words[i:i + piece_size])) + chunks = [] current = [] current_words = 0 @@ -81,10 +100,12 @@ def simple_chunk(text: str, chunk_size: int = 800, overlap: int = 100) -> list[s para_words = len(para.split()) if current_words + para_words > chunk_size and current: chunks.append("\n\n".join(current)) - # overlap: keep last paragraph - if overlap > 0 and current: - current = [current[-1]] - current_words = len(current[0].split()) + # overlap: arrastra un tail de 'overlap' palabras (no el párrafo + # completo — eso duplicaba el tamaño cuando los párrafos eran grandes) + if overlap > 0: + tail = "\n\n".join(current).split()[-overlap:] + current = [" ".join(tail)] + current_words = len(tail) else: current = [] current_words = 0 @@ -241,7 +262,10 @@ class ContentProcessor: threshold=settings.quality_threshold, words=words) continue - embedding = await self.ollama.embed(chunk[:1000]) + # Embeber el chunk completo (ya acotado a ~chunk_size palabras). + # Antes truncaba a 1000 chars → el vector solo representaba el + # principio de cada chunk, degradando el ranking del RAG. + embedding = await self.ollama.embed(chunk) await self.db.add_chunk( session_id=session_id,