diff --git a/src/processor/processor.py b/src/processor/processor.py index 98ab1e6..c7deef8 100644 --- a/src/processor/processor.py +++ b/src/processor/processor.py @@ -70,9 +70,28 @@ class OllamaClient: def simple_chunk(text: str, chunk_size: int = 800, overlap: int = 100) -> list[str]: """ Split text into overlapping chunks by approximate word count. - Respects paragraph boundaries when possible. + Respects paragraph/line boundaries when possible. + + Acepta párrafos separados por uno o más saltos de línea (Wikipedia y + trafilatura usan '\n' simple, lo que antes dejaba el documento entero como + un único 'párrafo' → un solo chunk gigante). Además subdivide por palabras + cualquier párrafo que por sí solo supere chunk_size. """ - paragraphs = [p.strip() for p in text.split("\n\n") if p.strip()] + raw_paragraphs = [p.strip() for p in re.split(r"\n+", text) if p.strip()] + + # Subdivide párrafos sobredimensionados en piezas de (chunk_size - overlap) + # palabras; así, al reinyectar 'overlap' palabras de solapamiento, ningún + # chunk resultante supera chunk_size. + piece_size = max(1, chunk_size - max(0, overlap)) + paragraphs: list[str] = [] + for para in raw_paragraphs: + words = para.split() + if len(words) <= chunk_size: + paragraphs.append(para) + else: + for i in range(0, len(words), piece_size): + paragraphs.append(" ".join(words[i:i + piece_size])) + chunks = [] current = [] current_words = 0 @@ -81,10 +100,12 @@ def simple_chunk(text: str, chunk_size: int = 800, overlap: int = 100) -> list[s para_words = len(para.split()) if current_words + para_words > chunk_size and current: chunks.append("\n\n".join(current)) - # overlap: keep last paragraph - if overlap > 0 and current: - current = [current[-1]] - current_words = len(current[0].split()) + # overlap: arrastra un tail de 'overlap' palabras (no el párrafo + # completo — eso duplicaba el tamaño cuando los párrafos eran grandes) + if overlap > 0: + tail = "\n\n".join(current).split()[-overlap:] + current = [" ".join(tail)] + current_words = len(tail) else: current = [] current_words = 0 @@ -241,7 +262,10 @@ class ContentProcessor: threshold=settings.quality_threshold, words=words) continue - embedding = await self.ollama.embed(chunk[:1000]) + # Embeber el chunk completo (ya acotado a ~chunk_size palabras). + # Antes truncaba a 1000 chars → el vector solo representaba el + # principio de cada chunk, degradando el ranking del RAG. + embedding = await self.ollama.embed(chunk) await self.db.add_chunk( session_id=session_id,