PDF & Office to Markdown for RAG — tables, chunks, OCR
Pricing
from $1.00 / 1,000 page converteds
Convert PDFs, scans, images, Word, Excel and PowerPoint into clean Markdown for LLMs and RAG: real tables (Markdown, CSV, cells), correct reading order in multi-column layouts, headers and footers removed, and ready-to-embed chunks with page and section. Automatic OCR only on scanned pages.
# Herramienta 2 (DEC-006). Imagen slim de Debian con el SDK de Apify para Python.FROM apify/actor-python:3.13
# OCR: idiomas del selector de la entrada. Sin otros servicios de red ni scripts de instalación.USER rootRUN apt-get update \ && apt-get install -y --no-install-recommends tesseract-ocr tesseract-ocr-spa tesseract-ocr-por \ tesseract-ocr-fra tesseract-ocr-deu tesseract-ocr-ita libglib2.0-0 \ && rm -rf /var/lib/apt/lists/*USER myuser
COPY requirements-actor.txt ./RUN pip install --no-cache-dir -r requirements-actor.txt && tesseract --list-langs
COPY documentos.py demo_pdf.py main.py ./RUN python -m compileall -q documentos.py demo_pdf.py main.py
CMD ["python3", "main.py"]{ "$schema": "https://apify.com/schemas/v1/actor.ide.json", "actorSpecification": 1, "name": "documents-to-markdown-rag", "title": "PDF & Office to Markdown for RAG — tables, chunks, OCR", "description": "Convert PDFs, scans, images, DOCX, XLSX and PPTX into clean Markdown for LLMs and RAG: tables as real tables (Markdown, CSV, cells), two-column reading order, repeated headers/footers removed, and ready-to-embed chunks with page numbers and section titles. Automatic OCR (Spanish, English, Portuguese and more) only on scanned pages.", "version": "0.1", "buildTag": "latest", "inputSchema": "./input_schema.json", "outputSchema": "./output_schema.json", "storages": { "dataset": "./dataset_schema.json" }, "dockerfile": "./Dockerfile"}{ "actorSpecification": 1, "fields": {}, "views": { "overview": { "title": "Overview", "transformation": { "fields": ["file", "status", "format", "page_count", "stats.n_tables", "stats.n_ocr_pages", "stats.tokens_est", "billing.page", "billing.ocr-page", "markdown_file", "warnings"], "flatten": ["stats", "billing"] }, "display": { "component": "table", "properties": { "file": {"label": "File"}, "status": {"label": "Status"}, "format": {"label": "Format"}, "page_count": {"label": "Pages", "format": "number"}, "stats.n_tables": {"label": "Tables", "format": "number"}, "stats.n_ocr_pages": {"label": "OCR pages", "format": "number"}, "stats.tokens_est": {"label": "Tokens (est.)", "format": "number"}, "billing.page": {"label": "Billed pages", "format": "number"}, "billing.ocr-page": {"label": "Billed OCR pages", "format": "number"}, "markdown_file": {"label": "Markdown file", "format": "link"}, "warnings": {"label": "Warnings", "format": "array"} } } }, "chunks": { "title": "Chunks (chunks mode)", "transformation": { "fields": ["file", "chunk_id", "section", "pages", "tokens_est", "text"] }, "display": { "component": "table", "properties": { "file": {"label": "File"}, "chunk_id": {"label": "#", "format": "number"}, "section": {"label": "Section"}, "pages": {"label": "Pages", "format": "array"}, "tokens_est": {"label": "Tokens", "format": "number"}, "text": {"label": "Text"} } } } }}{ "title": "Documents to Markdown for RAG", "type": "object", "schemaVersion": 1, "properties": { "documents": { "title": "Documents (files or URLs)", "type": "array", "description": "Upload files or paste public URLs. Formats: PDF (text or scanned), PNG, JPG, WEBP, TIFF, BMP, DOCX, XLSX, PPTX, TXT, MD, CSV. Up to 50 MB per file. If you upload files here, also select the storage they went to below. Leave empty to see a demo.", "editor": "fileupload" }, "storage": { "title": "Storage with uploaded files", "type": "string", "resourceType": "keyValueStore", "resourcePermissions": ["READ"], "description": "The key-value store where your uploaded files are. This Actor runs with limited permissions and can only read the storage you pick here. If you leave the list above empty, every supported document in this storage is converted." }, "output_mode": { "title": "Output: one item per", "type": "string", "description": "document: one item with Markdown, tables and chunks. chunks: one item per chunk, ready for a vector database. pages: one item per page.", "editor": "select", "enum": ["document", "chunks", "pages"], "enumTitles": ["Document", "Chunk (for vector databases)", "Page"], "default": "document" }, "chunk_tokens": { "title": "Chunk size (tokens, approx.)", "type": "integer", "description": "Target chunk size, estimated as 4 characters per token. Chunks follow section boundaries and never split a table without repeating its header row. 0 = no chunks.", "minimum": 0, "maximum": 8000, "default": 800 }, "chunk_overlap": { "title": "Chunk overlap (tokens, approx.)", "type": "integer", "description": "Text repeated at the start of the next chunk when a section is split by size.", "minimum": 0, "maximum": 1000, "default": 80 }, "ocr": { "title": "OCR", "type": "string", "description": "auto: OCR only pages without a usable text layer (and images). always: OCR every page (for PDFs with broken text). never: faster and cheaper, scanned pages come out empty.", "editor": "select", "enum": ["auto", "always", "never"], "enumTitles": ["Automatic", "Always", "Never"], "default": "auto" }, "ocr_languages": { "title": "OCR languages", "type": "string", "description": "Languages in the scanned documents. The first one has priority.", "editor": "select", "enum": ["spa+eng", "eng", "eng+spa+por", "por+eng", "fra+eng", "deu+eng", "ita+eng"], "enumTitles": ["Spanish + English", "English", "English + Spanish + Portuguese", "Portuguese + English", "French + English", "German + English", "Italian + English"], "default": "spa+eng" }, "pages": { "title": "Page range", "type": "string", "description": "Pages or slides to convert, e.g. 1-5, 8, 12-. Empty = all.", "editor": "textfield" }, "max_pages": { "title": "Max pages per document", "type": "integer", "description": "Safety limit per document (cost control).", "minimum": 1, "maximum": 2000, "default": 200 }, "include_pages": { "title": "Include per-page Markdown (document mode)", "type": "boolean", "description": "Adds a 'pages' array with the Markdown, method (text/ocr) and OCR confidence of each page.", "default": false }, "save_markdown_files": { "title": "Save a .md file per document", "type": "boolean", "description": "Also saves each document's Markdown as a file in the run's key-value store, with a download link in the result.", "default": true } }, "required": []}{ "$schema": "https://apify.com/schemas/v1/output.ide.json", "actorOutputSchemaVersion": 1, "title": "Results", "properties": { "results": { "type": "string", "title": "Converted documents", "template": "{{links.apiDefaultDatasetUrl}}/items" }, "markdown_files": { "type": "string", "title": "Markdown files", "template": "{{links.apiDefaultKeyValueStoreUrl}}/keys" } }}tests/__pycache__/storage/1"""PDF de demostración (ficticio, en inglés). Generado por tests/fixtures.py::pdf_demo; no editar a mano."""2import base643
4DEMO_B64 = (5 "JVBERi0xLjQKJZOMi54gUmVwb3J0TGFiIEdlbmVyYXRlZCBQREYgZG9jdW1lbnQgKG9wZW5zb3VyY2UpCjEgMCBvYmoKPDwKL0Yx"6 "IDIgMCBSIC9GMiAzIDAgUgo+PgplbmRvYmoKMiAwIG9iago8PAovQmFzZUZvbnQgL0hlbHZldGljYSAvRW5jb2RpbmcgL1dpbkFu"7 "c2lFbmNvZGluZyAvTmFtZSAvRjEgL1N1YnR5cGUgL1R5cGUxIC9UeXBlIC9Gb250Cj4+CmVuZG9iagozIDAgb2JqCjw8Ci9CYXNl"8 "Rm9udCAvSGVsdmV0aWNhLUJvbGQgL0VuY29kaW5nIC9XaW5BbnNpRW5jb2RpbmcgL05hbWUgL0YyIC9TdWJ0eXBlIC9UeXBlMSAv"9 "VHlwZSAvRm9udAo+PgplbmRvYmoKNCAwIG9iago8PAovQ29udGVudHMgOSAwIFIgL01lZGlhQm94IFsgMCAwIDU5NS4yNzU2IDg0"10 "MS44ODk4IF0gL1BhcmVudCA4IDAgUiAvUmVzb3VyY2VzIDw8Ci9Gb250IDEgMCBSIC9Qcm9jU2V0IFsgL1BERiAvVGV4dCAvSW1h"11 "Z2VCIC9JbWFnZUMgL0ltYWdlSSBdCj4+IC9Sb3RhdGUgMCAvVHJhbnMgPDwKCj4+IAogIC9UeXBlIC9QYWdlCj4+CmVuZG9iago1"12 "IDAgb2JqCjw8Ci9Db250ZW50cyAxMCAwIFIgL01lZGlhQm94IFsgMCAwIDU5NS4yNzU2IDg0MS44ODk4IF0gL1BhcmVudCA4IDAg"13 "UiAvUmVzb3VyY2VzIDw8Ci9Gb250IDEgMCBSIC9Qcm9jU2V0IFsgL1BERiAvVGV4dCAvSW1hZ2VCIC9JbWFnZUMgL0ltYWdlSSBd"14 "Cj4+IC9Sb3RhdGUgMCAvVHJhbnMgPDwKCj4+IAogIC9UeXBlIC9QYWdlCj4+CmVuZG9iago2IDAgb2JqCjw8Ci9QYWdlTW9kZSAv"15 "VXNlTm9uZSAvUGFnZXMgOCAwIFIgL1R5cGUgL0NhdGFsb2cKPj4KZW5kb2JqCjcgMCBvYmoKPDwKL0F1dGhvciAoTElGIHBydWVi"16 "YXMpIC9DcmVhdGlvbkRhdGUgKEQ6MjAyNjExMDIyMTI0MzUrMDAnMDAnKSAvQ3JlYXRvciAoXCh1bnNwZWNpZmllZFwpKSAvS2V5"17 "d29yZHMgKCkgL01vZERhdGUgKEQ6MjAyNjExMDIyMTI0MzUrMDAnMDAnKSAvUHJvZHVjZXIgKFJlcG9ydExhYiBQREYgTGlicmFy"18 "eSAtIFwob3BlbnNvdXJjZVwpKSAKICAvU3ViamVjdCAoXCh1bnNwZWNpZmllZFwpKSAvVGl0bGUgKERyaXAgaXJyaWdhdGlvbiBy"19 "ZXBvcnQgXChzYW1wbGVcKSkgL1RyYXBwZWQgL0ZhbHNlCj4+CmVuZG9iago4IDAgb2JqCjw8Ci9Db3VudCAyIC9LaWRzIFsgNCAw"20 "IFIgNSAwIFIgXSAvVHlwZSAvUGFnZXMKPj4KZW5kb2JqCjkgMCBvYmoKPDwKL0ZpbHRlciBbIC9BU0NJSTg1RGVjb2RlIC9GbGF0"21 "ZURlY29kZSBdIC9MZW5ndGggMTA3Ngo+PgpzdHJlYW0KR2IiLydnUSgjSCY6Ti8zUzRATSU8Q2kiWVw4UCpRPkdQcWBDZldpIV5i"22 "OjtYW0AnWF1RX150NGhacF9wUC1ePj1AJC87a29qU2M1XFhdKSpSMVZrUjVYPVltWmlKdCRHXlc0Tik8KGx0bVctNnFpJE5WTkd1"23 "VFdlRFU7JGpsW1RYJksiL1ZVNHQzcU5KLy49R1BcSDpKTWxrRENYMlRdYlhnRCFuJSdtZXVYIjE3ZyRhRUZXIXFaaWktY0UzW104"24 "XkxeN2FXdWtTX3NBIkdebGREPHA3S0VjWEw7V2tdQ3RLX1NCc1JkX145TzhHdGZXTFgkM05qbi8zdCIwRkBoWEBiJSY+XnNqPyZC"25 "I0clXk06PHU0MEBVKll1NmpDWzdwLmBuUTAxUmBeXzg4UFNzSWQsMSNzSklRREZlJzBDRWBoJyctSiNlcEhbWUlUSD1CVCUpMmVM"26 "Sz1NJ3BNI0RjUVs+c0ZSa0FbOWFoQSRsNTRbP01nLUouPEVbTC9DXCxNSTsqXiVVUU84QUMzcj5IW088Jzo2KDJLbmwsWStsYmYh"27 "cjVsTT83IzRiPFJGJCpnREU4QkhgSjtkWU4hKFVpMFswQFQ3KDIsamhJLFlaZy9Aay9zLihgM1JWKjxnOSIjL1c+a2deb2ZMZCld"28 "LlxzbU9UPVQzJDM/VzNQXl45XDMjbW1uYCgmNUtAMnJlcWFvUzZvazwvb2diXEJ1IlNeOnU3RDxEKCVnVidjX0pXPXJiZFNEYVNh"29 "V1BjMkJLLVZAay9JUCNRQik9IUMtJzszRkdAdGxoZTg4bDJkbiFmXi1hU01bXGBmUVk6cTo9Sj0hZVg5UGBjI0BxdHNATipTWF43"30 "cjg6ZGIoKitIZ1clQCRpTDYwUFw0T0hhUVxSWyptPy1NPzxSOmVnbGxVODs5b11hMFJDUFNqVGY0O1xYaTA0L3FoUjhsMklKaG1h"31 "JkYuSikhJ1kjUFhZaCJlVz1IVl1nOj1GXDlEM1BfVSgiITwwOTFPT1xYdV5QaHMsQV9wYXFcYzhjSWBKT2lVbD4jK0tDNTVqKV4q"32 "LXNucGgpYWQqLlpMQW9wOk5JRSJmdWRVPVk/Y2wpOl9NR0twJ21YVDZSXF9RZFYwPiw+ZVAsWnEsLT8kamM6QCIsNktbUSwha2xl"33 "QTlROWBfR1VcZHMkXktCYkdvbVNKOkdIMm1CJ2o4KyJAOGU/WGEiZk5NQytbVW8ncFUtQ1w7SzxtKl1pT2xUYl5uKWVaLVBNOihc"34 "RC9pa2VvPUNgQUQ+S1NGPjhtczgnTmEyYzBuPSRoJ3IyS0RRPUFZdWdHP3BgJ3E7ZEhRJzBiPW40O2QhNmE/Z2QjRS9NZVFgWVhW"35 "WEpYTkYjO1oycSoobVpuPyxwX3IoXipuc1MnZVVYYzk/Qms/PTMwKHVbSyhKXGI4NElpVGVCQWwmfj5lbmRzdHJlYW0KZW5kb2Jq"36 "CjEwIDAgb2JqCjw8Ci9GaWx0ZXIgWyAvQVNDSUk4NURlY29kZSAvRmxhdGVEZWNvZGUgXSAvTGVuZ3RoIDQ5OQo+PgpzdHJlYW0K"37 "R2IiL2A0YEIoLyY7R0UtTUthLydXMi0uWW5zVWJvXGNnUklLcVFgXiJWUmRTbCkrRlJLcD5WW2NxY24xLk1RJkcqOl5iUUhRRVYi"38 "IlswVC4jQiI9aWguYyddb2A3KkxGNEJZdT0hb1ptbDkqcSNgT0RncS5FRTVaS3Q+PWVLQlwvTyZCUHQ2PkxQbE9uLk4iYCo4IWBA"39 "bHFbWUFNLlY2a0k/cC04OU08STA6ZDpWJW82a0pQTW1UTG1lUUA6cGwrLWJgcEItOC9yL2lVYE8zKlY6OyEta1xjTSNAaVU7ZF5Y"40 "MD9ZKjVhNEJjaTItdWZEMEFFXkhCSFlvZ01bNW45IkVNPU84NVVIOE5VUW9gaVJVRWolY01KYmU6MWI2LExpNTciWyYyREpsPGVy"41 "SzJCYkpSMTw5cSY1cWteXGN1JGFGOSxrK1BXaj1YNDoyXCVHQ2YsRkhtL1dbaSpGLDNhNWN0P01YTjZANHBEa1U5XTtXM0lEUCwy"42 "XFVWbCdcRzg6YzFCL01xKlA3JjNNcklmR24tWzdEbFE6X3AiQ3QuKDllY1VCZCRuTFFjdUtsNCtKRidAPkExRE8/JypHIUM6O0xW"43 "Lldkc0NaZ0RTJj4tMThcQTVwZjRXYFByJzE5MSJwVVkwRFNoXG5yUyJmTjNHKld+PmVuZHN0cmVhbQplbmRvYmoKeHJlZgowIDEx"44 "CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDA2MSAwMDAwMCBuIAowMDAwMDAwMTAyIDAwMDAwIG4gCjAwMDAwMDAyMDkgMDAw"45 "MDAgbiAKMDAwMDAwMDMyMSAwMDAwMCBuIAowMDAwMDAwNTI0IDAwMDAwIG4gCjAwMDAwMDA3MjggMDAwMDAgbiAKMDAwMDAwMDc5"46 "NiAwMDAwMCBuIAowMDAwMDAxMDk0IDAwMDAwIG4gCjAwMDAwMDExNTkgMDAwMDAgbiAKMDAwMDAwMjMyNiAwMDAwMCBuIAp0cmFp"47 "bGVyCjw8Ci9JRCAKWzw0NTFlZGFkMGIzY2JhOWQ3MTM5NDk5NDlhOWJjMTAyMz48NDUxZWRhZDBiM2NiYTlkNzEzOTQ5OTQ5YTli"48 "YzEwMjM+XQolIFJlcG9ydExhYiBnZW5lcmF0ZWQgUERGIGRvY3VtZW50IC0tIGRpZ2VzdCAob3BlbnNvdXJjZSkKCi9JbmZvIDcg"49 "MCBSCi9Sb290IDYgMCBSCi9TaXplIDExCj4+CnN0YXJ0eHJlZgoyOTE2CiUlRU9GCg=="50)51
52
53def demo_bytes() -> bytes:54 return base64.b64decode(DEMO_B64)1"""Herramienta 2 (DEC-006): documentos -> Markdown, tablas y chunks para RAG.2
3Formatos: PDF (capa de texto y escaneados), imágenes (OCR), DOCX, XLSX, PPTX, TXT/MD/CSV.4La interfaz pública (claves del JSON) está en inglés porque el mercado de RAG es global.5
6Dependencias, todas con licencias permisivas: pdfplumber/pdfminer.six (MIT), pypdfium2 (Apache/BSD),7opencv-python-headless (Apache 2.0), Tesseract (Apache 2.0), python-docx, openpyxl, python-pptx (MIT).8Nada AGPL. Sin red: todo pasa en memoria o en temporales que se borran.9"""10from __future__ import annotations11
12import csv13import io14import math15import os16import re17import shutil18import statistics19import subprocess20import tempfile21import time22from collections import Counter23
24VERSION = "0.1.0"25CHARS_PAGINA_EQ = 3000 # "página equivalente" para formatos sin páginas (DOCX, XLSX, TXT)26MIN_TEXTO = 40 # menos caracteres útiles en una página de PDF => se trata como escaneada27MAX_FILAS_HOJA = 5000 # tope de filas por hoja de cálculo28FORMATOS = {".pdf": "pdf", ".png": "image", ".jpg": "image", ".jpeg": "image", ".webp": "image",29 ".tif": "image", ".tiff": "image", ".bmp": "image", ".docx": "docx", ".xlsx": "xlsx",30 ".xlsm": "xlsx", ".pptx": "pptx", ".txt": "text", ".md": "text", ".markdown": "text", ".csv": "csv"}31VIÑETA = re.compile(r"^\s*([•●▪◦·‣∙\-–—\*]|\(cid:\d+\)|\(?\d{1,2}[.)]|\(?[a-zA-Z][.)])\s+")32SIMBOLOS_VIÑETA = "•●▪◦·‣∙-–—*"33
34
35def _es_simbolo(marca: str) -> bool:36 """Viñeta gráfica (se reemplaza por '- ') y no numeración (que se conserva)."""37 return marca in SIMBOLOS_VIÑETA or marca.startswith("(cid:")38
39
40class FormatoNoSoportado(ValueError):41 pass42
43
44# --------------------------------------------------------------------------------------------45# Bloques: la representación intermedia común a todos los formatos46# --------------------------------------------------------------------------------------------47def _b(kind: str, page, text: str = "", level: int = 0, rows=None) -> dict:48 return {"kind": kind, "page": page, "text": text, "level": level, "rows": rows}49
50
51def _celda(v) -> str:52 if v is None:53 return ""54 if hasattr(v, "isoformat"):55 return v.isoformat()56 if isinstance(v, float):57 return repr(int(v)) if v.is_integer() else repr(v)58 return re.sub(r"\s+", " ", str(v)).strip()59
60
61def _limpiar_filas(rows) -> list[list[str]]:62 filas = [[_celda(c) for c in r] for r in rows if r is not None]63 filas = [r for r in filas if any(r)]64 if not filas:65 return []66 ancho = max(len(r) for r in filas)67 filas = [r + [""] * (ancho - len(r)) for r in filas]68 vacias = [j for j in range(ancho) if not any(r[j] for r in filas)]69 return [[c for j, c in enumerate(r) if j not in vacias] for r in filas]70
71
72def tabla_markdown(rows: list[list[str]]) -> str:73 if not rows:74 return ""75 esc = [[c.replace("|", "\\|") for c in r] for r in rows]76 cab, cuerpo = esc[0], esc[1:]77 lineas = ["| " + " | ".join(cab) + " |", "| " + " | ".join("---" for _ in cab) + " |"]78 lineas += ["| " + " | ".join(r) + " |" for r in cuerpo]79 return "\n".join(lineas)80
81
82def tabla_csv(rows: list[list[str]]) -> str:83 buf = io.StringIO()84 csv.writer(buf, lineterminator="\n").writerows(rows)85 return buf.getvalue()86
87
88def _bloque_md(b: dict) -> str:89 if b["kind"] == "heading":90 return "#" * max(1, min(b["level"], 6)) + " " + b["text"]91 if b["kind"] == "list":92 return "- " + b["text"]93 if b["kind"] == "table":94 return tabla_markdown(b["rows"])95 if b["kind"] == "note":96 return "> " + b["text"]97 return b["text"]98
99
100def _a_markdown(bloques: list[dict]) -> str:101 partes, previo = [], None102 for b in bloques:103 md = _bloque_md(b)104 if not md.strip():105 continue106 sep = "\n" if (previo == "list" and b["kind"] == "list") else "\n\n"107 partes.append((sep if partes else "") + md)108 previo = b["kind"]109 return "".join(partes).strip() + ("\n" if partes else "")110
111
112def _unir_linea(previo: str, nueva: str) -> str:113 """Une líneas de un párrafo y deshace cortes con guion ('infor-' + 'mación')."""114 if previo.endswith("-") and nueva[:1].islower() and not previo.endswith(" -"):115 return previo[:-1] + nueva116 return previo + " " + nueva117
118
119# --------------------------------------------------------------------------------------------120# PDF con capa de texto121# --------------------------------------------------------------------------------------------122def _desapilar(rows) -> list[list]:123 """Tablas con líneas cada N filas: pdfplumber junta N filas en una celda ('Alabama\nAlaska\n...').124
125 Si todas las celdas no vacías de la fila tienen la misma cantidad k ≥ 2 de renglones, se parte en k filas.126 """127 out = []128 for r in rows:129 if r is None:130 continue131 partes = [(c or "").split("\n") if c else [] for c in r]132 ks = {len(p) for p in partes if p}133 if len(ks) == 1 and (k := ks.pop()) >= 2 and sum(1 for p in partes if p) >= 2:134 out += [[p[i] if p else "" for p in partes] for i in range(k)]135 else:136 out.append(r)137 return out138
139
140def _es_negrita(fontname: str) -> bool:141 f = (fontname or "").lower()142 return any(k in f for k in ("bold", "black", "heavy", "semibold")) or f.endswith(",b")143
144
145def _filas(words: list[dict]) -> list[list[dict]]:146 filas, actual, top = [], [], None147 for w in sorted(words, key=lambda w: (round(w["top"], 1), w["x0"])):148 tol = max(2.0, 0.5 * float(w.get("size") or 10))149 if actual and abs(w["top"] - top) <= tol:150 actual.append(w)151 else:152 if actual:153 filas.append(sorted(actual, key=lambda w: w["x0"]))154 actual, top = [w], w["top"]155 if actual:156 filas.append(sorted(actual, key=lambda w: w["x0"]))157 return filas158
159
160def _medianeras(words: list[dict], ancho: float) -> list[float]:161 """Busca los espacios verticales entre columnas de texto corrido (2 o 3 columnas). Devuelve sus x.162
163 Exige que cada columna tenga renglones de prosa (≥ 4 palabras por fila en promedio), para no164 confundir formularios 'etiqueta ... valor' con texto en columnas.165 """166 filas = _filas(words)167 if len(words) < 60 or len(filas) < 8:168 return []169 res = 2.0170 nb = int(ancho / res) + 2171 cobertura = [0] * nb172 for fila in filas:173 marcados = set()174 for w in fila:175 marcados.update(range(max(0, int(w["x0"] / res)), min(nb, int(w["x1"] / res) + 1)))176 for i in marcados:177 cobertura[i] += 1178 umbral = max(1, int(0.1 * len(filas)))179 x_min = min(w["x0"] for w in words)180 x_max = max(w["x1"] for w in words)181 lo, hi = int((x_min + 0.15 * (x_max - x_min)) / res), int((x_min + 0.85 * (x_max - x_min)) / res)182 candidatas, i = [], lo183 while i < hi:184 if cobertura[i] <= umbral:185 k, j = i, i186 while k > 0 and cobertura[k - 1] <= umbral:187 k -= 1188 while j < nb and cobertura[j] <= umbral:189 j += 1190 if (j - k) * res >= 8:191 candidatas.append(((k + j) * res / 2, (j - k) * res))192 i = j193 else:194 i += 1195 if not candidatas:196 return []197
198 def valida(xs: list[float]) -> bool:199 cortes = [-1e9] + xs + [1e9]200 for a, b in zip(cortes, cortes[1:]):201 col = [w for w in words if w["x0"] >= a and w["x1"] <= b]202 if len(col) < 0.15 * len(words) or len(col) / max(1, len(_filas(col))) < 4:203 return False204 return True205
206 todas = sorted(x for x, _ in candidatas)207 if len(todas) <= 3 and valida(todas):208 return todas209 for x, _ in sorted(candidatas, key=lambda c: -c[1]): # si no, la medianera más ancha sola210 if valida([x]):211 return [x]212 return []213
214
215def _columna(x0: float, x1: float, cortes: list[float]):216 """Índice de columna de un objeto, o 'S' si cruza alguna medianera."""217 for c in cortes:218 if x0 < c < x1:219 return "S"220 return sum(1 for c in cortes if x0 >= c)221
222
223def _linea(words: list[dict], lado: str) -> dict:224 tam = statistics.median(float(w.get("size") or 10) for w in words)225 return {"text": " ".join(w["text"] for w in words), "top": min(w["top"] for w in words),226 "bottom": max(w["bottom"] for w in words), "x0": words[0]["x0"], "x1": words[-1]["x1"],227 "size": tam, "bold": all(_es_negrita(w.get("fontname", "")) for w in words), "side": lado,228 "kind": "line"}229
230
231def _pdf_items_pagina(page) -> tuple[list[dict], int]:232 """Líneas y tablas de una página, en orden de lectura (respeta dos columnas). Devuelve (items, n_tablas)."""233 tablas = []234 try:235 encontradas = page.find_tables()236 except Exception: # noqa: BLE001 — pdfplumber puede fallar en PDFs raros; se sigue sin tablas237 encontradas = []238 for t in encontradas:239 filas = _limpiar_filas(_desapilar(t.extract()))240 titulo = None241 if len(filas) >= 3 and len(filas[0]) >= 3 and sum(1 for c in filas[0] if c) == 1:242 titulo = next(c for c in filas[0] if c) # primera fila con una sola celda: es el título de la tabla243 filas = _limpiar_filas(filas[1:])244 if len(filas) >= 2 and len(filas[0]) >= 2:245 tablas.append({"kind": "table", "rows": filas, "top": t.bbox[1], "bottom": t.bbox[3],246 "x0": t.bbox[0], "x1": t.bbox[2], "bbox": t.bbox, "caption": titulo})247
248 def fuera(obj) -> bool:249 if obj.get("object_type") != "char":250 return True251 cx, cy = (obj["x0"] + obj["x1"]) / 2, (obj["top"] + obj["bottom"]) / 2252 return not any(b[0] <= cx <= b[2] and b[1] <= cy <= b[3] for b in (t["bbox"] for t in tablas))253
254 p = page.filter(fuera) if tablas else page255 words = [w for w in p.extract_words(extra_attrs=["size", "fontname"], keep_blank_chars=False)256 if w.get("upright", True)] # el texto vertical de los márgenes (sellos, IDs de arXiv) no es contenido257 cortes = _medianeras(words, float(page.width))258
259 lineas = []260 for fila in _filas(words):261 if not cortes:262 lineas.append(_linea(fila, "S"))263 continue264 tam = statistics.median(float(w.get("size") or 10) for w in fila)265 cruza = any(_columna(w["x0"], w["x1"], cortes) == "S" for w in fila)266 partes: dict = {}267 for w in fila:268 partes.setdefault(_columna(w["x0"], w["x1"], cortes), []).append(w)269 claves = sorted(k for k in partes if k != "S")270 for a, b in zip(claves, claves[1:]): # un título a todo el ancho con un espacio justo en la medianera271 if partes[b][0]["x0"] - partes[a][-1]["x1"] < 0.6 * tam:272 cruza = True273 if cruza:274 lineas.append(_linea(fila, "S"))275 else:276 lineas += [_linea(partes[k], k) for k in claves]277 for t in tablas:278 t["side"] = _columna(t["x0"], t["x1"], cortes) if cortes else "S"279
280 items = sorted(lineas + tablas, key=lambda it: (it["top"], it["x0"]))281 if not cortes:282 return items, len(tablas)283 orden, banda = [], {}284 for it in items:285 if it["side"] == "S":286 orden += [x for k in sorted(banda) for x in banda[k]]287 banda = {}288 orden.append(it)289 else:290 banda.setdefault(it["side"], []).append(it)291 return orden + [x for k in sorted(banda) for x in banda[k]], len(tablas)292
293
294def _normalizar_rep(t: str) -> str:295 return re.sub(r"\d+", "#", t.lower()).strip()296
297
298def _quitar_encabezados(paginas: list[dict]) -> int:299 """Saca encabezados y pies que se repiten en ≥ 30% de las páginas (números de página incluidos).300
301 30% y no 50%: en documentos a doble faz el encabezado alterna entre páginas pares e impares.302 """303 con_texto = [p for p in paginas if p["method"] == "text"]304 if len(con_texto) < 2:305 return 0306 cuenta = Counter()307 for p in con_texto:308 h = p["height"]309 vistos = {_normalizar_rep(it["text"]) for it in p["items"]310 if it["kind"] == "line" and (it["top"] < 0.08 * h or it["bottom"] > 0.92 * h)}311 cuenta.update(vistos)312 repetidos = {t for t, n in cuenta.items() if n >= max(2, 0.3 * len(con_texto)) and len(t) <= 160}313 quitadas = 0314 for p in con_texto:315 h = p["height"]316 antes = len(p["items"])317 p["items"] = [it for it in p["items"] if not (318 it["kind"] == "line" and (it["top"] < 0.08 * h or it["bottom"] > 0.92 * h)319 and _normalizar_rep(it["text"]) in repetidos)]320 quitadas += antes - len(p["items"])321 return quitadas322
323
324def _niveles_titulo(paginas: list[dict], cuerpo: float, chars_por_tam: Counter) -> dict:325 """Tamaños de letra que son títulos: ≥ 1,2 × cuerpo, renglones cortos y < 15% del texto total."""326 total = sum(chars_por_tam.values()) or 1327 tams = sorted({round(it["size"] * 2) / 2 for p in paginas for it in p.get("items", [])328 if it["kind"] == "line" and it["size"] >= 1.2 * cuerpo and len(it["text"]) <= 150329 and chars_por_tam[round(it["size"] * 2) / 2] < 0.15 * total},330 reverse=True)331 return {t: min(i + 1, 4) for i, t in enumerate(tams)}332
333
334def _interlineado(items: list[dict]) -> float:335 """Espacio típico entre renglones consecutivos de un mismo párrafo (mediana de la página)."""336 lineas = [it for it in items if it["kind"] == "line"]337 brechas = [b["top"] - a["bottom"] for a, b in zip(lineas, lineas[1:])338 if a["side"] == b["side"] and abs(a["size"] - b["size"]) < 0.5 and 0 <= b["top"] - a["bottom"] < 2 * a["size"]]339 return statistics.median(brechas) if brechas else 2.0340
341
342def _items_a_bloques(items: list[dict], pagina: int, cuerpo: float, niveles: dict) -> list[dict]:343 bloques, ult = [], None # ult: último ítem de línea agregado a un párrafo o lista344 base = _interlineado(items)345 nivel_negrita = min(len(niveles) + 1, 4)346 for it in items:347 if it["kind"] == "table":348 if it.get("caption"):349 bloques.append(_b("para", pagina, f"**{it['caption']}**"))350 bloques.append(_b("table", pagina, rows=it["rows"]))351 ult = None352 continue353 texto = it["text"].strip()354 if not texto:355 continue356 tam = round(it["size"] * 2) / 2357 es_titulo = tam in niveles358 es_negrita = (not es_titulo and it["bold"] and len(texto) <= 80 and it["size"] >= 0.95 * cuerpo359 and re.search(r"[A-Za-zÀ-ÿ]", texto) and not re.search(r"[.:;,]$", texto)360 and len(texto.split()) >= 2)361 if es_titulo or es_negrita:362 nivel = niveles.get(tam, nivel_negrita)363 prev = bloques[-1] if bloques else None364 if (prev and prev["kind"] == "heading" and prev["level"] == nivel and ult is not None365 and ult.get("_titulo") and 0 <= it["top"] - ult["bottom"] <= 0.4 * it["size"]):366 prev["text"] += " " + texto367 else:368 bloques.append(_b("heading", pagina, texto, nivel))369 ult = dict(it, _titulo=True)370 continue371 m = VIÑETA.match(texto)372 sigue = (ult is not None and not ult.get("_titulo") and bloques and bloques[-1]["kind"] in ("para", "list")373 and it["side"] == ult["side"] and -2 <= it["top"] - ult["bottom"] <= base + max(2.0, 0.3 * it["size"])374 and abs(it["size"] - ult["size"]) < 1.5)375 if m:376 bloques.append(_b("list", pagina, texto[m.end():].strip() if _es_simbolo(m.group(1)) else texto))377 elif sigue and not (bloques[-1]["kind"] == "list" and it["x0"] <= ult["x0"] - 5):378 bloques[-1]["text"] = _unir_linea(bloques[-1]["text"], texto)379 else:380 bloques.append(_b("para", pagina, texto))381 ult = it382 return bloques383
384
385def _texto_roto(t: str) -> bool:386 """Capa de texto con fuentes mal codificadas: '(cid:12)' o mayoría de caracteres no imprimibles."""387 if t.count("(cid:") >= 5:388 return True389 raros = sum(1 for c in t if not (c.isprintable() or c.isspace()))390 return len(t) > 0 and raros / len(t) > 0.3391
392
393# --------------------------------------------------------------------------------------------394# OCR (Tesseract): páginas escaneadas e imágenes395# --------------------------------------------------------------------------------------------396_IDIOMAS_CACHE: list[str] | None = None397
398
399def _idiomas_tesseract() -> list[str]:400 global _IDIOMAS_CACHE401 if _IDIOMAS_CACHE is None:402 out = subprocess.run(["tesseract", "--list-langs"], capture_output=True, text=True).stdout403 _IDIOMAS_CACHE = [x.strip() for x in out.splitlines()[1:] if x.strip()]404 return _IDIOMAS_CACHE405
406
407def _tsv(img, idioma: str, psm: int = 3) -> list[dict]:408 import cv2409 with tempfile.TemporaryDirectory() as tmp:410 ruta = os.path.join(tmp, "p.png")411 cv2.imwrite(ruta, img)412 r = subprocess.run(["tesseract", ruta, "stdout", "-l", idioma, "--psm", str(psm), "tsv"],413 capture_output=True, text=True, timeout=300)414 filas = []415 for i, linea in enumerate(r.stdout.splitlines()):416 partes = linea.split("\t")417 if i == 0 or len(partes) < 12 or partes[0] != "5" or not partes[11].strip():418 continue419 filas.append({"block": int(partes[2]), "par": int(partes[3]), "line": int(partes[4]),420 "left": int(partes[6]), "top": int(partes[7]), "height": int(partes[9]),421 "conf": float(partes[10]), "text": partes[11].strip()})422 return filas423
424
425def _conf(palabras: list[dict]) -> float:426 c = [w["conf"] for w in palabras if w["conf"] >= 0]427 return sum(c) / len(c) if c else 0.0428
429
430def _preparar(img):431 """Gris y escala mínima para que la letra del cuerpo tenga ~30 px de alto."""432 import cv2433 g = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY) if img.ndim == 3 else img434 if max(g.shape) < 1600:435 f = 1600 / max(g.shape)436 g = cv2.resize(g, None, fx=f, fy=f, interpolation=cv2.INTER_CUBIC)437 return g438
439
440def _binarizar(g):441 """Fotos con luz despareja: normaliza el fondo y binariza (mismo método que la herramienta 1)."""442 import cv2443 fondo = cv2.morphologyEx(g, cv2.MORPH_CLOSE, cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (41, 41)))444 norm = cv2.divide(g, fondo, scale=255)445 return cv2.adaptiveThreshold(norm, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY, 31, 15)446
447
448def ocr_bloques(img, pagina, idioma: str, adv: list[str]) -> tuple[list[dict], float]:449 """OCR de una imagen -> (bloques, confianza media 0-100)."""450 if not shutil.which("tesseract"):451 adv.append("Tesseract is not installed: OCR skipped")452 return [], 0.0453 disp = _idiomas_tesseract()454 pedidos = [x for x in idioma.split("+") if x]455 usar = [x for x in pedidos if x in disp] or (["eng"] if "eng" in disp else disp[:1])456 faltan = [x for x in pedidos if x not in disp]457 if faltan:458 msg = f"OCR language(s) not available: {', '.join(faltan)}; used {'+'.join(usar)}"459 if msg not in adv:460 adv.append(msg)461 lang = "+".join(usar)462 g = _preparar(img)463 palabras = _tsv(g, lang)464 conf = _conf(palabras)465 if conf < 70:466 otra = _tsv(_binarizar(g), lang)467 if _conf(otra) > conf and len(otra) >= 0.5 * len(palabras):468 palabras, conf = otra, _conf(otra)469 if not palabras:470 return [], 0.0471 altura = statistics.median(w["height"] for w in palabras)472 bloques = []473 grupos: dict = {}474 for w in palabras:475 grupos.setdefault((w["block"], w["par"]), {}).setdefault(w["line"], []).append(w)476 for clave in sorted(grupos): # orden de bloques de Tesseract: ya lee columna por columna477 lineas = [sorted(ws, key=lambda w: w["left"]) for _, ws in sorted(grupos[clave].items())]478 textos = [" ".join(w["text"] for w in ln) for ln in lineas]479 alto = statistics.median(w["height"] for ln in lineas for w in ln)480 if len(lineas) <= 2 and alto >= 1.4 * altura and len(" ".join(textos)) <= 120:481 bloques.append(_b("heading", pagina, " ".join(textos), 2))482 continue483 actual = None484 for t in textos:485 m = VIÑETA.match(t)486 if m:487 actual = _b("list", pagina, t[m.end():].strip() if _es_simbolo(m.group(1)) else t)488 bloques.append(actual)489 elif actual is None:490 actual = _b("para", pagina, t)491 bloques.append(actual)492 else:493 actual["text"] = _unir_linea(actual["text"], t)494 return bloques, conf495
496
497# --------------------------------------------------------------------------------------------498# Lectores por formato499# --------------------------------------------------------------------------------------------500def parse_paginas(spec: str | None, total: int) -> list[int]:501 """'1-5, 8, 12-' -> [1,2,3,4,5,8,12,...,total]. Vacío o None = todas."""502 if not spec or not str(spec).strip():503 return list(range(1, total + 1))504 elegidas = set()505 for parte in str(spec).replace(" ", "").split(","):506 if not parte:507 continue508 m = re.fullmatch(r"(\d+)?-(\d+)?|(\d+)", parte)509 if not m:510 raise ValueError(f"invalid page range: '{parte}' (use e.g. 1-5, 8, 12-)")511 if m.group(3):512 a = b = int(m.group(3))513 else:514 a, b = int(m.group(1) or 1), int(m.group(2) or total)515 elegidas.update(range(max(1, a), min(total, b) + 1))516 return sorted(elegidas)517
518
519def _fecha_pdf(v: str) -> str:520 """'D:20261102211926+00'00'' -> '2026-11-02T21:19:26+00:00'. Otros valores quedan igual."""521 m = re.fullmatch(r"D:(\d{4})(\d{2})(\d{2})(\d{2})?(\d{2})?(\d{2})?([Z+\-])?(\d{2})?'?(\d{2})?'?", v)522 if not m:523 return v524 a, me, d, h, mi, se, z, zh, zm = m.groups()525 out = f"{a}-{me}-{d}T{h or '00'}:{mi or '00'}:{se or '00'}"526 if z == "Z" or (z and zh):527 out += "+00:00" if z == "Z" else f"{z}{zh}:{zm or '00'}"528 return out529
530
531def _leer_pdf(path, paginas_spec, ocr, idioma, max_paginas, adv) -> dict:532 import cv2533 import numpy as np534 import pdfplumber535 import pypdfium2 as pdfium536
537 paginas, meta = [], {}538 with pdfplumber.open(path) as pdf:539 total = len(pdf.pages)540 raw = pdf.metadata or {}541 meta = {k.lower(): _fecha_pdf(_celda(v)) for k, v in raw.items()542 if k in ("Title", "Author", "Subject", "Keywords", "Creator", "Producer", "CreationDate", "ModDate")543 and _celda(v) and _celda(v).lower() not in ("(unspecified)", "unknown", "untitled")}544 elegidas = parse_paginas(paginas_spec, total)545 if len(elegidas) > max_paginas:546 adv.append(f"processed {max_paginas} of {len(elegidas)} selected pages (max_pages limit)")547 elegidas = elegidas[:max_paginas]548 doc = None549 try:550 for n in elegidas:551 page = pdf.pages[n - 1]552 texto = page.extract_text() or ""553 util = len(re.sub(r"\s", "", texto))554 necesita_ocr = ocr == "always" or (ocr == "auto" and (util < MIN_TEXTO or _texto_roto(texto))555 and (page.images or _texto_roto(texto) or util > 0))556 if necesita_ocr:557 if doc is None:558 doc = pdfium.PdfDocument(path)559 pg = doc[n - 1]560 try:561 pil = pg.render(scale=300 / 72).to_pil().convert("RGB")562 finally:563 pg.close()564 img = cv2.cvtColor(np.array(pil), cv2.COLOR_RGB2BGR)565 bloques, conf = ocr_bloques(img, n, idioma, adv)566 paginas.append({"page": n, "method": "ocr", "blocks": bloques, "ocr_confidence": round(conf, 1),567 "tables": 0})568 continue569 if util == 0:570 if ocr == "never" and page.images:571 adv.append(f"page {n} looks scanned but OCR is disabled")572 paginas.append({"page": n, "method": "empty", "blocks": [], "tables": 0})573 continue574 if _texto_roto(texto):575 adv.append(f"page {n}: the text layer looks garbled (bad font encoding); try ocr='always'")576 items, nt = _pdf_items_pagina(page)577 paginas.append({"page": n, "method": "text", "items": items, "height": float(page.height),578 "tables": nt})579 finally:580 if doc is not None:581 doc.close()582
583 quitadas = _quitar_encabezados(paginas)584 tams = Counter()585 for p in paginas:586 for it in p.get("items", []):587 if it["kind"] == "line":588 tams[round(it["size"] * 2) / 2] += len(it["text"])589 cuerpo = tams.most_common(1)[0][0] if tams else 10.0590 niveles = _niveles_titulo(paginas, cuerpo, tams)591 for p in paginas:592 if p["method"] == "text":593 p["blocks"] = _items_a_bloques(p.pop("items"), p["page"], cuerpo, niveles)594 p.pop("height", None)595 return {"page_count": total, "pages": paginas, "metadata": meta, "removed_header_footer_lines": quitadas}596
597
598def _leer_imagen(path, idioma, adv) -> dict:599 import cv2600 img = cv2.imread(path)601 if img is None:602 raise ValueError("could not open the image")603 bloques, conf = ocr_bloques(img, 1, idioma, adv)604 return {"page_count": 1, "metadata": {},605 "pages": [{"page": 1, "method": "ocr", "blocks": bloques, "ocr_confidence": round(conf, 1), "tables": 0}]}606
607
608def _nivel_estilo(nombre: str) -> int:609 n = (nombre or "").lower()610 if n in ("title", "título", "titulo"):611 return 1612 m = re.match(r"(heading|título|titulo|encabezado)\s*(\d)", n)613 return int(m.group(2)) if m else 0614
615
616def _leer_docx(path, adv) -> dict:617 import docx618 from docx.table import Table619 from docx.text.paragraph import Paragraph620
621 d = docx.Document(path)622 bloques = []623 for el in d.element.body.iterchildren():624 tag = el.tag.rsplit("}", 1)[-1]625 if tag == "p":626 p = Paragraph(el, d)627 texto = re.sub(r"\s+", " ", p.text).strip()628 if not texto:629 continue630 estilo = p.style.name if p.style is not None else ""631 nivel = _nivel_estilo(estilo)632 es_lista = "list" in estilo.lower() or "lista" in estilo.lower() or el.find(633 ".//{http://schemas.openxmlformats.org/wordprocessingml/2006/main}numPr") is not None634 if nivel:635 bloques.append(_b("heading", None, texto, nivel))636 bloques[-1]["_title"] = estilo.lower() in ("title", "título", "titulo")637 elif es_lista:638 bloques.append(_b("list", None, texto))639 else:640 bloques.append(_b("para", None, texto))641 elif tag == "tbl":642 t = Table(el, d)643 filas = []644 for r in t.rows:645 celdas, vistas = [], set()646 for c in r.cells:647 if id(c._tc) in vistas: # celdas combinadas: python-docx repite la misma celda648 continue649 vistas.add(id(c._tc))650 celdas.append(c.text)651 filas.append(celdas)652 filas = _limpiar_filas(filas)653 if filas:654 bloques.append(_b("table", None, rows=filas))655 if any(b["kind"] == "heading" and b["level"] == 1 and b.get("_title") for b in bloques):656 for b in bloques: # con estilo 'Title', los 'Heading N' bajan un nivel657 if b["kind"] == "heading" and not b.get("_title"):658 b["level"] = min(b["level"] + 1, 6)659 cp = d.core_properties660 meta = {k: _celda(getattr(cp, k, None)) for k in ("title", "author", "subject", "keywords", "created", "modified")661 if _celda(getattr(cp, k, None))}662 return {"page_count": None, "metadata": meta, "pages": [{"page": None, "method": "text", "blocks": bloques,663 "tables": sum(b["kind"] == "table" for b in bloques)}]}664
665
666def _leer_xlsx(path, adv) -> dict:667 import openpyxl668 wb = openpyxl.load_workbook(path, read_only=True, data_only=True)669 bloques = []670 try:671 for ws in wb.worksheets:672 filas = []673 for i, r in enumerate(ws.iter_rows(values_only=True)):674 if i >= MAX_FILAS_HOJA:675 adv.append(f"sheet '{ws.title}': only the first {MAX_FILAS_HOJA} rows were read")676 break677 filas.append(list(r))678 filas = _limpiar_filas(filas)679 if not filas:680 continue681 bloques.append(_b("heading", None, f"Sheet: {ws.title}", 2))682 bloques.append(_b("table", None, rows=filas))683 finally:684 wb.close()685 return {"page_count": None, "metadata": {}, "pages": [{"page": None, "method": "text", "blocks": bloques,686 "tables": sum(b["kind"] == "table" for b in bloques)}]}687
688
689def _leer_pptx(path, paginas_spec, max_paginas, adv) -> dict:690 from pptx import Presentation691 from pptx.enum.shapes import MSO_SHAPE_TYPE692
693 prs = Presentation(path)694 total = len(prs.slides)695 elegidas = parse_paginas(paginas_spec, total)[:max_paginas]696 paginas = []697
698 def formas(shapes):699 for s in sorted(shapes, key=lambda s: ((s.top or 0), (s.left or 0))):700 if s.shape_type == MSO_SHAPE_TYPE.GROUP:701 yield from formas(s.shapes)702 else:703 yield s704
705 for n, slide in enumerate(prs.slides, 1):706 if n not in elegidas:707 continue708 titulo = slide.shapes.title.text.strip() if slide.shapes.title is not None and slide.shapes.title.has_text_frame else ""709 bloques = [_b("heading", n, f"Slide {n}" + (f": {titulo}" if titulo else ""), 2)]710 for s in formas(slide.shapes):711 if slide.shapes.title is not None and s.shape_id == slide.shapes.title.shape_id:712 continue713 if getattr(s, "has_table", False) and s.has_table:714 filas = _limpiar_filas([[c.text for c in r.cells] for r in s.table.rows])715 if filas:716 bloques.append(_b("table", n, rows=filas))717 elif getattr(s, "has_text_frame", False) and s.has_text_frame:718 for par in s.text_frame.paragraphs:719 t = re.sub(r"\s+", " ", "".join(r.text for r in par.runs)).strip()720 if t:721 bloques.append(_b("list" if par.level > 0 or len(s.text_frame.paragraphs) > 1 else "para", n, t))722 if slide.has_notes_slide:723 notas = re.sub(r"\s+", " ", slide.notes_slide.notes_text_frame.text or "").strip()724 if notas:725 bloques.append(_b("note", n, "Notes: " + notas))726 con_contenido = len(bloques) > 1 or bool(titulo)727 paginas.append({"page": n, "method": "text" if con_contenido else "empty",728 "blocks": bloques if con_contenido else [], "tables": sum(b["kind"] == "table" for b in bloques)})729 return {"page_count": total, "metadata": {}, "pages": paginas}730
731
732def _leer_texto(path, formato, adv) -> dict:733 with open(path, "rb") as f:734 crudo = f.read()735 for enc in ("utf-8-sig", "cp1252", "latin-1"):736 try:737 texto = crudo.decode(enc)738 break739 except UnicodeDecodeError:740 continue741 if formato == "csv":742 try:743 dialecto = csv.Sniffer().sniff(texto[:4096], delimiters=",;\t|")744 except csv.Error:745 dialecto = csv.excel746 filas = _limpiar_filas(list(csv.reader(io.StringIO(texto), dialecto))[:MAX_FILAS_HOJA])747 bloques = [_b("table", None, rows=filas)] if filas else []748 else:749 bloques = []750 for par in re.split(r"\n\s*\n", texto):751 par = par.strip()752 if not par:753 continue754 m = re.match(r"^(#{1,6})\s+(.*)", par)755 if m and "\n" not in par:756 bloques.append(_b("heading", None, m.group(2).strip(), len(m.group(1))))757 else:758 bloques.append(_b("para", None, par))759 return {"page_count": None, "metadata": {}, "pages": [{"page": None, "method": "text", "blocks": bloques,760 "tables": sum(b["kind"] == "table" for b in bloques)}]}761
762
763# --------------------------------------------------------------------------------------------764# Chunks para RAG765# --------------------------------------------------------------------------------------------766def tokens_aprox(texto: str) -> int:767 """Estimación sin tokenizer: ~4 caracteres por token (inglés/español)."""768 return math.ceil(len(texto) / 4)769
770
771def _partir_texto(texto: str, max_chars: int) -> list[str]:772 frases = re.split(r"(?<=[.!?;:])\s+", texto)773 partes, actual = [], ""774 for f in frases:775 while len(f) > max_chars: # frase larguísima: se corta por palabras776 corte = f.rfind(" ", 0, max_chars)777 corte = corte if corte > max_chars // 2 else max_chars778 if actual:779 partes.append(actual)780 actual = ""781 partes.append(f[:corte].strip())782 f = f[corte:].strip()783 if actual and len(actual) + 1 + len(f) > max_chars:784 partes.append(actual)785 actual = f786 else:787 actual = (actual + " " + f).strip()788 if actual:789 partes.append(actual)790 return partes791
792
793def _partir_bloque(b: dict, max_chars: int) -> list[dict]:794 if len(_bloque_md(b)) <= max_chars:795 return [b]796 if b["kind"] == "table":797 cab, cuerpo = b["rows"][0], b["rows"][1:]798 trozos, actual = [], []799 for fila in cuerpo:800 if actual and len(tabla_markdown([cab] + actual + [fila])) > max_chars:801 trozos.append(dict(b, rows=[cab] + actual)) # cada trozo repite el encabezado de la tabla802 actual = []803 actual.append(fila)804 if actual:805 trozos.append(dict(b, rows=[cab] + actual))806 return trozos807 return [dict(b, text=t) for t in _partir_texto(b["text"], max_chars)]808
809
810def _cola(texto: str, n: int) -> str:811 if n <= 0 or len(texto) <= n:812 return "" if n <= 0 else texto813 cola = texto[-n:]814 m = re.search(r"(?<=[.!?])\s+", cola)815 if m and m.end() < len(cola) - 20:816 return cola[m.end():]817 esp = cola.find(" ")818 return cola[esp + 1:] if esp >= 0 else cola819
820
821def armar_chunks(bloques: list[dict], chunk_tokens: int = 800, overlap_tokens: int = 80) -> list[dict]:822 """Chunks que respetan secciones, no parten tablas sin repetir el encabezado y llevan página y sección."""823 if not chunk_tokens or chunk_tokens <= 0:824 return []825 max_chars = max(200, chunk_tokens * 4)826 solape = min(max(0, overlap_tokens) * 4, max_chars // 2)827 chunks, ruta, partes, paginas, seccion = [], {}, [], [], ""828
829 def cerrar(por_tamano: bool):830 nonlocal partes, paginas831 texto = "\n\n".join(partes).strip()832 if texto:833 ps = sorted({p for p in paginas if p is not None})834 chunks.append({"chunk_id": len(chunks) + 1, "text": texto, "section": seccion,835 "pages": [ps[0], ps[-1]] if ps else None, "tokens_est": tokens_aprox(texto)})836 cola = _cola(texto, solape) if por_tamano else ""837 partes = [cola] if cola else []838 paginas = paginas[-1:] if cola else []839
840 for b in bloques:841 if b["kind"] == "heading":842 largo = len("\n\n".join(partes))843 if largo >= 0.25 * max_chars:844 cerrar(False)845 ruta = {k: v for k, v in ruta.items() if k < b["level"]}846 ruta[b["level"]] = b["text"]847 if all(x.startswith("#") for x in partes): # el chunk todavía solo tiene títulos848 seccion = " > ".join(ruta[k] for k in sorted(ruta))849 for pieza in _partir_bloque(b, max_chars):850 md = _bloque_md(pieza)851 actual = len("\n\n".join(partes))852 if partes and actual + 2 + len(md) > max_chars:853 cerrar(pieza["kind"] != "table")854 if pieza["kind"] == "table":855 partes, paginas = [], []856 seccion = " > ".join(ruta[k] for k in sorted(ruta))857 if not partes:858 seccion = " > ".join(ruta[k] for k in sorted(ruta))859 partes.append(md)860 paginas.append(pieza["page"])861 cerrar(False)862 return chunks863
864
865# --------------------------------------------------------------------------------------------866# Entrada principal867# --------------------------------------------------------------------------------------------868def detectar_formato(path: str) -> str:869 ext = os.path.splitext(path)[1].lower()870 if ext in FORMATOS:871 return FORMATOS[ext]872 with open(path, "rb") as f:873 cab = f.read(8)874 if cab.startswith(b"%PDF-"):875 return "pdf"876 if ext in (".doc", ".xls", ".ppt"):877 raise FormatoNoSoportado(f"legacy Office format ({ext}) is not supported; save it as {ext}x or PDF")878 raise FormatoNoSoportado(f"unsupported format ({ext or 'unknown'}): use PDF, image, DOCX, XLSX, PPTX, TXT, MD or CSV")879
880
881def _unir_cortes(bloques: list[dict]) -> list[dict]:882 """Une un párrafo cortado por un salto de columna o de página ('...reduce el consumo de' + 'agua en...')."""883 out = []884 for b in bloques:885 prev = out[-1] if out else None886 if (prev and prev["kind"] == "para" and b["kind"] == "para"887 and not re.search(r"[.!?:;\"”»)]$", prev["text"]) and b["text"][:1].islower()):888 out[-1] = dict(prev, text=_unir_linea(prev["text"], b["text"]))889 else:890 out.append(b)891 return out892
893
894def convert(path: str, pages: str | None = None, ocr: str = "auto", ocr_languages: str = "spa+eng",895 max_pages: int = 200, chunk_tokens: int = 800, chunk_overlap: int = 80) -> dict:896 """Convierte un documento. Devuelve el ítem de salida completo (ver README para el esquema)."""897 t0 = time.perf_counter()898 adv: list[str] = []899 if ocr not in ("auto", "always", "never"):900 raise ValueError("ocr must be 'auto', 'always' or 'never'")901 formato = detectar_formato(path)902 if formato == "pdf":903 r = _leer_pdf(path, pages, ocr, ocr_languages, max_pages, adv)904 elif formato == "image":905 if ocr == "never":906 raise ValueError("an image needs OCR, but ocr is set to 'never'")907 r = _leer_imagen(path, ocr_languages, adv)908 elif formato == "docx":909 r = _leer_docx(path, adv)910 elif formato == "xlsx":911 r = _leer_xlsx(path, adv)912 elif formato == "pptx":913 r = _leer_pptx(path, pages, max_pages, adv)914 else:915 r = _leer_texto(path, formato, adv)916
917 bloques = _unir_cortes([b for p in r["pages"] for b in p["blocks"]])918 tablas, n = [], 0919 for b in bloques:920 if b["kind"] == "table":921 n += 1922 tablas.append({"table_id": n, "page": b["page"], "n_rows": len(b["rows"]), "n_cols": len(b["rows"][0]),923 "rows": b["rows"], "markdown": tabla_markdown(b["rows"]), "csv": tabla_csv(b["rows"])})924 paginas_out = []925 for p in r["pages"]:926 md = _a_markdown(p["blocks"])927 item = {"page": p["page"], "method": p["method"] if (md or p["method"] == "empty") else "empty",928 "chars": len(md), "tables": p.get("tables", 0), "markdown": md}929 if "ocr_confidence" in p:930 item["ocr_confidence"] = p["ocr_confidence"]931 if md and p["ocr_confidence"] < 60:932 adv.append(f"page {p['page']}: low OCR confidence ({p['ocr_confidence']:.0f}/100); check the text")933 paginas_out.append(item)934
935 markdown = _a_markdown(bloques)936 if formato == "pdf" and any(p["method"] == "ocr" for p in paginas_out) and n == 0:937 adv.append("tables are only detected on pages with a text layer; scanned pages come out as text")938
939 # Unidades de cobro (DEC-006): nada se cobra si no hay salida.940 if formato in ("pdf", "image", "pptx"):941 eventos = {"page": sum(1 for p in paginas_out if p["method"] == "text" and p["chars"] > 0),942 "ocr-page": sum(1 for p in paginas_out if p["method"] == "ocr" and p["chars"] > 0)}943 else:944 eventos = {"page": math.ceil(len(markdown) / CHARS_PAGINA_EQ) if markdown else 0, "ocr-page": 0}945
946 out = {947 "status": "ok" if markdown else "empty",948 "format": formato,949 "page_count": r.get("page_count"),950 "pages_processed": len(paginas_out) if formato in ("pdf", "pptx", "image") else None,951 "metadata": r.get("metadata", {}),952 "markdown": markdown,953 "tables": tablas,954 "chunks": armar_chunks(bloques, chunk_tokens, chunk_overlap),955 "pages": paginas_out,956 "billing": eventos,957 "stats": {"chars": len(markdown), "tokens_est": tokens_aprox(markdown), "n_tables": n,958 "n_ocr_pages": sum(1 for p in paginas_out if p["method"] == "ocr"),959 "removed_header_footer_lines": r.get("removed_header_footer_lines", 0),960 "seconds": round(time.perf_counter() - t0, 2), "engine_version": VERSION},961 "warnings": adv,962 }963 if not markdown:964 adv.append("no text could be extracted")965 return out966
967
968if __name__ == "__main__":969 import json970 import sys971 for f in sys.argv[1:]:972 res = convert(f)973 print(res["markdown"])974 print(json.dumps({k: v for k, v in res.items() if k not in ("markdown", "pages")}, ensure_ascii=False,975 indent=1)[:3000])1"""Actor de Apify: documentos -> Markdown, tablas y chunks para RAG (herramienta 2, DEC-006).2
3Red: solo descarga los archivos que el usuario pasa en la entrada. El token de Apify se manda4únicamente a api.apify.com (archivos subidos al key-value store elegido por el usuario).5Nada se guarda fuera del dataset y del key-value store de la corrida; los temporales se borran.6Cobro: `page` por página con texto y `ocr-page` por página que necesitó OCR. Sin salida, no se cobra.7"""8from __future__ import annotations9
10import asyncio11import json12import os13import re14import tempfile15import urllib.error16import urllib.request17from urllib.parse import quote, unquote, urlparse18
19from apify import Actor20
21from demo_pdf import demo_bytes22from documentos import FORMATOS, FormatoNoSoportado, convert23
24API = "https://api.apify.com/v2"25MAX_BYTES = 50 * 1024 * 102426MAX_ITEM_BYTES = 8_000_000 # el límite de un ítem de dataset en Apify es ~9 MB27EVENTOS = ("page", "ocr-page")28TIPOS_MIME = {29 "application/pdf": ".pdf", "image/png": ".png", "image/jpeg": ".jpg", "image/webp": ".webp",30 "image/tiff": ".tif", "image/bmp": ".bmp", "text/plain": ".txt", "text/markdown": ".md", "text/csv": ".csv",31 "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",32 "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",33 "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",34}35AVISO_403 = ("no permission to read this file: if you uploaded it from the input form, also select its storage "36 "in 'Storage with uploaded files' (this Actor runs with limited permissions)")37
38
39def _get(url: str, max_bytes: int = MAX_BYTES):40 """GET con el token de Apify solo hacia api.apify.com. Devuelve (bytes, mime)."""41 headers = {"User-Agent": "documents-to-markdown-rag-actor"}42 token = os.environ.get("APIFY_TOKEN")43 if token and urlparse(url).hostname == "api.apify.com":44 headers["Authorization"] = f"Bearer {token}"45 try:46 with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=90) as r:47 return r.read(max_bytes + 1), (r.headers.get_content_type() or "").lower()48 except urllib.error.HTTPError as e:49 if e.code in (401, 403) and urlparse(url).hostname == "api.apify.com":50 raise ValueError(AVISO_403) from e51 raise ValueError(f"download failed (HTTP {e.code})") from e52 except urllib.error.URLError as e:53 raise ValueError(f"download failed ({e.reason})") from e54
55
56def _refs_de_almacen(almacen: str, limite: int = 1000) -> list[str]:57 """URLs de los documentos de un key-value store elegido en la entrada (permiso READ)."""58 base = f"{API}/key-value-stores/{quote(almacen, safe='~')}"59 datos, _ = _get(f"{base}/keys?limit={limite}", max_bytes=5 * 1024 * 1024)60 claves = [i["key"] for i in json.loads(datos)["data"]["items"]]61 return [f"{base}/records/{quote(k)}" for k in claves if os.path.splitext(k)[1].lower() in FORMATOS]62
63
64def _descargar(ref: str, destino: str, local_ok: bool) -> tuple[str, str]:65 """Devuelve (ruta_local, nombre). Lanza ValueError con un mensaje para el usuario."""66 if local_ok and os.path.isfile(ref): # solo fuera de la plataforma (pruebas locales)67 return ref, os.path.basename(ref)68 u = urlparse(ref)69 if u.scheme not in ("http", "https"):70 raise ValueError("not an http(s) URL or an uploaded file")71 datos, mime = _get(ref)72 if len(datos) > MAX_BYTES:73 raise ValueError("file is larger than 50 MB")74 nombre = unquote(os.path.basename(u.path)) or "document"75 ext = os.path.splitext(nombre)[1].lower()76 if ext not in FORMATOS:77 ext = TIPOS_MIME.get(mime) or (".pdf" if datos[:5] == b"%PDF-" else "")78 if ext not in FORMATOS:79 raise ValueError(f"unsupported format ({mime or 'unknown'}): use PDF, image, DOCX, XLSX, PPTX, TXT, MD or CSV")80 ruta = os.path.join(destino, f"input{ext}")81 with open(ruta, "wb") as f:82 f.write(datos)83 return ruta, nombre84
85
86def _convertir(ref: str, local_ok: bool, opciones: dict) -> dict:87 with tempfile.TemporaryDirectory() as tmp:88 ruta, nombre = _descargar(ref, tmp, local_ok)89 r = convert(ruta, **opciones)90 r["file"] = nombre91 return r92
93
94def _convertir_demo(opciones: dict) -> dict:95 with tempfile.TemporaryDirectory() as tmp:96 ruta = os.path.join(tmp, "demo.pdf")97 with open(ruta, "wb") as f:98 f.write(demo_bytes())99 return convert(ruta, **dict(opciones, pages=None))100
101
102def _clave_kv(nombre: str, i: int) -> str:103 base = re.sub(r"[^a-zA-Z0-9!\-_.'()]", "_", os.path.splitext(nombre)[0])[:180] or "document"104 return f"{i:03d}-{base}.md"105
106
107def _items_de(r: dict, entrada: str, modo: str, incluir_paginas: bool) -> list[dict]:108 """Arma los ítems del dataset para un documento según el modo de salida."""109 base = {"input": entrada, "file": r["file"]}110 if modo == "chunks":111 n = len(r["chunks"])112 return [dict(base, chunk_id=c["chunk_id"], chunk_count=n, section=c["section"], pages=c["pages"],113 tokens_est=c["tokens_est"], text=c["text"], format=r["format"],114 title=r["metadata"].get("title")) for c in r["chunks"]]115 if modo == "pages":116 return [dict(base, page=p["page"], method=p["method"], chars=p["chars"], markdown=p["markdown"],117 ocr_confidence=p.get("ocr_confidence"),118 tables=[t for t in r["tables"] if t["page"] == p["page"]]) for p in r["pages"]]119 item = dict(base, **{k: v for k, v in r.items() if k != "file"})120 if not incluir_paginas:121 item.pop("pages", None)122 return [item]123
124
125def _cupo() -> dict:126 """Cuántos eventos se pueden cobrar todavía dentro del tope que fijó el usuario (None = sin tope)."""127 cm = Actor.get_charging_manager()128 return {e: cm.calculate_max_event_charge_count_within_limit(e) for e in EVENTOS}129
130
131async def main() -> None:132 async with Actor:133 entrada = await Actor.get_input() or {}134 refs = [x.strip() for x in (entrada.get("documents") or []) if isinstance(x, str) and x.strip()]135 almacen = (entrada.get("storage") or "").strip()136 modo = entrada.get("output_mode") or "document"137 incluir_paginas = bool(entrada.get("include_pages", False))138 guardar_md = entrada.get("save_markdown_files", True) is not False139 opciones = {140 "pages": entrada.get("pages") or None,141 "ocr": entrada.get("ocr") or "auto",142 "ocr_languages": entrada.get("ocr_languages") or "spa+eng",143 "max_pages": int(entrada.get("max_pages") or 200),144 "chunk_tokens": int(entrada.get("chunk_tokens") if entrada.get("chunk_tokens") is not None else 800),145 "chunk_overlap": int(entrada.get("chunk_overlap") if entrada.get("chunk_overlap") is not None else 80),146 }147 if modo == "chunks" and opciones["chunk_tokens"] <= 0:148 opciones["chunk_tokens"] = 800149 local_ok = not Actor.is_at_home()150
151 if almacen and not refs:152 try:153 refs = await asyncio.to_thread(_refs_de_almacen, almacen)154 except Exception as e: # noqa: BLE001155 await Actor.fail(status_message=f"Could not read the selected storage: {e}")156 return157 if not refs:158 await Actor.fail(status_message="The selected storage has no supported documents.")159 return160 refs = list(dict.fromkeys(refs))161
162 if not refs: # demostración: sin entrada, un documento de ejemplo incluido en la imagen; no se cobra163 r = await asyncio.to_thread(_convertir_demo, opciones)164 r["file"] = "demo.pdf"165 r["status"] = "demo"166 r["billing"] = {e: 0 for e in EVENTOS}167 r["warnings"].insert(0, "Demo with a bundled sample document (not charged). "168 "Add your own files or URLs to convert them.")169 for it in _items_de(r, "demo", modo, incluir_paginas):170 await Actor.push_data(dict(it, status="demo"))171 await Actor.set_status_message("Demo run with a sample document: add your files or URLs.",172 is_terminal=True)173 return174
175 ok = vacios = errores = 0176 paginas_cobradas = {e: 0 for e in EVENTOS}177 for i, ref in enumerate(refs, 1):178 cupo = _cupo()179 if all(v is not None and v <= 0 for v in cupo.values()):180 Actor.log.info("Reached the maximum cost per run you set; stopping here.")181 break182 await Actor.set_status_message(f"Converting document {i} of {len(refs)}")183 try:184 r = await asyncio.to_thread(_convertir, ref, local_ok, opciones)185 except (ValueError, FormatoNoSoportado) as e:186 await Actor.push_data({"input": ref, "status": "error", "warnings": [str(e)]})187 errores += 1188 continue189 except Exception as e: # noqa: BLE001 — se informa y se sigue con el resto190 Actor.log.warning(f"Document {i}: {type(e).__name__}: {e}")191 await Actor.push_data({"input": ref, "status": "error",192 "warnings": ["could not read the file (damaged, encrypted or unsupported?)"]})193 errores += 1194 continue195
196 if r["status"] != "ok":197 vacios += 1198 await Actor.push_data({"input": ref, "file": r["file"], "status": r["status"],199 "format": r["format"], "page_count": r["page_count"],200 "billing": {e: 0 for e in EVENTOS}, "warnings": r["warnings"]})201 continue202
203 if guardar_md:204 clave = _clave_kv(r["file"], i)205 await Actor.set_value(clave, r["markdown"], content_type="text/markdown; charset=utf-8")206 kv = await Actor.open_key_value_store()207 r["markdown_file"] = await kv.get_public_url(clave) if Actor.is_at_home() else clave208
209 items = _items_de(r, ref, modo, incluir_paginas)210 if modo == "document" and len(json.dumps(items[0], ensure_ascii=False).encode()) > MAX_ITEM_BYTES:211 items = _items_de(r, ref, "chunks", incluir_paginas)212 if items:213 items[0]["warnings"] = ["document too large for one dataset item: delivered as chunks"]214 limite = False215 for e in EVENTOS:216 n = r["billing"].get(e, 0)217 if n:218 res = await Actor.charge(e, count=n)219 paginas_cobradas[e] += res.charged_count220 limite = limite or res.event_charge_limit_reached221 if items:222 await Actor.push_data(items)223 ok += 1224 if limite:225 Actor.log.info("Reached the maximum cost per run you set; stopping here.")226 break227
228 await Actor.set_status_message(229 f"Done: {ok} converted, {vacios} without text, {errores} errors "230 f"({paginas_cobradas['page']} pages, {paginas_cobradas['ocr-page']} OCR pages).", is_terminal=True)231
232
233if __name__ == "__main__":234 asyncio.run(main())1# Dependencias fijadas del actor (sello de confianza). Licencias permisivas, nada AGPL.2apify==4.2.03pdfplumber==0.11.104pypdfium2==5.13.05opencv-python-headless==5.0.0.936numpy==2.5.37python-docx==1.2.08openpyxl==3.1.59python-pptx==1.0.2