1"""Document -> Markdown conversion core (no Apify dependencies, testable locally).
2
3PDF: pdfplumber text and tables, headings from font size; pages without a text layer go to Tesseract OCR.
4Office/HTML/text: markitdown. Legacy formats (doc, xls, ppt, odt, rtf, ...): LibreOffice first.
5Images: Tesseract OCR.
6"""
7from __future__ import annotations
8
9import os
10import re
11import shutil
12import statistics
13import subprocess
14import tempfile
15from dataclasses import dataclass, field
16
17import pdfplumber
18import pypdfium2 as pdfium
19import pytesseract
20from PIL import Image, ImageOps
21
22PDF_EXT = {'.pdf'}
23IMAGE_EXT = {'.png', '.jpg', '.jpeg', '.tif', '.tiff', '.bmp', '.webp', '.gif'}
24SHEET_EXT = {'.xlsx', '.xlsm'}
25MARKITDOWN_EXT = {'.docx', '.pptx', '.html', '.htm', '.csv', '.json', '.xml', '.txt', '.md', '.epub'}
26
27LIBREOFFICE_TARGET = {
28 '.doc': ('html:XHTML Writer File:UTF8', '.html'), '.odt': ('html:XHTML Writer File:UTF8', '.html'),
29 '.rtf': ('html:XHTML Writer File:UTF8', '.html'), '.wpd': ('html:XHTML Writer File:UTF8', '.html'),
30 '.dot': ('html:XHTML Writer File:UTF8', '.html'), '.dotx': ('html:XHTML Writer File:UTF8', '.html'),
31 '.docm': ('html:XHTML Writer File:UTF8', '.html'),
32 '.xls': ('xlsx', '.xlsx'), '.ods': ('xlsx', '.xlsx'), '.xlsb': ('xlsx', '.xlsx'),
33 '.ppt': ('pptx', '.pptx'), '.odp': ('pptx', '.pptx'), '.pps': ('pptx', '.pptx'), '.ppsx': ('pptx', '.pptx'),
34 '.pptm': ('pptx', '.pptx'),
35}
36SUPPORTED_EXT = PDF_EXT | IMAGE_EXT | SHEET_EXT | MARKITDOWN_EXT | set(LIBREOFFICE_TARGET)
37
38
39MIN_TEXT_CHARS = 25
40OCR_DPI = 300
41
42CHARS_PER_PAGE = 3000
43
44
45class ConversionError(Exception):
46 pass
47
48
49@dataclass
50class Result:
51 markdown: str = ''
52 page_count: int = 0
53 ocr_pages: int = 0
54 source_pages: int | None = None
55 metadata: dict = field(default_factory=dict)
56 warnings: list[str] = field(default_factory=list)
57
58
59def detect_ext(path: str, name_hint: str = '', content_type: str = '') -> str:
60 """Extension from the file name or, failing that, from magic bytes / content type."""
61 for candidate in (name_hint, path):
62 ext = os.path.splitext(candidate.split('?')[0])[1].lower()
63 if ext in SUPPORTED_EXT:
64 return ext
65 with open(path, 'rb') as f:
66 head = f.read(8)
67 if head.startswith(b'%PDF'):
68 return '.pdf'
69 if head.startswith(b'\x89PNG'):
70 return '.png'
71 if head.startswith(b'\xff\xd8'):
72 return '.jpg'
73 if head[:4] in (b'II*\x00', b'MM\x00*'):
74 return '.tiff'
75 if head.startswith(b'{\\rtf'):
76 return '.rtf'
77 ct = content_type.split(';')[0].strip().lower()
78 by_type = {
79 'application/pdf': '.pdf', 'text/html': '.html', 'text/plain': '.txt', 'text/csv': '.csv',
80 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
81 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',
82 'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx',
83 'application/msword': '.doc', 'application/vnd.ms-excel': '.xls', 'application/vnd.ms-powerpoint': '.ppt',
84 'application/vnd.oasis.opendocument.text': '.odt', 'application/rtf': '.rtf',
85 }
86 if ct in by_type:
87 return by_type[ct]
88 if head.startswith(b'PK'):
89 return '.docx'
90 raise ConversionError(f'unsupported file type (name {name_hint!r}, content type {content_type!r})')
91
92
93ZIP_EXT = {'.docx', '.docm', '.dotx', '.xlsx', '.xlsm', '.xlsb', '.pptx', '.pptm', '.ppsx', '.odt', '.ods', '.odp',
94 '.epub'}
95OLE_EXT = {'.doc', '.dot', '.xls', '.ppt', '.pps'}
96
97
98def check_signature(path: str, ext: str) -> None:
99 """LibreOffice and markitdown read unreadable binaries as plain text; refuse them instead of charging for junk."""
100 with open(path, 'rb') as f:
101 head = f.read(1024)
102 if ((ext in ZIP_EXT and not head.startswith(b'PK'))
103 or (ext in OLE_EXT and not head.startswith((b'\xd0\xcf\x11\xe0', b'PK')))
104 or (ext == '.rtf' and not head.startswith(b'{\\rtf'))
105 or (ext == '.pdf' and b'%PDF' not in head)):
106 raise ConversionError(f'not a valid {ext} file (damaged or wrong extension)')
107
108
109def convert(path: str, ext: str, *, ocr_mode: str = 'auto', ocr_languages: str = 'eng',
110 max_pages: int = 500, page_budget=None) -> Result:
111 """page_budget(text_pages, ocr_pages) -> bool lets the caller stop before work it cannot charge for."""
112 check_signature(path, ext)
113 if ext in PDF_EXT:
114 return _convert_pdf(path, ocr_mode, ocr_languages, max_pages, page_budget)
115 if ext in IMAGE_EXT:
116 return _convert_image(path, ocr_languages, max_pages)
117 if ext in LIBREOFFICE_TARGET:
118 target, new_ext = LIBREOFFICE_TARGET[ext]
119 with tempfile.TemporaryDirectory() as tmp:
120 converted = _libreoffice(path, target, new_ext, tmp)
121 return _convert_sheet(converted) if new_ext in SHEET_EXT else _convert_markitdown(converted)
122 if ext in SHEET_EXT:
123 return _convert_sheet(path)
124 return _convert_markitdown(path)
125
126
127
128
129def _convert_pdf(path, ocr_mode, langs, max_pages, page_budget) -> Result:
130 res = Result()
131 try:
132 pdf = pdfplumber.open(path)
133 except Exception as e:
134 raise ConversionError(f'cannot open PDF: {e}') from e
135 with pdf:
136 res.source_pages = len(pdf.pages)
137 info = pdf.metadata or {}
138 res.metadata = {k.lower(): str(v) for k, v in info.items() if k in ('Title', 'Author', 'Subject', 'CreationDate') and v}
139 pages = pdf.pages[:max_pages]
140 if len(pdf.pages) > max_pages:
141 res.warnings.append(f'only the first {max_pages} of {len(pdf.pages)} pages were converted')
142 body_size = _body_font_size(pages)
143 parts = []
144 doc = None
145 for n, page in enumerate(pages, 1):
146 text_md = '' if ocr_mode == 'always' else _pdf_page_markdown(page, body_size)
147 need_ocr = ocr_mode == 'always' or (ocr_mode == 'auto' and _looks_scanned(page, text_md))
148 if page_budget and not page_budget(0 if need_ocr else 1, 1 if need_ocr else 0):
149 res.warnings.append(f'stopped at page {n}: maximum charge per run reached')
150 break
151 if need_ocr:
152 if doc is None:
153 doc = pdfium.PdfDocument(path)
154 img = doc[n - 1].render(scale=OCR_DPI / 72).to_pil()
155 text_md = _ocr_image(img, langs)
156 res.ocr_pages += 1
157 else:
158 res.page_count += 1
159 if text_md.strip():
160 parts.append(text_md.strip())
161 if doc is not None:
162 doc.close()
163 res.markdown = '\n\n'.join(parts) + '\n'
164 return res
165
166
167def _looks_scanned(page, text_md: str) -> bool:
168 """No text layer at all, or only a few characters on top of a page-sized image (scanner stamp, OCR'd title)."""
169 if not page.chars:
170 return bool(page.images) or len(page.curves) + len(page.lines) > 20
171 if len(text_md.strip()) >= MIN_TEXT_CHARS:
172 return False
173 page_area = float(page.width * page.height) or 1.0
174 return any(float((im['x1'] - im['x0']) * (im['bottom'] - im['top'])) > 0.5 * page_area for im in page.images)
175
176
177def _body_font_size(pages) -> float:
178 sizes = []
179 for page in pages[:20]:
180 sizes.extend(round(c['size'], 1) for c in page.chars[:4000])
181 return statistics.median(sizes) if sizes else 10.0
182
183
184def _pdf_page_markdown(page, body_size: float) -> str:
185 """Text regions between tables (top to bottom) with tables rendered as Markdown."""
186 try:
187 tables = sorted(page.find_tables(), key=lambda t: t.bbox[1])
188 except Exception:
189 tables = []
190 out, top = [], 0
191 for t in tables:
192 x0, t_top, x1, t_bottom = t.bbox
193 if t_top > top:
194 out.append(_region_text(page, top, t_top, body_size))
195 rows = t.extract()
196 if rows and len(rows) >= 2 and max(len(r) for r in rows) >= 2:
197 out.append(table_to_markdown(rows))
198 else:
199 out.append(_region_text(page, t_top, t_bottom, body_size))
200 top = max(top, t_bottom)
201 if top < page.height:
202 out.append(_region_text(page, top, page.height, body_size))
203 return '\n\n'.join(p for p in out if p.strip())
204
205
206BULLET = re.compile(r'^[•●▪◦\-–*]\s+')
207NUMBERED = re.compile(r'^(\d{1,3})[.)]\s+')
208
209
210def _region_text(page, top, bottom, body_size) -> str:
211 bottom = min(bottom, page.height)
212 if bottom - top < 1:
213 return ''
214 region = page.crop((0, max(0, top), page.width, bottom), strict=False)
215 lines = region.extract_text_lines(return_chars=True, strip=True)
216 md, para = [], []
217
218 def flush():
219 if para:
220 md.append(' '.join(para))
221 para.clear()
222
223 for ln in lines:
224 text = ln['text'].strip()
225 if not text:
226 continue
227 size = statistics.median(c['size'] for c in ln['chars']) if ln['chars'] else body_size
228 if size >= body_size * 1.45 and len(text) < 120:
229 flush()
230 md.append('# ' + text)
231 elif size >= body_size * 1.15 and len(text) < 120:
232 flush()
233 md.append('## ' + text)
234 elif BULLET.match(text):
235 flush()
236 md.append('- ' + BULLET.sub('', text))
237 elif NUMBERED.match(text):
238 flush()
239 md.append(NUMBERED.sub(r'\1. ', text))
240 else:
241 para.append(text)
242 flush()
243 return '\n\n'.join(md)
244
245
246def table_to_markdown(rows) -> str:
247 width = max(len(r) for r in rows)
248 clean = [[_cell(c) for c in list(r) + [''] * (width - len(r))] for r in rows]
249 lines = ['| ' + ' | '.join(clean[0]) + ' |', '| ' + ' | '.join(['---'] * width) + ' |']
250 lines += ['| ' + ' | '.join(r) + ' |' for r in clean[1:]]
251 return '\n'.join(lines)
252
253
254def _cell(c) -> str:
255 return re.sub(r'\s+', ' ', str(c or '')).replace('|', '\\|').strip()
256
257
258
259
260def _ocr_image(img: Image.Image, langs: str) -> str:
261 img = ImageOps.exif_transpose(img).convert('L')
262 try:
263 text = pytesseract.image_to_string(img, lang=langs, config='--psm 3')
264 except pytesseract.TesseractError as e:
265 raise ConversionError(f'OCR failed: {e}') from e
266
267 paras = [re.sub(r'\s*\n\s*', ' ', p).strip() for p in re.split(r'\n\s*\n', text)]
268 return '\n\n'.join(p for p in paras if p)
269
270
271def _convert_image(path, langs, max_pages) -> Result:
272 res = Result()
273 try:
274 img = Image.open(path)
275 except Exception as e:
276 raise ConversionError(f'cannot open image: {e}') from e
277 frames = getattr(img, 'n_frames', 1)
278 res.source_pages = frames
279 parts = []
280 for i in range(min(frames, max_pages)):
281 img.seek(i)
282 parts.append(_ocr_image(img.copy(), langs))
283 res.ocr_pages += 1
284 if frames > max_pages:
285 res.warnings.append(f'only the first {max_pages} of {frames} frames were converted')
286 res.markdown = '\n\n'.join(p for p in parts if p) + '\n'
287 return res
288
289
290
291
292_markitdown = None
293
294
295def _convert_markitdown(path) -> Result:
296 global _markitdown
297 if _markitdown is None:
298 from markitdown import MarkItDown
299 _markitdown = MarkItDown(enable_plugins=False)
300 try:
301 out = _markitdown.convert_local(path)
302 except Exception as e:
303 raise ConversionError(f'cannot convert document: {e}') from e
304 res = Result(markdown=_tidy(out.text_content or '') + '\n')
305 if getattr(out, 'title', None):
306 res.metadata['title'] = out.title
307 res.page_count = _char_pages(res.markdown)
308 return res
309
310
311def _char_pages(md: str) -> int:
312 return max(1, -(-len(md.strip()) // CHARS_PER_PAGE))
313
314
315def _tidy(md: str) -> str:
316 md = re.sub(r'<!--\s*Slide number:\s*(\d+)\s*-->', r'## Slide \1', md)
317 md = re.sub(r'[]', '', md)
318 md = _fix_tables(md)
319 md = re.sub(r'\n{3,}', '\n\n', md)
320 return md.strip()
321
322
323SEPARATOR = re.compile(r'^\|(\s*:?-{3,}:?\s*\|)+$')
324
325
326def _fix_tables(md: str) -> str:
327 """Every pipe table gets a separator row; an empty header row is replaced by the first data row."""
328 out, block = [], []
329
330 def flush():
331 if block:
332 rows = [r for r in block if not SEPARATOR.match(r.strip())]
333 if rows and not re.sub(r'[|\s]', '', rows[0]) and len(rows) > 1:
334 rows = rows[1:]
335 if rows:
336 ncol = rows[0].strip().count('|') - 1
337 out.extend([rows[0], '| ' + ' | '.join(['---'] * max(1, ncol)) + ' |'] + rows[1:])
338 block.clear()
339
340 for line in md.splitlines():
341 s = line.strip()
342 if s.startswith('|') and s.endswith('|') and len(s) > 1:
343 block.append(s)
344 else:
345 flush()
346 out.append(line)
347 flush()
348 return '\n'.join(out)
349
350
351def _convert_sheet(path) -> Result:
352 """XLSX via openpyxl: text cells as written, numbers and dates by the cell's own format."""
353 from openpyxl import load_workbook
354 try:
355 wb = load_workbook(path, read_only=True, data_only=True)
356 except Exception as e:
357 raise ConversionError(f'cannot open spreadsheet: {e}') from e
358 parts = []
359 for ws in wb.worksheets:
360 rows = [[_sheet_value(c) for c in row] for row in ws.iter_rows()]
361 rows = [r for r in rows if any(v for v in r)]
362 if not rows:
363 continue
364 used = [j for j in range(max(len(r) for r in rows)) if any(j < len(r) and r[j] for r in rows)]
365 rows = [[r[j] if j < len(r) else '' for j in used] for r in rows]
366 parts.append(f'## {ws.title}\n\n' + (table_to_markdown(rows) if len(used) > 1 or len(rows) > 1
367 else rows[0][0]))
368 wb.close()
369 md = '\n\n'.join(parts) + '\n'
370 return Result(markdown=md, page_count=_char_pages(md))
371
372
373def _sheet_value(cell) -> str:
374 v = cell.value
375 if v is None:
376 return ''
377 if isinstance(v, bool):
378 return 'TRUE' if v else 'FALSE'
379 if isinstance(v, (int, float)):
380 fmt = (getattr(cell, 'number_format', '') or '').split(';')[0]
381 m = re.search(r'0[.,](0+)', fmt)
382 if m:
383 s = f'{v:,.{len(m.group(1))}f}' if ',' in fmt.split('.')[0] else f'{v:.{len(m.group(1))}f}'
384 elif isinstance(v, float) and v.is_integer():
385 s = str(int(v))
386 else:
387 s = repr(v) if isinstance(v, float) else str(v)
388 return s + '%' if fmt.endswith('%') and m else s
389 if hasattr(v, 'isoformat'):
390 s = v.isoformat()
391 return s[:-9] if s.endswith('T00:00:00') else s
392 return str(v)
393
394
395def _libreoffice(path, target, new_ext, outdir) -> str:
396 soffice = shutil.which('soffice') or shutil.which('libreoffice')
397 if not soffice:
398 raise ConversionError('LibreOffice is not available')
399 profile = tempfile.mkdtemp(prefix='lo-profile-')
400 try:
401 subprocess.run([soffice, f'-env:UserInstallation=file://{profile}', '--headless', '--norestore',
402 '--convert-to', target, '--outdir', outdir, path],
403 check=True, capture_output=True, timeout=180)
404 except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
405 raise ConversionError(f'LibreOffice conversion failed: {e}') from e
406 finally:
407 shutil.rmtree(profile, ignore_errors=True)
408 out = os.path.join(outdir, os.path.splitext(os.path.basename(path))[0] + new_ext)
409 if not os.path.exists(out):
410 raise ConversionError('LibreOffice produced no output')
411 return out
412
413
414
415
416def extract_tables(md: str) -> list[list[list[str]]]:
417 """Markdown pipe tables -> list of tables (rows of cells), header row first."""
418 tables, cur = [], []
419 for line in md.splitlines() + ['']:
420 s = line.strip()
421 if s.startswith('|') and s.endswith('|') and len(s) > 1:
422 cells = [c.strip().replace('\\|', '|') for c in re.split(r'(?<!\\)\|', s[1:-1])]
423 if not all(re.fullmatch(r':?-{2,}:?', c) for c in cells if c):
424 cur.append(cells)
425 elif cur:
426 if len(cur) >= 2 or (cur and len(cur[0]) >= 2):
427 tables.append(cur)
428 cur = []
429 return tables
430
431
432def chunk_markdown(md: str, size: int) -> list[dict]:
433 """Split on blank lines into chunks of at most ~size characters, each carrying its nearest heading."""
434 chunks, buf, buf_heading, heading = [], [], '', ''
435 for block in re.split(r'\n\s*\n', md):
436 block = block.strip()
437 if not block:
438 continue
439 if block.startswith('#'):
440 heading = block.lstrip('#').strip().split('\n')[0]
441 for piece in (block[i:i + size] for i in range(0, len(block), size)):
442 if buf and len('\n\n'.join(buf + [piece])) > size:
443 chunks.append({'heading': buf_heading, 'text': '\n\n'.join(buf)})
444 buf = []
445 if not buf:
446 buf_heading = heading
447 buf.append(piece)
448 if buf:
449 chunks.append({'heading': buf_heading, 'text': '\n\n'.join(buf)})
450 return [dict(index=i, **c) for i, c in enumerate(chunks)]
