1"""Document -> Markdown conversion core (no Apify dependencies, testable locally).
2
3PDF: pdfplumber text and tables, headings from font size; pages without a text layer go to Tesseract OCR.
4Office/HTML/text: markitdown. Legacy formats (doc, xls, ppt, odt, rtf, ...): LibreOffice first.
5Images: Tesseract OCR.
6"""
7from __future__ import annotations
8
9import os
10import re
11import shutil
12import statistics
13import subprocess
14import tempfile
15from dataclasses import dataclass, field
16
17import numpy as np
18import pdfplumber
19import pypdfium2 as pdfium
20import pytesseract
21from PIL import Image, ImageFilter, ImageOps
22
23PDF_EXT = {'.pdf'}
24IMAGE_EXT = {'.png', '.jpg', '.jpeg', '.tif', '.tiff', '.bmp', '.webp', '.gif'}
25SHEET_EXT = {'.xlsx', '.xlsm'}
26MARKITDOWN_EXT = {'.docx', '.pptx', '.html', '.htm', '.csv', '.json', '.xml', '.txt', '.md', '.epub'}
27
28LIBREOFFICE_TARGET = {
29 '.doc': ('html:XHTML Writer File:UTF8', '.html'), '.odt': ('html:XHTML Writer File:UTF8', '.html'),
30 '.rtf': ('html:XHTML Writer File:UTF8', '.html'), '.wpd': ('html:XHTML Writer File:UTF8', '.html'),
31 '.dot': ('html:XHTML Writer File:UTF8', '.html'), '.dotx': ('html:XHTML Writer File:UTF8', '.html'),
32 '.docm': ('html:XHTML Writer File:UTF8', '.html'),
33 '.xls': ('xlsx', '.xlsx'), '.ods': ('xlsx', '.xlsx'), '.xlsb': ('xlsx', '.xlsx'),
34 '.ppt': ('pptx', '.pptx'), '.odp': ('pptx', '.pptx'), '.pps': ('pptx', '.pptx'), '.ppsx': ('pptx', '.pptx'),
35 '.pptm': ('pptx', '.pptx'),
36}
37SUPPORTED_EXT = PDF_EXT | IMAGE_EXT | SHEET_EXT | MARKITDOWN_EXT | set(LIBREOFFICE_TARGET)
38
39
40MIN_TEXT_CHARS = 25
41OCR_DPI = 300
42
43os.environ.setdefault('OMP_THREAD_LIMIT', '1')
44
45CHARS_PER_PAGE = 3000
46
47
48class ConversionError(Exception):
49 pass
50
51
52@dataclass
53class Result:
54 markdown: str = ''
55 page_count: int = 0
56 ocr_pages: int = 0
57 source_pages: int | None = None
58 metadata: dict = field(default_factory=dict)
59 warnings: list[str] = field(default_factory=list)
60
61
62def detect_ext(path: str, name_hint: str = '', content_type: str = '') -> str:
63 """Extension from the file name or, failing that, from magic bytes / content type."""
64 for candidate in (name_hint, path):
65 ext = os.path.splitext(candidate.split('?')[0])[1].lower()
66 if ext in SUPPORTED_EXT:
67 return ext
68 with open(path, 'rb') as f:
69 head = f.read(8)
70 if head.startswith(b'%PDF'):
71 return '.pdf'
72 if head.startswith(b'\x89PNG'):
73 return '.png'
74 if head.startswith(b'\xff\xd8'):
75 return '.jpg'
76 if head[:4] in (b'II*\x00', b'MM\x00*'):
77 return '.tiff'
78 if head.startswith(b'{\\rtf'):
79 return '.rtf'
80 ct = content_type.split(';')[0].strip().lower()
81 by_type = {
82 'application/pdf': '.pdf', 'text/html': '.html', 'text/plain': '.txt', 'text/csv': '.csv',
83 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
84 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',
85 'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx',
86 'application/msword': '.doc', 'application/vnd.ms-excel': '.xls', 'application/vnd.ms-powerpoint': '.ppt',
87 'application/vnd.oasis.opendocument.text': '.odt', 'application/rtf': '.rtf',
88 }
89 if ct in by_type:
90 return by_type[ct]
91 if head.startswith(b'PK'):
92 return '.docx'
93 raise ConversionError(f'unsupported file type (name {name_hint!r}, content type {content_type!r})')
94
95
96ZIP_EXT = {'.docx', '.docm', '.dotx', '.xlsx', '.xlsm', '.xlsb', '.pptx', '.pptm', '.ppsx', '.odt', '.ods', '.odp',
97 '.epub'}
98OLE_EXT = {'.doc', '.dot', '.xls', '.ppt', '.pps'}
99
100
101def check_signature(path: str, ext: str) -> None:
102 """LibreOffice and markitdown read unreadable binaries as plain text; refuse them instead of charging for junk."""
103 with open(path, 'rb') as f:
104 head = f.read(1024)
105 if ((ext in ZIP_EXT and not head.startswith(b'PK'))
106 or (ext in OLE_EXT and not head.startswith((b'\xd0\xcf\x11\xe0', b'PK')))
107 or (ext == '.rtf' and not head.startswith(b'{\\rtf'))
108 or (ext == '.pdf' and b'%PDF' not in head)):
109 raise ConversionError(f'not a valid {ext} file (damaged or wrong extension)')
110
111
112def convert(path: str, ext: str, *, ocr_mode: str = 'auto', ocr_languages: str = 'eng',
113 max_pages: int = 500, page_budget=None) -> Result:
114 """page_budget(text_pages, ocr_pages) -> bool lets the caller stop before work it cannot charge for."""
115 check_signature(path, ext)
116 if ext in PDF_EXT:
117 return _convert_pdf(path, ocr_mode, ocr_languages, max_pages, page_budget)
118 if ext in IMAGE_EXT:
119 return _convert_image(path, ocr_languages, max_pages)
120 if ext in LIBREOFFICE_TARGET:
121 target, new_ext = LIBREOFFICE_TARGET[ext]
122 with tempfile.TemporaryDirectory() as tmp:
123 converted = _libreoffice(path, target, new_ext, tmp)
124 return _convert_sheet(converted) if new_ext in SHEET_EXT else _convert_markitdown(converted)
125 if ext in SHEET_EXT:
126 return _convert_sheet(path)
127 return _convert_markitdown(path)
128
129
130
131
132def _convert_pdf(path, ocr_mode, langs, max_pages, page_budget) -> Result:
133 res = Result()
134 try:
135 pdf = pdfplumber.open(path)
136 except Exception as e:
137 raise ConversionError(f'cannot open PDF: {e}') from e
138 with pdf:
139 res.source_pages = len(pdf.pages)
140 info = pdf.metadata or {}
141 res.metadata = {k.lower(): str(v) for k, v in info.items() if k in ('Title', 'Author', 'Subject', 'CreationDate') and v}
142 pages = pdf.pages[:max_pages]
143 if len(pdf.pages) > max_pages:
144 res.warnings.append(f'only the first {max_pages} of {len(pdf.pages)} pages were converted')
145 body_size = _body_font_size(pages)
146 parts = []
147 doc = None
148 for n, page in enumerate(pages, 1):
149 text_md = '' if ocr_mode == 'always' else _pdf_page_markdown(page, body_size)
150 need_ocr = ocr_mode == 'always' or (ocr_mode == 'auto' and _looks_scanned(page, text_md))
151 if page_budget and not page_budget(0 if need_ocr else 1, 1 if need_ocr else 0):
152 res.warnings.append(f'stopped at page {n}: maximum charge per run reached')
153 break
154 if need_ocr:
155 if doc is None:
156 doc = pdfium.PdfDocument(path)
157 img = doc[n - 1].render(scale=OCR_DPI / 72).to_pil()
158 text_md = _ocr_image(img, langs)
159 res.ocr_pages += 1
160 else:
161 res.page_count += 1
162 if text_md.strip():
163 parts.append(text_md.strip())
164 if doc is not None:
165 doc.close()
166 res.markdown = '\n\n'.join(parts) + '\n'
167 return res
168
169
170def _looks_scanned(page, text_md: str) -> bool:
171 """No text layer at all, or only a few characters on top of a page-sized image (scanner stamp, OCR'd title)."""
172 if not page.chars:
173 return bool(page.images) or len(page.curves) + len(page.lines) > 20
174 if len(text_md.strip()) >= MIN_TEXT_CHARS:
175 return False
176 page_area = float(page.width * page.height) or 1.0
177 return any(float((im['x1'] - im['x0']) * (im['bottom'] - im['top'])) > 0.5 * page_area for im in page.images)
178
179
180def _body_font_size(pages) -> float:
181 sizes = []
182 for page in pages[:20]:
183 sizes.extend(round(c['size'], 1) for c in page.chars[:4000])
184 return statistics.median(sizes) if sizes else 10.0
185
186
187def _pdf_page_markdown(page, body_size: float) -> str:
188 """Text regions between tables (top to bottom) with tables rendered as Markdown."""
189 try:
190 tables = sorted(page.find_tables(), key=lambda t: t.bbox[1])
191 except Exception:
192 tables = []
193 out, top = [], 0
194 for t in tables:
195 x0, t_top, x1, t_bottom = t.bbox
196 if t_top > top:
197 out.append(_region_text(page, top, t_top, body_size))
198 rows = t.extract()
199 if rows and len(rows) >= 2 and max(len(r) for r in rows) >= 2:
200 out.append(table_to_markdown(rows))
201 else:
202 out.append(_region_text(page, t_top, t_bottom, body_size))
203 top = max(top, t_bottom)
204 if top < page.height:
205 out.append(_region_text(page, top, page.height, body_size))
206 return '\n\n'.join(p for p in out if p.strip())
207
208
209BULLET = re.compile(r'^[•●▪◦\-–*]\s+')
210NUMBERED = re.compile(r'^(\d{1,3})[.)]\s+')
211
212
213def _region_text(page, top, bottom, body_size) -> str:
214 bottom = min(bottom, page.height)
215 if bottom - top < 1:
216 return ''
217 region = page.crop((0, max(0, top), page.width, bottom), strict=False)
218 lines = region.extract_text_lines(return_chars=True, strip=True)
219 md, para = [], []
220
221 def flush():
222 if para:
223 md.append(' '.join(para))
224 para.clear()
225
226 for ln in lines:
227 text = ln['text'].strip()
228 if not text:
229 continue
230 size = statistics.median(c['size'] for c in ln['chars']) if ln['chars'] else body_size
231 if size >= body_size * 1.45 and len(text) < 120:
232 flush()
233 md.append('# ' + text)
234 elif size >= body_size * 1.15 and len(text) < 120:
235 flush()
236 md.append('## ' + text)
237 elif BULLET.match(text):
238 flush()
239 md.append('- ' + BULLET.sub('', text))
240 elif NUMBERED.match(text):
241 flush()
242 md.append(NUMBERED.sub(r'\1. ', text))
243 else:
244 para.append(text)
245 flush()
246 return '\n\n'.join(md)
247
248
249def table_to_markdown(rows) -> str:
250 width = max(len(r) for r in rows)
251 clean = [[_cell(c) for c in list(r) + [''] * (width - len(r))] for r in rows]
252 lines = ['| ' + ' | '.join(clean[0]) + ' |', '| ' + ' | '.join(['---'] * width) + ' |']
253 lines += ['| ' + ' | '.join(r) + ' |' for r in clean[1:]]
254 return '\n'.join(lines)
255
256
257def _cell(c) -> str:
258 return re.sub(r'\s+', ' ', str(c or '')).replace('|', '\\|').strip()
259
260
261
262
263def _ocr_image(img: Image.Image, langs: str) -> str:
264 img = ImageOps.exif_transpose(img).convert('L')
265 grids = []
266 try:
267 if _has_rule(img):
268 straight = _deskew(img)
269 grids = _find_grids(straight)
270 except Exception:
271 grids = []
272 if grids:
273 return _ocr_with_grids(straight, grids, langs)
274 data = _tesseract_data(img, langs)
275 try:
276 if _find_borderless(data):
277 straight = _deskew(img)
278 moved = _to_straight(data, img, straight)
279 grids = _find_borderless(moved)
280 except Exception:
281 grids = []
282 if grids:
283 return _ocr_with_grids(straight, grids, langs, moved)
284 return _fix_lookalikes(_ocr_layout(_data_text(data)), langs)
285
286
287def _tesseract(img: Image.Image, langs: str, psm: int, extra: str = '') -> str:
288 try:
289 return pytesseract.image_to_string(img, lang=langs, config=f'--psm {psm} {extra}'.strip())
290 except pytesseract.TesseractError as e:
291 raise ConversionError(f'OCR failed: {e}') from e
292
293
294def _tesseract_data(img: Image.Image, langs: str) -> dict:
295 try:
296 return pytesseract.image_to_data(img, lang=langs, config='--psm 3', output_type=pytesseract.Output.DICT)
297 except pytesseract.TesseractError as e:
298 raise ConversionError(f'OCR failed: {e}') from e
299
300
301def _data_text(d: dict) -> str:
302 """Words of image_to_data back to text as image_to_string lays it out: a blank line between paragraphs."""
303 out, prev = [], None
304 for i, t in enumerate(d['text']):
305 if d['level'][i] != 5 or not t.strip():
306 continue
307 conf = float(d['conf'][i])
308 if ((len(t.strip()) <= 2 and conf < 10) or (not re.search(r'[^\W_]', t) and conf < 60)
309 or re.fullmatch(r'_+', t.strip())):
310 continue
311 key = (d['block_num'][i], d['par_num'][i], d['line_num'][i])
312 if key != prev and t.startswith('(') and ')' not in t and conf < 75 and not any(
313 ')' in d['text'][j] for j in range(i + 1, len(d['text']))
314 if (d['block_num'][j], d['par_num'][j], d['line_num'][j]) == key):
315 t = t[1:]
316 if prev is not None:
317 out.append('\n\n' if key[:2] != prev[:2] else '\n' if key != prev else ' ')
318 out.append(t)
319 prev = key
320 return ''.join(out) + '\n'
321
322
323
324
325BL_MIN_ROWS = 3
326BL_MIN_COLS = 3
327
328
329def _find_borderless(d: dict) -> list[tuple[list[int], list[int]]]:
330 words = [(d['left'][i], d['top'][i], d['left'][i] + d['width'][i], d['top'][i] + d['height'][i])
331 for i, t in enumerate(d['text'])
332 if d['level'][i] == 5 and re.search(r'\w', t or '') and d['height'][i] > 5]
333 if len(words) < BL_MIN_ROWS * BL_MIN_COLS:
334 return []
335 hmed = statistics.median(w[3] - w[1] for w in words)
336 rows = []
337 for w in sorted(words, key=lambda w: (w[1] + w[3]) / 2):
338 yc = (w[1] + w[3]) / 2
339 if rows and abs(yc - rows[-1][0] / rows[-1][1]) < 0.6 * hmed:
340 rows[-1][0] += yc
341 rows[-1][1] += 1
342 rows[-1][2].append(w)
343 else:
344 rows.append([yc, 1, [w]])
345 gap = 0.9 * hmed
346 lines = []
347 for _, _, ws in rows:
348 ws.sort()
349 segs = [[ws[0][0], ws[0][2]]]
350 for w in ws[1:]:
351 if w[0] - segs[-1][1] > gap:
352 segs.append([w[0], w[2]])
353 else:
354 segs[-1][1] = max(segs[-1][1], w[2])
355 lines.append((min(w[1] for w in ws), max(w[3] for w in ws), segs))
356 tables, run = [], []
357 for ln in lines + [None]:
358 if ln is not None and len(ln[2]) >= BL_MIN_COLS and (not run or ln[0] - run[-1][1] < 2.5 * hmed):
359 run.append(ln)
360 continue
361 if len(run) >= BL_MIN_ROWS:
362 tables.append(run)
363 run = [ln] if ln is not None and len(ln[2]) >= BL_MIN_COLS else []
364 grids = []
365 for run in tables:
366 spans = sorted([a, b] for _, _, segs in run for a, b in segs)
367 cols = [spans[0]]
368 for a, b in spans[1:]:
369 if a - cols[-1][1] > 0.5 * hmed:
370 cols.append([a, b])
371 else:
372 cols[-1][1] = max(cols[-1][1], b)
373 if len(cols) < BL_MIN_COLS:
374 continue
375 pad = int(0.3 * hmed)
376 xs = [cols[0][0] - pad] + [(a[1] + b[0]) // 2 for a, b in zip(cols, cols[1:])] + [cols[-1][1] + pad]
377 ys = ([run[0][0] - pad] + [(a[1] + b[0]) // 2 for a, b in zip(run, run[1:])] + [run[-1][1] + pad])
378 grids.append((ys, xs))
379 return grids
380
381
382
383
384
385GRID_DARK = 170
386GRID_MIN_ROWS = 2
387GRID_MIN_COLS = 2
388
389
390def _has_rule(img: Image.Image) -> bool:
391 """A long horizontal dark run (any table rule; on a page skewed by up to ~3 deg it is still >= w/12)."""
392 small = np.asarray(img.resize((max(1, img.width // 2), max(1, img.height // 2)), Image.BILINEAR)) < GRID_DARK
393 wide = small | np.roll(small, 1, axis=1) | np.roll(small, -1, axis=1)
394 return bool(_runs(wide, max(40, small.shape[1] // 24)).any())
395
396
397def _deskew(img: Image.Image) -> Image.Image:
398 """Rotate by the angle (-3..3 deg, step 0.1) that makes text lines sharpest in the row profile."""
399 small = img.resize((max(1, img.width // 4), max(1, img.height // 4)), Image.BILINEAR)
400 best_angle, best_score = 0.0, -1.0
401 for tenth in range(-30, 31):
402 a = tenth / 10
403 ink = np.asarray(small.rotate(a, resample=Image.BILINEAR, fillcolor=255)) < GRID_DARK
404 prof = ink.sum(axis=1).astype(np.float64)
405 score = float(((prof[1:] - prof[:-1]) ** 2).sum())
406 if score > best_score:
407 best_angle, best_score = a, score
408 if abs(best_angle) < 0.15:
409 return img
410 straight = img.rotate(best_angle, resample=Image.BICUBIC, fillcolor=255)
411 straight.info['deskew_angle'] = best_angle
412 return straight
413
414
415def _runs(mask: np.ndarray, length: int) -> np.ndarray:
416 """Per row of a boolean mask: True where a run of >= length True values ends (window sum test)."""
417 cs = np.cumsum(mask, axis=1, dtype=np.int32)
418 cs = np.concatenate([np.zeros((mask.shape[0], 1), np.int32), cs], axis=1)
419 return (cs[:, length:] - cs[:, :-length]) >= length
420
421
422def _segments(idx: np.ndarray, gap: int = 2) -> list[tuple[int, int]]:
423 """Sorted indices -> (first, last) groups of neighbours."""
424 out = []
425 for i in idx.tolist():
426 if out and i - out[-1][1] <= gap:
427 out[-1] = (out[-1][0], i)
428 else:
429 out.append((i, i))
430 return out
431
432
433def _find_grids(img: Image.Image) -> list[tuple[list[int], list[int]]]:
434 """Ruled tables as (row line y's, column line x's), top to bottom. Empty when there is no full grid."""
435 ink = np.asarray(img) < GRID_DARK
436 h, w = ink.shape
437 wide, tall = ink.copy(), ink.copy()
438 for d in (1, 2):
439 wide |= np.roll(ink, d, axis=1) | np.roll(ink, -d, axis=1)
440 tall |= np.roll(ink, d, axis=0) | np.roll(ink, -d, axis=0)
441 min_h = max(80, w // 12)
442 hrun = _runs(wide, min_h)
443 hlines = []
444 for y0, y1 in _segments(np.flatnonzero(hrun.any(axis=1))):
445 if y1 - y0 > 12:
446 continue
447 xs = np.flatnonzero(hrun[y0:y1 + 1].any(axis=0))
448 hlines.append(((y0 + y1) // 2, int(xs.min()), int(xs.max()) + min_h))
449 grids, used = [], set()
450 for i in sorted(range(len(hlines)), key=lambda k: hlines[k][1] - hlines[k][2]):
451 if i in used:
452 continue
453 y, x0, x1 = hlines[i]
454
455 group = [j for j, (yy, a, b) in enumerate(hlines)
456 if j not in used and a >= x0 - 15 and b <= x1 + 15 and b - a >= 0.6 * (x1 - x0)]
457 rows = sorted(hlines[j][0] for j in group)
458 if len(rows) < GRID_MIN_ROWS + 1:
459 continue
460 top, bottom = rows[0], rows[-1]
461
462 left = max(0, x0 - 15)
463 box = tall[top + 3:bottom - 2, left:min(w, x1 + 15)]
464 if box.shape[0] < 10:
465 continue
466 full = np.flatnonzero(box.mean(axis=0) >= 0.9)
467 cols = [left + (c0 + c1) // 2 for c0, c1 in _segments(full) if c1 - c0 <= 12]
468 if len(cols) < GRID_MIN_COLS + 1 or cols[0] > x0 + 15 or cols[-1] < x1 - 15:
469 continue
470 used.update(group)
471 grids.append((rows, cols))
472 return sorted(grids, key=lambda g: g[0][0])
473
474
475def _to_straight(data: dict, img: Image.Image, straight: Image.Image) -> dict:
476 """Words read on the page as scanned, with boxes moved to where they are on the straightened page: text
477 outside tables keeps Tesseract's own reading (straightening resamples the image and can change a comma)."""
478 angle = straight.info.get('deskew_angle', 0.0) if straight is not img else 0.0
479 if not angle:
480 return data
481 a = np.radians(angle)
482 cx, cy = img.width / 2, img.height / 2
483 out = dict(data)
484 out['left'], out['top'] = [], []
485 for x, y, w, h in zip(data['left'], data['top'], data['width'], data['height']):
486 xc, yc = x + w / 2 - cx, y + h / 2 - cy
487 nx, ny = cx + xc * np.cos(a) + yc * np.sin(a), cy - xc * np.sin(a) + yc * np.cos(a)
488 out['left'].append(int(nx - w / 2))
489 out['top'].append(int(ny - h / 2))
490 return out
491
492
493def _ocr_with_grids(img: Image.Image, grids, langs: str, data: dict | None = None) -> str:
494 """Text above, beside and below each table from one OCR pass of the page; tables cell by cell."""
495 if data is None:
496 data = _tesseract_data(img, langs)
497 boxes = [(cols[0], rows[0], cols[-1], rows[-1]) for rows, cols in grids]
498 segments = [[] for _ in range(len(grids) + 1)]
499 for i, t in enumerate(data['text']):
500 if data['level'][i] != 5 or not t.strip():
501 continue
502 xc, yc = data['left'][i] + data['width'][i] / 2, data['top'][i] + data['height'][i] / 2
503 if any(x0 <= xc <= x1 and y0 <= yc <= y1 for x0, y0, x1, y1 in boxes):
504 continue
505 k = next((n for n, (x0, y0, x1, y1) in enumerate(boxes) if yc <= y1), len(grids))
506 segments[k].append(i)
507 blocks = []
508 for k in range(len(grids) + 1):
509 if segments[k]:
510 text = _ocr_layout(_data_text({key: [v[i] for i in segments[k]] for key, v in data.items()}))
511 if text:
512 blocks.append(text)
513 if k < len(grids):
514 rows, cols = grids[k]
515 crops = [[img.crop((c0 + 5, r0 + 5, c1 - 4, r1 - 4)) for c0, c1 in zip(cols, cols[1:])]
516 for r0, r1 in zip(rows, rows[1:])]
517 table = _read_table(crops, langs)
518 if any(any(c for c in r) for r in table):
519 blocks.append(table_to_markdown(_fix_columns(table)))
520 return _fix_lookalikes(_join_broken('\n\n'.join(blocks)), langs)
521
522
523def _read_sheet(cells: list[Image.Image], langs: str) -> list[tuple[str, bool]]:
524 """Cells in one Tesseract call: a sheet of cells one under another (a call per cell is ~30 times slower).
525 Per cell: its text and whether it looks misread (stray marks, a lone letter, letters in a number)."""
526 pads = [ImageOps.expand(c, border=20, fill=255) for c in cells]
527 sheet = Image.new('L', (max(p.width for p in pads), sum(p.height + 30 for p in pads)), 255)
528 spans, y = [], 0
529 for p in pads:
530 sheet.paste(p, (0, y))
531 spans.append((y, y + p.height))
532 y += p.height + 30
533 try:
534 d = pytesseract.image_to_data(sheet, lang=langs, config='--psm 6 -c preserve_interword_spaces=1',
535 output_type=pytesseract.Output.DICT)
536 except pytesseract.TesseractError as e:
537 raise ConversionError(f'OCR failed: {e}') from e
538 words = [[] for _ in cells]
539 for i, t in enumerate(d['text']):
540 if d['level'][i] != 5 or not t.strip():
541 continue
542 yc = d['top'][i] + d['height'][i] / 2
543 k = next((n for n, (a, b) in enumerate(spans) if a <= yc < b), None)
544 if k is not None:
545 words[k].append((d['left'][i], t.strip()))
546 out = []
547 for ws in words:
548 ws.sort()
549 kept = [t for _, t in ws if re.search(r'[^\W_]', t)]
550 text = ' '.join(kept).strip(' |_:;`\'"(')
551 bad = (len(kept) != len(ws) or (len(text) == 1 and not text.isdigit())
552 or bool(re.search(r'\d', text) and re.search(r'[^\d\s.,\-]', text)
553 and not re.search(r'[^\W\d_]{2}', text))
554 or bool(re.search(r'\d[.,;:]$|^[^\w-]|^\d{4,}(?:[.,]\d+)?$', text)))
555 out.append((text, bad))
556 return out
557
558
559def _read_table(crops, langs: str) -> list[list[str]]:
560 """All cells in one call; cells that look misread are read again in one call with scan specks filtered
561 out, and only those still in doubt one by one."""
562 flat = [c for row in crops for c in row]
563 first = _read_sheet(flat, langs)
564 texts = [t for t, _ in first]
565 doubt = [k for k, (_, bad) in enumerate(first) if bad]
566 if doubt:
567 again = _read_sheet([flat[k].filter(ImageFilter.MedianFilter(3)) for k in doubt], langs)
568 still = []
569 for k, (text, bad) in zip(doubt, again):
570 old = texts[k]
571 same = re.sub(r'\s', '', text) == re.sub(r'\s', '', old)
572 if text and (not bad or (same and text.count(' ') > old.count(' '))):
573 texts[k] = text
574 else:
575 still.append(k)
576 for k in still[:6]:
577 texts[k] = _read_cell(flat[k], langs) or texts[k]
578 n = len(crops[0])
579 table = [texts[r * n:(r + 1) * n] for r in range(len(crops))]
580 _even_thousands(table, crops, langs)
581 for r, row in enumerate(table):
582 for c, cell in enumerate(row):
583 row[c] = _gap_thousands(crops[r][c], cell)
584 _cyrillic_cells(table, crops, langs)
585 return table
586
587
588def _gap_thousands(cell: Image.Image, text: str) -> str:
589 """'2500' where the image shows '2 500': Tesseract can drop a narrow thousands gap. The glyphs are measured:
590 when every thousands boundary is a gap at least twice as wide as the gaps between the other characters,
591 the spaces go back. Anything that does not split into exactly one glyph per character stays as read."""
592 m = re.fullmatch(r'(\d{4,})([.,]\d+)?', text)
593 if not m:
594 return text
595 ink = (np.asarray(cell.filter(ImageFilter.MedianFilter(3))) < 140).any(axis=0)
596 runs, x = [], 0
597 while x < len(ink):
598 if ink[x]:
599 start = x
600 while x < len(ink) and ink[x]:
601 x += 1
602 runs.append((start, x))
603 else:
604 x += 1
605 if len(runs) != len(text):
606 return text
607 gaps = [b[0] - a[1] for a, b in zip(runs, runs[1:])]
608 n = len(m.group(1))
609 marks = [i for i in range(1, n) if (n - i) % 3 == 0]
610 others = [g for i, g in enumerate(gaps, 1) if i not in marks]
611 if not others:
612 return text
613 usual = statistics.median(others)
614 if all(gaps[i - 1] >= max(2 * usual, usual + 6) for i in marks):
615 digits = m.group(1)
616 grouped = ''.join((' ' if i in marks else '') + ch for i, ch in enumerate(digits))
617 return grouped + (m.group(2) or '')
618 return text
619
620
621_GROUPED = re.compile(r'\d{1,3}(?:([ .,])\d{3})+(?:([.,])\d+)?')
622
623
624def _even_thousands(table, crops, langs: str) -> None:
625 """In a column where numbers group thousands with a space ('74 160,00'), a number grouped with a dot
626 ('40.530,00') is most likely a misread gap: read again, and when the digits stay the same, group it like
627 the rest of the column. A column that really uses dots is left alone."""
628 for c in range(len(table[0])):
629 styles = {}
630 for r in range(1, len(table)):
631 m = _GROUPED.fullmatch(table[r][c]) if c < len(table[r]) else None
632 if m:
633 styles[r] = (m.group(1), m.group(2))
634 if len(styles) < 3:
635 continue
636 seps = [g for g, _ in styles.values()]
637 main = max(set(seps), key=seps.count)
638 if main != ' ' or seps.count(main) * 2 <= len(seps):
639 continue
640 for r, (g, dec) in styles.items():
641 if g == main or g == dec:
642 continue
643 old = table[r][c]
644 again = _read_cell(crops[r][c].filter(ImageFilter.MedianFilter(3)), langs)
645 same = re.sub(r'\D', '', again) == re.sub(r'\D', '', old)
646 if same and _GROUPED.fullmatch(again) and _GROUPED.fullmatch(again).group(1) == main:
647 table[r][c] = again
648 else:
649 head, _, tail = old.rpartition(dec) if dec else (old, '', '')
650 table[r][c] = head.replace(g, main) + (dec + tail if dec else '')
651
652
653def _read_cell(cell: Image.Image, langs: str) -> str:
654 """A cell as one text line. Scan specks read as stray marks ('З -'): read again with specks filtered out
655 (not always: the filter can close the thousands gap in '2 500')."""
656 def read(im):
657 raw = _tesseract(ImageOps.expand(im, border=20, fill=255), langs, 7, '-c preserve_interword_spaces=1')
658 return re.sub(r'\s+', ' ', raw).strip(), _clean_cell(raw)
659 raw, text = read(cell)
660 if raw != text or (len(text) == 1 and not text.isdigit()):
661 _, again = read(cell.filter(ImageFilter.MedianFilter(3)))
662 if again and re.sub(r'\s', '', again) != re.sub(r'\s', '', text):
663 return again
664 if again and again.count(' ') > text.count(' '):
665 return again
666 return text
667
668
669def _cyrillic_cells(table, crops, langs: str) -> None:
670 """With Latin and Cyrillic languages together, a short cell of a Cyrillic table is often read as Latin
671 ('шт.' -> 'LUT.', 'М500' -> 'M500'). Such a cell is read again with the Cyrillic languages only and the new
672 reading is kept when its words are words of those languages."""
673 parts = langs.split('+')
674 cyr = [x for x in parts if x in _CYRILLIC_LANGS]
675 if not cyr or not set(parts) & _LATIN_LANGS:
676 return
677 text = ' '.join(c for r in table for c in r)
678 n_cyr, n_lat = len(re.findall(r'[\u0400-\u04ff]', text)), len(re.findall(r'[A-Za-z]', text))
679 if n_cyr < 1.5 * n_lat:
680 return
681 words = [_wordlist(x) for x in cyr]
682 latin = [(r, c) for r, row in enumerate(table) for c, cell in enumerate(row) if re.search(r'[A-Za-z]', cell)]
683 if not latin:
684 return
685 for (r, c), (again, _) in zip(latin, _read_sheet([crops[r][c] for r, c in latin], '+'.join(cyr))):
686 letters = [re.sub(r'[^\w]|\d|_', '', w).lower() for w in again.split()]
687 if again and not re.search(r'[A-Za-z]', again) and all(
688 not w or any(w in d for d in words) for w in letters):
689 table[r][c] = again
690
691
692def _clean_cell(text: str) -> str:
693 """One cell read as a line: parts after a wide gap that hold no letter or digit are scan specks."""
694 parts = [p for p in re.split(r'\s{3,}', text.strip()) if re.search(r'\w', p)]
695 tokens = re.sub(r'\s+', ' ', ' '.join(parts)).split()
696 while tokens and not re.search(r'\w', tokens[0]):
697 tokens.pop(0)
698 while tokens and not re.search(r'\w', tokens[-1]):
699 tokens.pop()
700 return ' '.join(tokens).strip(' |_:;`\'"(')
701
702
703_DIGIT_LOOKALIKE = str.maketrans({'З': '3', 'з': '3', 'О': '0', 'о': '0', 'O': '0', 'o': '0', 'l': '1', 'I': '1',
704 '|': '1', 'б': '6', 'Б': '6'})
705_NUMERO = re.compile(r'^(?:№|N[oо°º2]|М[оеo]|Ne|Nе|N)\.?$')
706
707
708def _fix_columns(table: list[list[str]]) -> list[list[str]]:
709 """In a column of numbers, letters that look like digits are digits ('З' -> '3'); its header read as
710 'Мо' / 'No' is the number sign '№'."""
711 if len(table) < 3:
712 return table
713 width = max(len(r) for r in table)
714 for c in range(width):
715 cells = [r[c] for r in table[1:] if c < len(r) and r[c]]
716 numeric = [x for x in cells if re.fullmatch(r'[\d\s.,\-]+', x)]
717 if not cells or len(numeric) * 2 < len(cells):
718 continue
719 for r in table[1:]:
720 if c < len(r) and r[c]:
721 r[c] = re.sub(r'^[^\w\-]+|[^\w]+$', '', r[c]) or r[c]
722 if c < len(r) and r[c] and r[c] not in numeric:
723 fixed = r[c].translate(_DIGIT_LOOKALIKE)
724 if re.fullmatch(r'[\d\s.,\-]+', fixed) and re.search(r'\d', r[c]):
725 r[c] = fixed
726 elif re.fullmatch(r'[\d\s.,\-]+', fixed) and len(r[c]) <= 2:
727 r[c] = fixed
728 if c < len(table[0]) and _NUMERO.match(table[0][c]) and all(re.fullmatch(r'\d+', x) for x in
729 (r[c] for r in table[1:] if c < len(r) and r[c])):
730 table[0][c] = '№'
731 return table
732
733
734def _table_row(line: str) -> str | None:
735 """A Tesseract line read from a ruled table ('| a | b | c', borders as pipes) -> '| a | b | c |'."""
736 s = line.strip()
737 if s.count('|') < 2:
738 return None
739 cells = [c.strip() for c in s.strip('|').split('|')]
740 if len(cells) < 2 or not any(cells):
741 return None
742 return '| ' + ' | '.join(cells) + ' |'
743
744
745def _ocr_layout(text: str) -> str:
746 """Join lines inside a paragraph, keep blank-line paragraph breaks; table rows stay one per line."""
747 out = []
748 for para in re.split(r'\n\s*\n', text):
749 lines, buf = [], []
750 for line in para.splitlines():
751 row = _table_row(line)
752 if row:
753 if buf:
754 lines.append(('p', ' '.join(buf)))
755 buf = []
756 lines.append(('t', row))
757 elif line.strip():
758 buf.append(line.strip())
759 if buf:
760 lines.append(('p', ' '.join(buf)))
761 for kind, value in lines:
762 if kind == 't' and out and out[-1][0] == 't':
763 out[-1] = ('t', out[-1][1] + '\n' + value)
764 else:
765 out.append((kind, value))
766 blocks = []
767 for kind, value in out:
768 if kind == 't':
769 rows = value.split('\n')
770 if len(rows) < 2:
771 blocks.append(' '.join(c.strip() for c in rows[0].strip('|').split('|') if c.strip()))
772 continue
773 value = _fix_tables('\n'.join(_even_rows(rows)))
774 blocks.append(value)
775 return _join_broken('\n\n'.join(b for b in blocks if b))
776
777
778def _join_broken(text: str) -> str:
779 """Tesseract puts a blank line between lines of one paragraph when line spacing is wide: a block that
780 ends without sentence-final punctuation is joined to the next one when that one starts lowercase."""
781 out = []
782 for block in text.split('\n\n'):
783 if (out and block and not block.startswith('|') and not out[-1].startswith('|')
784 and not re.search(r'[.!?:;…»")\]]$', out[-1]) and block[0].islower()):
785 out[-1] += ' ' + block
786 else:
787 out.append(block)
788 return '\n\n'.join(out)
789
790
791def _even_rows(rows: list[str]) -> list[str]:
792 """Same number of cells in every row. Tesseract may read a stray grid mark as an extra '|'
793 ('| Cement | 420 | | | 96.50'): when a row holds as many filled cells as the fullest row, its empty cells go."""
794 cells = [[c.strip() for c in r.strip().strip('|').split('|')] for r in rows]
795 width = max(sum(1 for c in row if c) for row in cells)
796 out = []
797 for row in cells:
798 if sum(1 for c in row if c) == width:
799 row = [c for c in row if c]
800 row = (row + [''] * width)[:width] if len(row) != width else row
801 out.append('| ' + ' | '.join(row) + ' |')
802 return out
803
804
805
806
807
808_LOOKALIKE = str.maketrans('AaBCcEeHIiKkMmOoPpTtXxYyn', 'АаВСсЕеНІіКкМмОоРрТтХхУуп')
809_CYRILLIC_LANGS = {'ukr', 'rus', 'bel', 'bul', 'srp', 'mkd'}
810_LATIN_LANGS = {'eng', 'deu', 'fra', 'spa', 'ita', 'por', 'pol', 'nld', 'ces'}
811_wordlists: dict[str, set[str]] = {}
812
813
814def _wordlist(lang: str) -> set[str]:
815 """Words of Tesseract's dictionary for a language (lower case), read once from its traineddata."""
816 if lang in _wordlists:
817 return _wordlists[lang]
818 words: set[str] = set()
819 try:
820 tessdata = os.environ.get('TESSDATA_PREFIX') or _tessdata_dir()
821 src = os.path.join(tessdata, f'{lang}.traineddata')
822 with tempfile.TemporaryDirectory() as tmp:
823 base = os.path.join(tmp, lang)
824 subprocess.run(['combine_tessdata', '-e', src, base + '.lstm-word-dawg', base + '.lstm-unicharset'],
825 capture_output=True, timeout=60, check=True)
826 subprocess.run(['dawg2wordlist', base + '.lstm-unicharset', base + '.lstm-word-dawg', base + '.txt'],
827 capture_output=True, timeout=120, check=True)
828 with open(base + '.txt', encoding='utf-8') as f:
829 words = {w.strip().lower() for w in f if w.strip()}
830 except Exception:
831 words = set()
832 _wordlists[lang] = words
833 return words
834
835
836def _tessdata_dir() -> str:
837 out = subprocess.run(['tesseract', '--list-langs'], capture_output=True, text=True, timeout=30)
838 m = re.search(r'"([^"]+)"', out.stdout + out.stderr)
839 return m.group(1) if m else '/usr/share/tesseract-ocr/5/tessdata'
840
841
842def _fix_lookalikes(text: str, langs: str) -> str:
843 parts = set(langs.split('+'))
844 if parts & _CYRILLIC_LANGS:
845 text = re.sub(r'(?<![^\W\d_])(?:Мо|Ме|Мe|Mo|Ne|Nе|М»|М°|N°)(?=\s*\d)', '№', text)
846 cyr = sorted(parts & _CYRILLIC_LANGS)
847 if not cyr or not parts & _LATIN_LANGS:
848 return text
849
850 suspects = {w for w in re.findall(r'[^\W\d_]{3,}', text)
851 if re.search(r'[A-Za-z]', w) and re.fullmatch(r'[\u0400-\u04ff]+', w.translate(_LOOKALIKE))}
852 if not suspects:
853 return text
854 english = _wordlist('eng') if 'eng' in parts else set()
855 dictionaries = [_wordlist(lang) for lang in cyr]
856 fixes = {}
857 for w in suspects:
858 cyr_word = w.translate(_LOOKALIKE)
859 if (w.lower() not in english or not w.isascii()) and any(cyr_word.lower() in d for d in dictionaries):
860 fixes[w] = cyr_word
861 if not fixes:
862 return text
863 pattern = re.compile(r'(?<![^\W\d_])(' + '|'.join(map(re.escape, sorted(fixes, key=len, reverse=True)))
864 + r')(?![^\W\d_])')
865 return pattern.sub(lambda m: fixes[m.group(1)], text)
866
867
868def _convert_image(path, langs, max_pages) -> Result:
869 res = Result()
870 try:
871 img = Image.open(path)
872 except Exception as e:
873 raise ConversionError(f'cannot open image: {e}') from e
874 frames = getattr(img, 'n_frames', 1)
875 res.source_pages = frames
876 parts = []
877 for i in range(min(frames, max_pages)):
878 img.seek(i)
879 parts.append(_ocr_image(img.copy(), langs))
880 res.ocr_pages += 1
881 if frames > max_pages:
882 res.warnings.append(f'only the first {max_pages} of {frames} frames were converted')
883 res.markdown = '\n\n'.join(p for p in parts if p) + '\n'
884 return res
885
886
887
888
889_markitdown = None
890
891
892def _convert_markitdown(path) -> Result:
893 global _markitdown
894 if _markitdown is None:
895 from markitdown import MarkItDown
896 _markitdown = MarkItDown(enable_plugins=False)
897 try:
898 out = _markitdown.convert_local(path)
899 except Exception as e:
900 raise ConversionError(f'cannot convert document: {e}') from e
901 res = Result(markdown=_tidy(out.text_content or '') + '\n')
902 if getattr(out, 'title', None):
903 res.metadata['title'] = out.title
904 res.page_count = _char_pages(res.markdown)
905 return res
906
907
908def _char_pages(md: str) -> int:
909 return max(1, -(-len(md.strip()) // CHARS_PER_PAGE))
910
911
912def _tidy(md: str) -> str:
913 md = re.sub(r'<!--\s*Slide number:\s*(\d+)\s*-->', r'## Slide \1', md)
914 md = re.sub(r'[]', '', md)
915 md = _fix_tables(md)
916 md = re.sub(r'\n{3,}', '\n\n', md)
917 return md.strip()
918
919
920SEPARATOR = re.compile(r'^\|(\s*:?-{3,}:?\s*\|)+$')
921
922
923def _fix_tables(md: str) -> str:
924 """Every pipe table gets a separator row; an empty header row is replaced by the first data row."""
925 out, block = [], []
926
927 def flush():
928 if block:
929 rows = [r for r in block if not SEPARATOR.match(r.strip())]
930 if rows and not re.sub(r'[|\s]', '', rows[0]) and len(rows) > 1:
931 rows = rows[1:]
932 if rows:
933 ncol = rows[0].strip().count('|') - 1
934 out.extend([rows[0], '| ' + ' | '.join(['---'] * max(1, ncol)) + ' |'] + rows[1:])
935 block.clear()
936
937 for line in md.splitlines():
938 s = line.strip()
939 if s.startswith('|') and s.endswith('|') and len(s) > 1:
940 block.append(s)
941 else:
942 flush()
943 out.append(line)
944 flush()
945 return '\n'.join(out)
946
947
948def _convert_sheet(path) -> Result:
949 """XLSX via openpyxl: text cells as written, numbers and dates by the cell's own format."""
950 from openpyxl import load_workbook
951 try:
952 wb = load_workbook(path, read_only=True, data_only=True)
953 except Exception as e:
954 raise ConversionError(f'cannot open spreadsheet: {e}') from e
955 parts = []
956 for ws in wb.worksheets:
957 rows = [[_sheet_value(c) for c in row] for row in ws.iter_rows()]
958 rows = [r for r in rows if any(v for v in r)]
959 if not rows:
960 continue
961 used = [j for j in range(max(len(r) for r in rows)) if any(j < len(r) and r[j] for r in rows)]
962 rows = [[r[j] if j < len(r) else '' for j in used] for r in rows]
963 parts.append(f'## {ws.title}\n\n' + (table_to_markdown(rows) if len(used) > 1 or len(rows) > 1
964 else rows[0][0]))
965 wb.close()
966 md = '\n\n'.join(parts) + '\n'
967 return Result(markdown=md, page_count=_char_pages(md))
968
969
970def _sheet_value(cell) -> str:
971 v = cell.value
972 if v is None:
973 return ''
974 if isinstance(v, bool):
975 return 'TRUE' if v else 'FALSE'
976 if isinstance(v, (int, float)):
977 fmt = (getattr(cell, 'number_format', '') or '').split(';')[0]
978 m = re.search(r'0[.,](0+)', fmt)
979 if m:
980 s = f'{v:,.{len(m.group(1))}f}' if ',' in fmt.split('.')[0] else f'{v:.{len(m.group(1))}f}'
981 elif isinstance(v, float) and v.is_integer():
982 s = str(int(v))
983 else:
984 s = repr(v) if isinstance(v, float) else str(v)
985 return s + '%' if fmt.endswith('%') and m else s
986 if hasattr(v, 'isoformat'):
987 s = v.isoformat()
988 return s[:-9] if s.endswith('T00:00:00') else s
989 return str(v)
990
991
992def _libreoffice(path, target, new_ext, outdir) -> str:
993 soffice = shutil.which('soffice') or shutil.which('libreoffice')
994 if not soffice:
995 raise ConversionError('LibreOffice is not available')
996 profile = tempfile.mkdtemp(prefix='lo-profile-')
997 try:
998 subprocess.run([soffice, f'-env:UserInstallation=file://{profile}', '--headless', '--norestore',
999 '--convert-to', target, '--outdir', outdir, path],
1000 check=True, capture_output=True, timeout=180)
1001 except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e:
1002 raise ConversionError(f'LibreOffice conversion failed: {e}') from e
1003 finally:
1004 shutil.rmtree(profile, ignore_errors=True)
1005 out = os.path.join(outdir, os.path.splitext(os.path.basename(path))[0] + new_ext)
1006 if not os.path.exists(out):
1007 raise ConversionError('LibreOffice produced no output')
1008 return out
1009
1010
1011
1012
1013def extract_tables(md: str) -> list[list[list[str]]]:
1014 """Markdown pipe tables -> list of tables (rows of cells), header row first."""
1015 tables, cur = [], []
1016 for line in md.splitlines() + ['']:
1017 s = line.strip()
1018 if s.startswith('|') and s.endswith('|') and len(s) > 1:
1019 cells = [c.strip().replace('\\|', '|') for c in re.split(r'(?<!\\)\|', s[1:-1])]
1020 if not all(re.fullmatch(r':?-{2,}:?', c) for c in cells if c):
1021 cur.append(cells)
1022 elif cur:
1023 if len(cur) >= 2 or (cur and len(cur[0]) >= 2):
1024 tables.append(cur)
1025 cur = []
1026 return tables
1027
1028
1029def chunk_markdown(md: str, size: int) -> list[dict]:
1030 """Split on blank lines into chunks of at most ~size characters, each carrying its nearest heading."""
1031 chunks, buf, buf_heading, heading = [], [], '', ''
1032 for block in re.split(r'\n\s*\n', md):
1033 block = block.strip()
1034 if not block:
1035 continue
1036 if block.startswith('#'):
1037 heading = block.lstrip('#').strip().split('\n')[0]
1038 for piece in (block[i:i + size] for i in range(0, len(block), size)):
1039 if buf and len('\n\n'.join(buf + [piece])) > size:
1040 chunks.append({'heading': buf_heading, 'text': '\n\n'.join(buf)})
1041 buf = []
1042 if not buf:
1043 buf_heading = heading
1044 buf.append(piece)
1045 if buf:
1046 chunks.append({'heading': buf_heading, 'text': '\n\n'.join(buf)})
1047 return [dict(index=i, **c) for i, c in enumerate(chunks)]