1"""Comprehensive tests for models and processing helpers."""
2
3import json
4import unittest
5
6import tiktoken
7from pathlib import Path
8from types import ModuleType
9from unittest.mock import patch
10
11from pydantic import ValidationError
12
13from src.models import MarkdownChunk, MarkdownSplitterInput, ProcessingMetrics
14from src.processing import (
15 cache_hit,
16 cache_record_key,
17 convert_to_markdown,
18 document_hash,
19 document_source,
20 format_chunks,
21 http_error_message,
22 read_capped,
23 resolve_markdown_text,
24 retry_delay,
25 source_fingerprint,
26 split_config_fingerprint,
27 split_documents,
28 validation_error_message,
29)
30
31
32
33
34
35
36
37class InputModelTextModeTests(unittest.TestCase):
38 """Tests for text-mode validation."""
39
40 def test_valid_text_mode(self) -> None:
41 obj = MarkdownSplitterInput(input_mode="text", markdown_text="# Hello")
42 self.assertEqual(obj.input_mode, "text")
43 self.assertEqual(obj.markdown_text, "# Hello")
44
45 def test_text_mode_requires_markdown_text_missing(self) -> None:
46 with self.assertRaises(ValidationError):
47 MarkdownSplitterInput(input_mode="text", markdown_text=None)
48
49 def test_text_mode_rejects_whitespace_only(self) -> None:
50 with self.assertRaises(ValidationError):
51 MarkdownSplitterInput(input_mode="text", markdown_text=" ")
52
53 def test_text_mode_default_input_mode(self) -> None:
54 obj = MarkdownSplitterInput(markdown_text="# Hello")
55 self.assertEqual(obj.input_mode, "text")
56
57 def test_text_mode_respects_max_input_chars(self) -> None:
58 with self.assertRaises(ValidationError):
59 MarkdownSplitterInput(markdown_text="x" * 11, max_input_chars=10)
60
61 def test_text_mode_accepts_exactly_max_input_chars(self) -> None:
62 obj = MarkdownSplitterInput(markdown_text="x" * 10, max_input_chars=10)
63 self.assertEqual(len(obj.markdown_text), 10)
64
65
66class InputModelFileModeTests(unittest.TestCase):
67 """Tests for file-mode validation."""
68
69 def test_valid_file_mode(self) -> None:
70 obj = MarkdownSplitterInput(
71 input_mode="file",
72 markdown_file="https://example.com/doc.md",
73 )
74 self.assertEqual(obj.input_mode, "file")
75 self.assertEqual(obj.markdown_file, "https://example.com/doc.md")
76
77 def test_file_mode_requires_markdown_file_missing(self) -> None:
78 with self.assertRaises(ValidationError):
79 MarkdownSplitterInput(input_mode="file", markdown_file=None)
80
81 def test_file_mode_rejects_whitespace_only_url(self) -> None:
82 with self.assertRaises(ValidationError):
83 MarkdownSplitterInput(input_mode="file", markdown_file=" ")
84
85
86class InputModeInferenceTests(unittest.TestCase):
87 """input_mode is inferred when omitted, so API callers who send only
88 markdown_text or only markdown_file get exactly what they sent."""
89
90 def test_infers_file_when_only_markdown_file_given(self) -> None:
91 obj = MarkdownSplitterInput(markdown_file="https://example.com/a.pdf")
92 self.assertEqual(obj.input_mode, "file")
93
94 def test_infers_text_when_only_markdown_text_given(self) -> None:
95 obj = MarkdownSplitterInput(markdown_text="# H")
96 self.assertEqual(obj.input_mode, "text")
97
98 def test_blank_markdown_file_counts_as_absent(self) -> None:
99 obj = MarkdownSplitterInput(markdown_text="# H", markdown_file=" ")
100 self.assertEqual(obj.input_mode, "text")
101
102 def test_both_sources_without_mode_is_rejected(self) -> None:
103
104 with self.assertRaisesRegex(ValidationError, "input_mode"):
105 MarkdownSplitterInput(
106 markdown_text="# H", markdown_file="https://example.com/a.pdf"
107 )
108
109 def test_explicit_mode_wins_when_both_sources_given(self) -> None:
110 obj = MarkdownSplitterInput(
111 input_mode="text",
112 markdown_text="# H",
113 markdown_file="https://example.com/a.pdf",
114 )
115 self.assertEqual(obj.input_mode, "text")
116
117 def test_no_source_is_rejected(self) -> None:
118 with self.assertRaisesRegex(ValidationError, "markdown_text or markdown_file"):
119 MarkdownSplitterInput()
120
121
122class InputModelKvsStoreTests(unittest.TestCase):
123 def test_kvs_source_requires_kvs_store(self) -> None:
124 with self.assertRaisesRegex(ValidationError, "kvs_store"):
125 MarkdownSplitterInput(markdown_file="kvs://MY_KEY")
126
127 def test_kvs_source_with_store_is_valid(self) -> None:
128 obj = MarkdownSplitterInput(markdown_file="kvs://MY_KEY", kvs_store="abc123")
129 self.assertEqual(obj.kvs_store, "abc123")
130
131 def test_kvs_store_not_required_for_urls(self) -> None:
132 obj = MarkdownSplitterInput(markdown_file="https://example.com/a.md")
133 self.assertIsNone(obj.kvs_store)
134
135
136class InputModelFieldBoundsTests(unittest.TestCase):
137 """Tests for numeric field boundaries."""
138
139 def test_max_chunk_chars_accepts_positive(self) -> None:
140 obj = MarkdownSplitterInput(markdown_text="# H", max_chunk_chars=1800)
141 self.assertEqual(obj.max_chunk_chars, 1800)
142
143 def test_chunk_size_defaults_to_512_tokens(self) -> None:
144
145
146 obj = MarkdownSplitterInput(markdown_text="# H")
147 self.assertEqual(obj.max_chunk_tokens, 512)
148 self.assertIsNone(obj.max_chunk_chars)
149 self.assertEqual(obj.tokenizer, "cl100k_base")
150
151 def test_max_chunk_tokens_bounds(self) -> None:
152 for bad in (49, 8193):
153 with self.assertRaises(ValidationError, msg=bad):
154 MarkdownSplitterInput(markdown_text="# H", max_chunk_tokens=bad)
155 for good in (50, 8192, None):
156 obj = MarkdownSplitterInput(markdown_text="# H", max_chunk_tokens=good)
157 self.assertEqual(obj.max_chunk_tokens, good)
158
159 def test_tokenizer_choices(self) -> None:
160 obj = MarkdownSplitterInput(markdown_text="# H", tokenizer="o200k_base")
161 self.assertEqual(obj.tokenizer, "o200k_base")
162 with self.assertRaises(ValidationError):
163 MarkdownSplitterInput(markdown_text="# H", tokenizer="gpt2")
164
165 def test_max_chunk_chars_null_disables_cap(self) -> None:
166 obj = MarkdownSplitterInput(markdown_text="# H", max_chunk_chars=None)
167 self.assertIsNone(obj.max_chunk_chars)
168
169 def test_max_chunk_chars_rejects_below_100(self) -> None:
170
171 with self.assertRaises(ValidationError):
172 MarkdownSplitterInput(markdown_text="# H", max_chunk_chars=99)
173
174 def test_max_chunk_chars_accepts_100(self) -> None:
175 obj = MarkdownSplitterInput(markdown_text="# H", max_chunk_chars=100)
176 self.assertEqual(obj.max_chunk_chars, 100)
177
178 def test_max_chunk_chars_rejects_zero(self) -> None:
179 with self.assertRaises(ValidationError):
180 MarkdownSplitterInput(markdown_text="# H", max_chunk_chars=0)
181
182 def test_max_chunk_chars_rejects_negative(self) -> None:
183 with self.assertRaises(ValidationError):
184 MarkdownSplitterInput(markdown_text="# H", max_chunk_chars=-1)
185
186 def test_max_chunk_chars_accepts_max_boundary(self) -> None:
187 obj = MarkdownSplitterInput(markdown_text="# H", max_chunk_chars=100_000)
188 self.assertEqual(obj.max_chunk_chars, 100_000)
189
190 def test_max_chunk_chars_rejects_above_max(self) -> None:
191 with self.assertRaises(ValidationError):
192 MarkdownSplitterInput(markdown_text="# H", max_chunk_chars=100_001)
193
194 def test_max_chunks_default(self) -> None:
195 obj = MarkdownSplitterInput(markdown_text="# H")
196 self.assertEqual(obj.max_chunks, 5_000)
197
198 def test_max_chunks_rejects_zero(self) -> None:
199 with self.assertRaises(ValidationError):
200 MarkdownSplitterInput(markdown_text="# H", max_chunks=0)
201
202 def test_max_input_chars_rejects_zero(self) -> None:
203 with self.assertRaises(ValidationError):
204 MarkdownSplitterInput(markdown_text="# H", max_input_chars=0)
205
206 def test_max_input_chars_rejects_above_ceiling(self) -> None:
207 with self.assertRaises(ValidationError):
208 MarkdownSplitterInput(markdown_text="# H", max_input_chars=5_000_001)
209
210 def test_max_input_chars_accepts_ceiling(self) -> None:
211 obj = MarkdownSplitterInput(markdown_text="# H", max_input_chars=5_000_000)
212 self.assertEqual(obj.max_input_chars, 5_000_000)
213
214 def test_max_chunks_accepts_max_boundary(self) -> None:
215 obj = MarkdownSplitterInput(markdown_text="# H", max_chunks=50_000)
216 self.assertEqual(obj.max_chunks, 50_000)
217
218 def test_max_chunks_rejects_above_max(self) -> None:
219 with self.assertRaises(ValidationError):
220 MarkdownSplitterInput(markdown_text="# H", max_chunks=50_001)
221
222 def test_invalid_input_mode_rejected(self) -> None:
223 with self.assertRaises(ValidationError):
224 MarkdownSplitterInput(input_mode="telepathy", markdown_text="# H")
225
226
227class InputModelHeaderValidatorTests(unittest.TestCase):
228 """Tests for headers_to_split_on validation."""
229
230 def test_valid_single_level(self) -> None:
231 obj = MarkdownSplitterInput(markdown_text="# H", headers_to_split_on=["#"])
232 self.assertEqual(obj.headers_to_split_on, ["#"])
233
234 def test_valid_all_six_levels(self) -> None:
235 obj = MarkdownSplitterInput(
236 markdown_text="# H",
237 headers_to_split_on=["#", "##", "###", "####", "#####", "######"],
238 )
239 self.assertEqual(len(obj.headers_to_split_on), 6)
240
241 def test_rejects_more_than_six_levels(self) -> None:
242 with self.assertRaises(ValidationError):
243 MarkdownSplitterInput(
244 markdown_text="# H",
245 headers_to_split_on=["#", "##", "###", "####", "#####", "######", "#######"],
246 )
247
248 def test_rejects_invalid_header_format(self) -> None:
249 with self.assertRaises(ValidationError):
250 MarkdownSplitterInput(markdown_text="# H", headers_to_split_on=["h1"])
251
252 def test_rejects_header_with_text(self) -> None:
253 with self.assertRaises(ValidationError):
254 MarkdownSplitterInput(markdown_text="# H", headers_to_split_on=["# Title"])
255
256 def test_default_headers_are_three_levels(self) -> None:
257
258 obj = MarkdownSplitterInput(markdown_text="# H")
259 self.assertEqual(obj.headers_to_split_on, ["#", "##", "###"])
260
261 def test_headers_are_normalised(self) -> None:
262 obj = MarkdownSplitterInput(
263 markdown_text="# H", headers_to_split_on=[" ## ", " # "]
264 )
265 self.assertEqual(obj.headers_to_split_on, ["##", "#"])
266
267 def test_rejects_seven_hashes(self) -> None:
268
269
270 with self.assertRaises(ValidationError):
271 MarkdownSplitterInput(markdown_text="# H", headers_to_split_on=["#######"])
272
273 def test_empty_header_list_allowed(self) -> None:
274
275
276 obj = MarkdownSplitterInput(markdown_text="# H", headers_to_split_on=[])
277 self.assertEqual(obj.headers_to_split_on, [])
278
279
280class InputModelStripHeadersTests(unittest.TestCase):
281 def test_strip_headers_defaults_true(self) -> None:
282 obj = MarkdownSplitterInput(markdown_text="# H")
283 self.assertTrue(obj.strip_headers)
284
285 def test_strip_headers_can_be_false(self) -> None:
286 obj = MarkdownSplitterInput(markdown_text="# H", strip_headers=False)
287 self.assertFalse(obj.strip_headers)
288
289
290class InputModelProxyTests(unittest.TestCase):
291 """Tests for proxy selection in file mode."""
292
293 def test_use_apify_proxy_defaults_false(self) -> None:
294 obj = MarkdownSplitterInput(markdown_text="# H")
295 self.assertFalse(obj.use_apify_proxy)
296
297 def test_proxy_url_defaults_none(self) -> None:
298 obj = MarkdownSplitterInput(markdown_text="# H")
299 self.assertIsNone(obj.proxy_url)
300
301 def test_proxy_url_accepts_http(self) -> None:
302 obj = MarkdownSplitterInput(
303 input_mode="file",
304 markdown_file="https://example.com/doc.pdf",
305 proxy_url="http://proxy.example.com:8080",
306 )
307 self.assertEqual(obj.proxy_url, "http://proxy.example.com:8080")
308
309 def test_proxy_url_accepts_https_and_socks5(self) -> None:
310 for url in ("https://user:pass@proxy.example.com:8080", "socks5://127.0.0.1:1080"):
311 obj = MarkdownSplitterInput(
312 input_mode="file", markdown_file="https://example.com/doc.pdf", proxy_url=url
313 )
314 self.assertEqual(obj.proxy_url, url)
315
316 def test_proxy_url_rejects_unsupported_scheme(self) -> None:
317 with self.assertRaises(ValidationError):
318 MarkdownSplitterInput(
319 input_mode="file",
320 markdown_file="https://example.com/doc.pdf",
321 proxy_url="ftp://proxy.example.com:21",
322 )
323
324 def test_proxy_url_rejects_garbage(self) -> None:
325 with self.assertRaises(ValidationError):
326 MarkdownSplitterInput(
327 input_mode="file",
328 markdown_file="https://example.com/doc.pdf",
329 proxy_url="not a url",
330 )
331
332 def test_both_proxy_sources_rejected(self) -> None:
333 with self.assertRaises(ValidationError):
334 MarkdownSplitterInput(
335 input_mode="file",
336 markdown_file="https://example.com/doc.pdf",
337 use_apify_proxy=True,
338 proxy_url="http://proxy.example.com:8080",
339 )
340
341 def test_apify_proxy_alone_is_valid(self) -> None:
342 obj = MarkdownSplitterInput(
343 input_mode="file",
344 markdown_file="https://example.com/doc.pdf",
345 use_apify_proxy=True,
346 )
347 self.assertTrue(obj.use_apify_proxy)
348
349 def test_proxy_url_normalises_uppercase_scheme(self) -> None:
350
351
352 obj = MarkdownSplitterInput(
353 input_mode="file",
354 markdown_file="https://example.com/doc.pdf",
355 proxy_url="HTTP://proxy.example.com:8080",
356 )
357 self.assertEqual(obj.proxy_url, "http://proxy.example.com:8080")
358
359 def test_proxy_url_returned_stripped(self) -> None:
360 obj = MarkdownSplitterInput(
361 input_mode="file",
362 markdown_file="https://example.com/doc.pdf",
363 proxy_url=" http://proxy.example.com:8080 ",
364 )
365 self.assertEqual(obj.proxy_url, "http://proxy.example.com:8080")
366
367 def test_empty_proxy_url_means_no_proxy(self) -> None:
368 for blank in ("", " "):
369 obj = MarkdownSplitterInput(
370 markdown_file="https://example.com/a.pdf", proxy_url=blank
371 )
372 self.assertIsNone(obj.proxy_url)
373
374 def test_empty_proxy_url_does_not_conflict_with_apify_proxy(self) -> None:
375 obj = MarkdownSplitterInput(
376 markdown_file="https://example.com/a.pdf", proxy_url="", use_apify_proxy=True
377 )
378 self.assertTrue(obj.use_apify_proxy)
379
380 def test_invalid_proxy_error_does_not_leak_credentials(self) -> None:
381
382 with self.assertRaises(ValidationError) as ctx:
383 MarkdownSplitterInput(
384 markdown_file="https://example.com/a.pdf",
385 proxy_url="ftp://user:s3cret@proxy.example.com:21",
386 )
387 self.assertNotIn("s3cret", str(ctx.exception))
388
389 def test_proxy_url_accepted_in_text_mode(self) -> None:
390
391
392 obj = MarkdownSplitterInput(
393 markdown_text="# H", proxy_url="http://proxy.example.com:8080"
394 )
395 self.assertEqual(obj.proxy_url, "http://proxy.example.com:8080")
396
397 def test_both_proxy_sources_error_names_the_fields(self) -> None:
398 with self.assertRaisesRegex(ValidationError, "use_apify_proxy or proxy_url"):
399 MarkdownSplitterInput(
400 input_mode="file",
401 markdown_file="https://example.com/doc.pdf",
402 use_apify_proxy=True,
403 proxy_url="http://proxy.example.com:8080",
404 )
405
406 def test_text_mode_required_error_is_actionable(self) -> None:
407 with self.assertRaisesRegex(ValidationError, "markdown_text is required"):
408 MarkdownSplitterInput(input_mode="text", markdown_text=None)
409
410 def test_file_mode_required_error_is_actionable(self) -> None:
411 with self.assertRaisesRegex(ValidationError, "markdown_file is required"):
412 MarkdownSplitterInput(input_mode="file", markdown_file=None)
413
414
415
416
417
418
419
420class MarkdownChunkModelTests(unittest.TestCase):
421 def test_valid_chunk(self) -> None:
422 chunk = MarkdownChunk(
423 content="hello",
424 metadata={"Header 1": "Intro"},
425 chunk_id="abc123",
426 char_count=5,
427 token_count=2,
428 )
429 self.assertEqual(chunk.content, "hello")
430 self.assertEqual(chunk.char_count, 5)
431
432 def test_char_count_rejects_negative(self) -> None:
433 with self.assertRaises(ValidationError):
434 MarkdownChunk(content="x", metadata={}, chunk_id="id", char_count=-1, token_count=0)
435
436 def test_token_count_rejects_negative(self) -> None:
437 with self.assertRaises(ValidationError):
438 MarkdownChunk(content="x", metadata={}, chunk_id="id", char_count=1, token_count=-1)
439
440 def test_empty_metadata_allowed(self) -> None:
441 chunk = MarkdownChunk(content="x", metadata={}, chunk_id="id", char_count=1, token_count=1)
442 self.assertEqual(chunk.metadata, {})
443
444
445
446
447
448
449
450class ProcessingMetricsTests(unittest.TestCase):
451 def test_valid_metrics(self) -> None:
452 m = ProcessingMetrics(
453 input_mode="text",
454 input_file_type="text/markdown",
455 input_chars=100,
456 chunk_count=5,
457 elapsed_ms=42,
458 document_hash="abc123",
459 )
460 self.assertEqual(m.input_mode, "text")
461 self.assertEqual(m.input_file_type, "text/markdown")
462
463 def test_input_file_type_defaults(self) -> None:
464 m = ProcessingMetrics(
465 input_mode="text",
466 input_chars=0,
467 chunk_count=0,
468 elapsed_ms=0,
469 document_hash="a",
470 )
471 self.assertEqual(m.input_file_type, "text/markdown")
472
473 def test_rejects_negative_input_chars(self) -> None:
474 with self.assertRaises(ValidationError):
475 ProcessingMetrics(
476 input_mode="text",
477 input_chars=-1,
478 chunk_count=0,
479 elapsed_ms=0,
480 )
481
482 def test_rejects_negative_chunk_count(self) -> None:
483 with self.assertRaises(ValidationError):
484 ProcessingMetrics(
485 input_mode="text",
486 input_chars=0,
487 chunk_count=-1,
488 elapsed_ms=0,
489 )
490
491 def test_rejects_negative_elapsed_ms(self) -> None:
492 with self.assertRaises(ValidationError):
493 ProcessingMetrics(
494 input_mode="text",
495 input_chars=0,
496 chunk_count=0,
497 elapsed_ms=-1,
498 document_hash="a",
499 )
500
501 def test_document_hash_required(self) -> None:
502 with self.assertRaises(ValidationError):
503 ProcessingMetrics(
504 input_mode="text",
505 input_chars=0,
506 chunk_count=0,
507 elapsed_ms=0,
508 )
509
510 def test_unchanged_defaults_false_and_cached_chunk_count_optional(self) -> None:
511 m = ProcessingMetrics(
512 input_mode="text",
513 input_chars=0,
514 chunk_count=0,
515 elapsed_ms=0,
516 document_hash="a",
517 )
518 self.assertFalse(m.unchanged)
519 self.assertIsNone(m.cached_chunk_count)
520
521
522class ProcessingMetricsChargeLimitTests(unittest.TestCase):
523 def test_charge_limit_reached_defaults_false(self) -> None:
524 m = ProcessingMetrics(
525 input_mode="text", input_chars=1, chunk_count=1, elapsed_ms=0, document_hash="a"
526 )
527 self.assertFalse(m.charge_limit_reached)
528
529 def test_charge_limit_reached_can_be_reported(self) -> None:
530 m = ProcessingMetrics(
531 input_mode="text",
532 input_chars=1,
533 chunk_count=125,
534 elapsed_ms=0,
535 document_hash="a",
536 charge_limit_reached=True,
537 )
538 self.assertTrue(m.charge_limit_reached)
539
540
541class InputModelSkipUnchangedTests(unittest.TestCase):
542 def test_skip_if_unchanged_defaults_false(self) -> None:
543 obj = MarkdownSplitterInput(markdown_text="# H")
544 self.assertFalse(obj.skip_if_unchanged)
545
546
547class DocumentHashTests(unittest.TestCase):
548 def test_same_text_same_hash(self) -> None:
549 self.assertEqual(document_hash("# Hello"), document_hash("# Hello"))
550
551 def test_different_text_different_hash(self) -> None:
552 self.assertNotEqual(document_hash("# Hello"), document_hash("# World"))
553
554 def test_hash_is_16_hex_chars(self) -> None:
555 h = document_hash("text")
556 self.assertEqual(len(h), 16)
557 int(h, 16)
558
559 def test_empty_text_allowed(self) -> None:
560 self.assertEqual(len(document_hash("")), 16)
561
562
563class SourceFingerprintTests(unittest.TestCase):
564 def test_same_bytes_and_type_same_fingerprint(self) -> None:
565 self.assertEqual(
566 source_fingerprint(b"%PDF-1.4", "application/pdf"),
567 source_fingerprint(b"%PDF-1.4", "application/pdf"),
568 )
569
570 def test_different_bytes_different_fingerprint(self) -> None:
571 self.assertNotEqual(
572 source_fingerprint(b"a", "text/html"), source_fingerprint(b"b", "text/html")
573 )
574
575 def test_content_type_changes_fingerprint(self) -> None:
576
577
578 self.assertNotEqual(
579 source_fingerprint(b"x", "text/html"), source_fingerprint(b"x", "text/plain")
580 )
581
582 def test_type_and_bytes_cannot_collide_by_concatenation(self) -> None:
583 self.assertNotEqual(
584 source_fingerprint(b"b", "text/a"), source_fingerprint(b"ab", "text/")
585 )
586
587 def test_is_16_lowercase_hex_chars(self) -> None:
588 self.assertRegex(source_fingerprint(b"x", "text/html"), r"^[0-9a-f]{16}$")
589
590
591class CacheRecordKeyTests(unittest.TestCase):
592 _KEY_RE = r"^state-[0-9a-f]{24}$"
593
594 def test_file_mode_keyed_by_source(self) -> None:
595 a = MarkdownSplitterInput(markdown_file="https://example.com/a.pdf")
596 b = MarkdownSplitterInput(markdown_file="https://example.com/b.pdf")
597 self.assertRegex(cache_record_key(a, "h1"), self._KEY_RE)
598 self.assertNotEqual(cache_record_key(a, "h1"), cache_record_key(b, "h1"))
599
600 self.assertEqual(cache_record_key(a, "h1"), cache_record_key(a, "h2"))
601
602 def test_file_mode_ignores_surrounding_whitespace(self) -> None:
603 a = MarkdownSplitterInput(markdown_file="https://example.com/a.pdf")
604 b = MarkdownSplitterInput(markdown_file=" https://example.com/a.pdf ")
605 self.assertEqual(cache_record_key(a, "h"), cache_record_key(b, "h"))
606
607 def test_text_mode_keyed_by_document_hash(self) -> None:
608 p = MarkdownSplitterInput(markdown_text="# H")
609 self.assertRegex(cache_record_key(p, "h1"), self._KEY_RE)
610 self.assertNotEqual(cache_record_key(p, "h1"), cache_record_key(p, "h2"))
611
612
613class DocumentSourceTests(unittest.TestCase):
614 def test_url_source(self) -> None:
615 p = MarkdownSplitterInput(markdown_file=" https://a.example/x.pdf ")
616 self.assertEqual(document_source(p), "https://a.example/x.pdf")
617
618 def test_kvs_source_includes_the_store(self) -> None:
619 a = MarkdownSplitterInput(markdown_file="kvs://manual", kvs_store="store-a")
620 b = MarkdownSplitterInput(markdown_file="kvs://manual", kvs_store="store-b")
621 self.assertEqual(document_source(a), "kvs://store-a/manual")
622 self.assertNotEqual(document_source(a), document_source(b))
623
624 def test_pasted_text_has_no_source(self) -> None:
625 self.assertIsNone(document_source(MarkdownSplitterInput(markdown_text="# H")))
626
627 def test_source_id_overrides(self) -> None:
628 for p in (
629 MarkdownSplitterInput(markdown_text="# H", source_id="crawl:page-1"),
630 MarkdownSplitterInput(markdown_file="https://a.example/x.pdf", source_id="crawl:page-1"),
631 ):
632 self.assertEqual(document_source(p), "crawl:page-1")
633
634
635class SourceIdTests(unittest.TestCase):
636 def test_blank_source_id_is_none(self) -> None:
637 self.assertIsNone(MarkdownSplitterInput(markdown_text="# H", source_id=" ").source_id)
638
639 def test_text_with_source_id_keeps_ids_of_untouched_sections(self) -> None:
640
641 def ids(text: str) -> dict[str, str]:
642 p = MarkdownSplitterInput(markdown_text=text, source_id="doc-1")
643 docs = split_documents(p, text)
644 return {
645 c.header_path: c.chunk_id
646 for c in format_chunks(docs, source=document_source(p), document_hash=document_hash(text))
647 }
648 before = ids("# A\nalpha\n# B\nbeta")
649 after = ids("# A\nalpha\n# B\nbeta (typo fixed)")
650 self.assertEqual(before["A"], after["A"])
651 self.assertNotEqual(before["B"], after["B"])
652
653 def test_text_cache_key_follows_source_id(self) -> None:
654 a = MarkdownSplitterInput(markdown_text="# v1", source_id="doc-1")
655 b = MarkdownSplitterInput(markdown_text="# v2", source_id="doc-1")
656 self.assertEqual(cache_record_key(a, "h1"), cache_record_key(b, "h2"))
657
658
659class RetryDelayTests(unittest.TestCase):
660 def test_transport_errors_are_retried(self) -> None:
661 self.assertEqual(retry_delay(None, None, attempt=0), 1.0)
662 self.assertIsNone(retry_delay(None, None, attempt=2))
663
664 def test_non_finite_or_negative_retry_after_falls_back_to_backoff(self) -> None:
665 for bad in ("nan", "inf", "-5"):
666 self.assertEqual(retry_delay(503, bad, attempt=0), 1.0, bad)
667
668 def test_success_and_client_errors_are_not_retried(self) -> None:
669 for status in (200, 301, 400, 403, 404, 410):
670 self.assertIsNone(retry_delay(status, None, attempt=0), status)
671
672 def test_rate_limit_and_server_errors_are_retried_with_backoff(self) -> None:
673 for status in (429, 500, 502, 503, 504):
674 self.assertEqual(retry_delay(status, None, attempt=0), 1.0, status)
675 self.assertEqual(retry_delay(503, None, attempt=1), 2.0)
676
677 def test_gives_up_after_max_attempts(self) -> None:
678 self.assertIsNone(retry_delay(503, None, attempt=2))
679
680 def test_honours_retry_after_seconds_within_cap(self) -> None:
681 self.assertEqual(retry_delay(429, "3", attempt=0), 3.0)
682
683 def test_caps_long_retry_after(self) -> None:
684
685 self.assertEqual(retry_delay(429, "120", attempt=0), 5.0)
686
687 def test_ignores_unparseable_retry_after(self) -> None:
688 self.assertEqual(retry_delay(429, "Wed, 21 Oct 2026 07:28:00 GMT", attempt=0), 1.0)
689
690
691class ReadCappedTests(unittest.IsolatedAsyncioTestCase):
692 @staticmethod
693 async def _parts(*parts: bytes):
694 for part in parts:
695 yield part
696
697 async def test_joins_parts_under_limit(self) -> None:
698 self.assertEqual(await read_capped(self._parts(b"ab", b"cd"), limit=10), b"abcd")
699
700 async def test_exactly_at_limit_passes(self) -> None:
701 self.assertEqual(await read_capped(self._parts(b"ab", b"cd"), limit=4), b"abcd")
702
703 async def test_over_limit_raises_without_reading_everything(self) -> None:
704 consumed: list[bytes] = []
705
706 async def parts():
707 for part in (b"abc", b"def", b"ghi"):
708 consumed.append(part)
709 yield part
710
711 with self.assertRaisesRegex(ValueError, "larger than"):
712 await read_capped(parts(), limit=5)
713 self.assertEqual(consumed, [b"abc", b"def"])
714
715
716class CacheHitTests(unittest.TestCase):
717 _STATE = {"document_hash": "d1", "source_hash": "s1", "split_config": "c1"}
718
719 def test_document_hash_match_is_a_hit(self) -> None:
720 self.assertTrue(cache_hit(self._STATE, "c1", document_hash="d1"))
721
722 def test_source_hash_match_is_a_hit(self) -> None:
723 self.assertTrue(cache_hit(self._STATE, "c1", source_hash="s1"))
724
725 def test_split_config_mismatch_is_a_miss(self) -> None:
726 self.assertFalse(cache_hit(self._STATE, "c2", document_hash="d1"))
727 self.assertFalse(cache_hit(self._STATE, "c2", source_hash="s1"))
728
729 def test_hash_mismatch_is_a_miss(self) -> None:
730 self.assertFalse(cache_hit(self._STATE, "c1", document_hash="other"))
731 self.assertFalse(cache_hit(self._STATE, "c1", source_hash="other"))
732
733 def test_no_hash_given_is_a_miss(self) -> None:
734 self.assertFalse(cache_hit(self._STATE, "c1"))
735
736 def test_state_without_source_hash_never_matches_source(self) -> None:
737
738
739 legacy = {"document_hash": "d1", "split_config": "c1"}
740 self.assertFalse(cache_hit(legacy, "c1", source_hash=None))
741 self.assertFalse(cache_hit(legacy, "c1", source_hash="s1"))
742
743 def test_non_dict_state_is_a_miss(self) -> None:
744 for state in (None, "garbage", 42, ["d1"]):
745 self.assertFalse(cache_hit(state, "c1", document_hash="d1"))
746
747
748class InputModelDefaultsTests(unittest.TestCase):
749 def test_max_input_chars_default_fits_a_long_manual(self) -> None:
750
751 self.assertEqual(MarkdownSplitterInput(markdown_text="# H").max_input_chars, 2_000_000)
752
753 def test_cache_namespace_defaults_empty(self) -> None:
754 self.assertEqual(MarkdownSplitterInput(markdown_text="# H").cache_namespace, "")
755
756
757class CacheNamespaceTests(unittest.TestCase):
758 def test_namespace_separates_pipelines(self) -> None:
759 dev = MarkdownSplitterInput(markdown_file="https://a.example/x.pdf", cache_namespace="dev")
760 prod = MarkdownSplitterInput(markdown_file="https://a.example/x.pdf", cache_namespace="prod")
761 self.assertNotEqual(cache_record_key(dev, ""), cache_record_key(prod, ""))
762
763 def test_kvs_store_is_part_of_the_key(self) -> None:
764 a = MarkdownSplitterInput(markdown_file="kvs://doc", kvs_store="store-a")
765 b = MarkdownSplitterInput(markdown_file="kvs://doc", kvs_store="store-b")
766 self.assertNotEqual(cache_record_key(a, ""), cache_record_key(b, ""))
767
768
769class ErrorMessageTests(unittest.TestCase):
770 def test_blocked_suggests_proxy(self) -> None:
771 for status in (401, 403, 429):
772 msg = http_error_message(status, "https://a.example/x.pdf")
773 self.assertIn(str(status), msg)
774 self.assertIn("use_apify_proxy", msg)
775
776 def test_not_found_names_the_url(self) -> None:
777 msg = http_error_message(404, "https://a.example/x.pdf")
778 self.assertIn("404", msg)
779 self.assertIn("https://a.example/x.pdf", msg)
780 self.assertNotIn("use_apify_proxy", msg)
781
782 def test_validation_errors_are_one_line_per_field(self) -> None:
783 with self.assertRaises(ValidationError) as ctx:
784 MarkdownSplitterInput(markdown_text="# H", max_chunks=0)
785 msg = validation_error_message(ctx.exception)
786 self.assertIn("max_chunks", msg)
787 self.assertNotIn("pydantic.dev", msg)
788 self.assertNotIn("\n", msg)
789
790 def test_model_level_errors_have_no_field_prefix(self) -> None:
791 with self.assertRaises(ValidationError) as ctx:
792 MarkdownSplitterInput()
793 self.assertEqual(
794 validation_error_message(ctx.exception),
795 "Invalid input: Provide markdown_text or markdown_file.",
796 )
797
798
799class SplitConfigFingerprintTests(unittest.TestCase):
800 def _params(self, **kw) -> MarkdownSplitterInput:
801 base = dict(markdown_text="# H")
802 base.update(kw)
803 return MarkdownSplitterInput(**base)
804
805 def test_same_config_same_fingerprint(self) -> None:
806 self.assertEqual(
807 split_config_fingerprint(self._params()),
808 split_config_fingerprint(self._params()),
809 )
810
811 def test_headers_change_fingerprint(self) -> None:
812 self.assertNotEqual(
813 split_config_fingerprint(self._params()),
814 split_config_fingerprint(self._params(headers_to_split_on=["#"])),
815 )
816
817 def test_char_cap_makes_token_cap_irrelevant(self) -> None:
818 a = split_config_fingerprint(self._params(max_chunk_chars=2000, max_chunk_tokens=512))
819 b = split_config_fingerprint(self._params(max_chunk_chars=2000, max_chunk_tokens=None))
820 self.assertEqual(a, b)
821
822 def test_token_settings_change_fingerprint(self) -> None:
823 base = split_config_fingerprint(self._params())
824 self.assertNotEqual(base, split_config_fingerprint(self._params(max_chunk_tokens=256)))
825 self.assertNotEqual(base, split_config_fingerprint(self._params(tokenizer="o200k_base")))
826
827 def test_max_chunk_chars_change_fingerprint(self) -> None:
828 self.assertNotEqual(
829 split_config_fingerprint(self._params()),
830 split_config_fingerprint(self._params(max_chunk_chars=1800)),
831 )
832
833 def test_strip_headers_change_fingerprint(self) -> None:
834 self.assertNotEqual(
835 split_config_fingerprint(self._params()),
836 split_config_fingerprint(self._params(strip_headers=False)),
837 )
838
839 def test_input_text_does_not_change_fingerprint(self) -> None:
840 self.assertEqual(
841 split_config_fingerprint(self._params(markdown_text="# A")),
842 split_config_fingerprint(self._params(markdown_text="# B")),
843 )
844
845 def test_limits_and_flags_do_not_change_fingerprint(self) -> None:
846
847
848 self.assertEqual(
849 split_config_fingerprint(self._params()),
850 split_config_fingerprint(
851 self._params(
852 max_chunks=99,
853 max_input_chars=42,
854 skip_if_unchanged=True,
855 use_apify_proxy=True,
856 )
857 ),
858 )
859
860
861
862
863
864
865
866class FormatChunksTests(unittest.TestCase):
867 class _Doc:
868 def __init__(self, page_content: str, metadata: dict) -> None:
869 self.page_content = page_content
870 self.metadata = metadata
871
872 def test_basic_chunk_metrics(self) -> None:
873 chunks = format_chunks([self._Doc("abc", {"Header 1": "H1"})])
874 self.assertEqual(len(chunks), 1)
875 self.assertEqual(chunks[0].char_count, 3)
876
877 def test_token_count_is_real_tokenizer_count(self) -> None:
878
879 text = "这是一个用于测试的中文句子。"
880 for name in ("cl100k_base", "o200k_base"):
881 chunk = format_chunks([self._Doc(text, {})], tokenizer=name)[0]
882 self.assertEqual(chunk.token_count, len(tiktoken.get_encoding(name).encode(text)))
883
884 def test_embedding_text_prepends_header_path(self) -> None:
885 chunk = format_chunks([self._Doc("Use pip.", {"Header 1": "Guide", "Header 2": "Install"})])[0]
886 self.assertEqual(chunk.embedding_text, "Guide > Install\n\nUse pip.")
887
888 def test_embedding_text_does_not_repeat_kept_headers(self) -> None:
889
890 doc = self._Doc("## Install\nUse pip.", {"Header 1": "Guide", "Header 2": "Install"})
891 chunk = format_chunks([doc], prepend_header_path=False)[0]
892 self.assertEqual(chunk.embedding_text, "## Install\nUse pip.")
893
894 def test_embedding_text_is_content_without_headers(self) -> None:
895 self.assertEqual(format_chunks([self._Doc("Intro text", {})])[0].embedding_text, "Intro text")
896
897 def test_empty_content_produces_zero_counts(self) -> None:
898 chunks = format_chunks([self._Doc("", {})])
899 self.assertEqual(chunks[0].char_count, 0)
900 self.assertEqual(chunks[0].token_count, 0)
901
902 def test_chunk_id_is_nonempty_string(self) -> None:
903 chunks = format_chunks([self._Doc("hello", {})])
904 self.assertIsInstance(chunks[0].chunk_id, str)
905 self.assertTrue(len(chunks[0].chunk_id) > 0)
906
907 def test_chunk_id_is_16_lowercase_hex_chars(self) -> None:
908 chunks = format_chunks([self._Doc("hello", {"Header 1": "World"})])
909 self.assertRegex(chunks[0].chunk_id, r"^[0-9a-f]{16}$")
910
911 def test_chunk_id_independent_of_metadata_key_order(self) -> None:
912 a = format_chunks([self._Doc("same", {"Header 1": "A", "Header 2": "B"})])[0].chunk_id
913 b = format_chunks([self._Doc("same", {"Header 2": "B", "Header 1": "A"})])[0].chunk_id
914 self.assertEqual(a, b)
915
916 def test_chunk_id_deterministic_for_unicode_content(self) -> None:
917 a = format_chunks([self._Doc("héllo ☃ слайд", {})])[0].chunk_id
918 b = format_chunks([self._Doc("héllo ☃ слайд", {})])[0].chunk_id
919 self.assertEqual(a, b)
920
921 def test_chunk_id_is_deterministic(self) -> None:
922 doc = self._Doc("hello", {"Header 1": "World"})
923 id1 = format_chunks([doc])[0].chunk_id
924 id2 = format_chunks([doc])[0].chunk_id
925 self.assertEqual(id1, id2)
926
927 def test_chunk_id_differs_for_different_content(self) -> None:
928 id1 = format_chunks([self._Doc("foo", {})])[0].chunk_id
929 id2 = format_chunks([self._Doc("bar", {})])[0].chunk_id
930 self.assertNotEqual(id1, id2)
931
932 def test_chunk_id_differs_for_different_metadata(self) -> None:
933 id1 = format_chunks([self._Doc("same", {"Header 1": "A"})])[0].chunk_id
934 id2 = format_chunks([self._Doc("same", {"Header 1": "B"})])[0].chunk_id
935 self.assertNotEqual(id1, id2)
936
937 def test_empty_list_returns_empty(self) -> None:
938 self.assertEqual(format_chunks([]), [])
939
940 def test_multiple_chunks_all_have_ids(self) -> None:
941 docs = [self._Doc(f"content {i}", {"Header 1": f"H{i}"}) for i in range(5)]
942 chunks = format_chunks(docs)
943 ids = {c.chunk_id for c in chunks}
944 self.assertEqual(len(ids), 5, "All chunk IDs must be unique")
945
946 def test_identical_chunks_get_distinct_ids(self) -> None:
947
948
949 doc = lambda: self._Doc("TODO", {"Header 1": "API", "Header 2": "Example"})
950 ids = [c.chunk_id for c in format_chunks([doc(), doc(), doc()])]
951 self.assertEqual(len(set(ids)), 3)
952
953 def test_first_occurrence_keeps_its_standalone_id(self) -> None:
954 alone = format_chunks([self._Doc("x", {})])[0].chunk_id
955 first, _second = format_chunks([self._Doc("x", {}), self._Doc("x", {})])
956 self.assertEqual(first.chunk_id, alone)
957
958 def test_identical_chunks_in_different_sources_get_distinct_ids(self) -> None:
959
960
961 doc = self._Doc("MIT", {"Header 2": "License"})
962 a = format_chunks([doc], source="https://a.example/x.pdf")[0].chunk_id
963 b = format_chunks([doc], source="https://b.example/y.pdf")[0].chunk_id
964 self.assertNotEqual(a, b)
965
966 def test_text_mode_ids_are_scoped_by_document_hash(self) -> None:
967 doc = self._Doc("MIT", {"Header 2": "License"})
968 a = format_chunks([doc], document_hash="aaaa")[0].chunk_id
969 b = format_chunks([doc], document_hash="bbbb")[0].chunk_id
970 self.assertNotEqual(a, b)
971
972 def test_ids_stable_for_same_source(self) -> None:
973 doc = self._Doc("MIT", {"Header 2": "License"})
974 a = format_chunks([doc], source="https://a.example/x.pdf", document_hash="h1")[0]
975 b = format_chunks([doc], source="https://a.example/x.pdf", document_hash="h2")[0]
976
977 self.assertEqual(a.chunk_id, b.chunk_id)
978
979 def test_provenance_fields(self) -> None:
980 chunks = format_chunks(
981 [self._Doc("a", {"Header 1": "Guide"}), self._Doc("b", {"Header 1": "Guide"})],
982 source="https://a.example/x.pdf",
983 document_hash="h1",
984 )
985 self.assertEqual([c.chunk_index for c in chunks], [0, 1])
986 self.assertEqual({c.chunk_total for c in chunks}, {2})
987 self.assertEqual({c.source for c in chunks}, {"https://a.example/x.pdf"})
988 self.assertEqual({c.document_hash for c in chunks}, {"h1"})
989
990 def test_header_path_follows_header_level_not_dict_order(self) -> None:
991 chunk = format_chunks(
992 [self._Doc("x", {"Header 3": "Windows", "Header 1": "Guide", "Header 2": "Install"})]
993 )[0]
994 self.assertEqual(chunk.header_path, "Guide > Install > Windows")
995
996 def test_header_path_empty_without_headers(self) -> None:
997 self.assertEqual(format_chunks([self._Doc("x", {})])[0].header_path, "")
998
999 def test_metadata_preserved(self) -> None:
1000 chunks = format_chunks([self._Doc("text", {"Header 1": "Intro", "Header 2": "Sub"})])
1001 self.assertEqual(chunks[0].metadata, {"Header 1": "Intro", "Header 2": "Sub"})
1002
1003 def test_none_page_content_treated_as_empty(self) -> None:
1004 doc = self._Doc.__new__(self._Doc)
1005 doc.page_content = None
1006 doc.metadata = {}
1007 chunks = format_chunks([doc])
1008 self.assertEqual(chunks[0].content, "")
1009 self.assertEqual(chunks[0].char_count, 0)
1010
1011
1012
1013
1014
1015
1016
1017class ResolveMarkdownTextTests(unittest.IsolatedAsyncioTestCase):
1018 async def test_text_mode_returns_text_and_mime(self) -> None:
1019 params = MarkdownSplitterInput(markdown_text="# Heading")
1020 text, mime = await resolve_markdown_text(
1021 params=params,
1022 fetch_remote=lambda _: (b"", "text/plain"),
1023 load_kvs=lambda _: "# kvs",
1024 )
1025 self.assertEqual(text, "# Heading")
1026 self.assertEqual(mime, "text/markdown")
1027
1028 async def test_file_mode_https_calls_converter(self) -> None:
1029 params = MarkdownSplitterInput(
1030 input_mode="file", markdown_file="https://example.com/doc.html"
1031 )
1032 text, mime = await resolve_markdown_text(
1033 params=params,
1034 fetch_remote=lambda _: (b"<h1>Title</h1>", "text/html"),
1035 load_kvs=lambda _: "",
1036 convert_remote_file=lambda _raw, _ct: "# Title",
1037 )
1038 self.assertEqual(text, "# Title")
1039 self.assertEqual(mime, "text/html")
1040
1041 async def test_file_mode_http_scheme_also_works(self) -> None:
1042 params = MarkdownSplitterInput(
1043 input_mode="file", markdown_file="http://example.com/doc.md"
1044 )
1045 text, mime = await resolve_markdown_text(
1046 params=params,
1047 fetch_remote=lambda _: (b"# fetched", "text/markdown"),
1048 load_kvs=lambda _: "",
1049 convert_remote_file=lambda _raw, _ct: "# fetched",
1050 )
1051 self.assertEqual(text, "# fetched")
1052
1053 async def test_file_mode_kvs_returns_value(self) -> None:
1054 params = MarkdownSplitterInput(
1055 input_mode="file", kvs_store="store", markdown_file="kvs://MY_KEY"
1056 )
1057 text, mime = await resolve_markdown_text(
1058 params=params,
1059 fetch_remote=lambda _: (b"", ""),
1060 load_kvs=lambda key: f"# from {key}",
1061 )
1062 self.assertEqual(text, "# from MY_KEY")
1063 self.assertEqual(mime, "text/markdown")
1064
1065 async def test_file_mode_kvs_strips_prefix(self) -> None:
1066 received_keys: list[str] = []
1067
1068 params = MarkdownSplitterInput(
1069 input_mode="file", kvs_store="store", markdown_file="kvs://SOME_KEY"
1070 )
1071 await resolve_markdown_text(
1072 params=params,
1073 fetch_remote=lambda _: (b"", ""),
1074 load_kvs=lambda key: received_keys.append(key) or "# content",
1075 )
1076 self.assertEqual(received_keys, ["SOME_KEY"])
1077
1078 async def test_file_mode_rejects_empty_kvs_key(self) -> None:
1079 params = MarkdownSplitterInput(
1080 input_mode="file", kvs_store="store", markdown_file="kvs://"
1081 )
1082 with self.assertRaises(ValueError, msg="kvs:// with no key should raise"):
1083 await resolve_markdown_text(
1084 params=params,
1085 fetch_remote=lambda _: (b"", ""),
1086 load_kvs=lambda _: "# content",
1087 )
1088
1089 async def test_file_mode_rejects_unsupported_scheme(self) -> None:
1090 params = MarkdownSplitterInput(
1091 input_mode="file", markdown_file="ftp://example.com/file.md"
1092 )
1093 with self.assertRaises(ValueError):
1094 await resolve_markdown_text(
1095 params=params,
1096 fetch_remote=lambda _: (b"", ""),
1097 load_kvs=lambda _: "# content",
1098 )
1099
1100 async def test_content_type_normalised_to_lowercase(self) -> None:
1101 params = MarkdownSplitterInput(
1102 input_mode="file", markdown_file="https://example.com/doc.pdf"
1103 )
1104 _, mime = await resolve_markdown_text(
1105 params=params,
1106 fetch_remote=lambda _: (b"%PDF-1.4", "Application/PDF"),
1107 load_kvs=lambda _: "",
1108 convert_remote_file=lambda _raw, _ct: "# pdf content",
1109 )
1110 self.assertEqual(mime, "application/pdf")
1111
1112 async def test_empty_content_type_falls_back_to_octet_stream(self) -> None:
1113 params = MarkdownSplitterInput(
1114 input_mode="file", markdown_file="https://example.com/doc"
1115 )
1116 _, mime = await resolve_markdown_text(
1117 params=params,
1118 fetch_remote=lambda _: (b"data", ""),
1119 load_kvs=lambda _: "",
1120 convert_remote_file=lambda _raw, _ct: "# content",
1121 )
1122 self.assertEqual(mime, "application/octet-stream")
1123
1124 async def test_converted_content_exceeding_max_input_chars_raises(self) -> None:
1125 params = MarkdownSplitterInput(
1126 input_mode="file",
1127 markdown_file="https://example.com/doc.md",
1128 max_input_chars=5,
1129 )
1130 with self.assertRaises(ValueError, msg="should raise when result exceeds limit"):
1131 await resolve_markdown_text(
1132 params=params,
1133 fetch_remote=lambda _: (b"data", "text/markdown"),
1134 load_kvs=lambda _: "",
1135 convert_remote_file=lambda _raw, _ct: "# content that is too long",
1136 )
1137
1138 async def test_whitespace_only_resolved_content_raises(self) -> None:
1139 params = MarkdownSplitterInput(
1140 input_mode="file", markdown_file="https://example.com/empty.md"
1141 )
1142 with self.assertRaises(ValueError):
1143 await resolve_markdown_text(
1144 params=params,
1145 fetch_remote=lambda _: (b" ", "text/markdown"),
1146 load_kvs=lambda _: "",
1147 convert_remote_file=lambda _raw, _ct: " ",
1148 )
1149
1150 async def test_converter_receives_correct_arguments(self) -> None:
1151 received: list[tuple] = []
1152
1153 params = MarkdownSplitterInput(
1154 input_mode="file", markdown_file="https://example.com/doc.pdf"
1155 )
1156 await resolve_markdown_text(
1157 params=params,
1158 fetch_remote=lambda _: (b"%PDF", "application/pdf"),
1159 load_kvs=lambda _: "",
1160 convert_remote_file=lambda raw, ct: received.append((raw, ct)) or "# result",
1161 )
1162 self.assertEqual(received, [(b"%PDF", "application/pdf")])
1163
1164 async def test_async_fetch_remote_is_awaited(self) -> None:
1165 async def async_fetch(_url: str) -> tuple[bytes, str]:
1166 return b"# async content", "text/markdown"
1167
1168 params = MarkdownSplitterInput(
1169 input_mode="file", markdown_file="https://example.com/doc.md"
1170 )
1171 text, _ = await resolve_markdown_text(
1172 params=params,
1173 fetch_remote=async_fetch,
1174 load_kvs=lambda _: "",
1175 convert_remote_file=lambda raw, _ct: raw.decode(),
1176 )
1177 self.assertEqual(text, "# async content")
1178
1179 async def test_async_load_kvs_is_awaited(self) -> None:
1180 async def async_load(_key: str) -> str:
1181 return "# async kvs"
1182
1183 params = MarkdownSplitterInput(
1184 input_mode="file", kvs_store="store", markdown_file="kvs://MY_KEY"
1185 )
1186 text, _ = await resolve_markdown_text(
1187 params=params,
1188 fetch_remote=lambda _: (b"", ""),
1189 load_kvs=async_load,
1190 )
1191 self.assertEqual(text, "# async kvs")
1192
1193 async def test_fetch_remote_non_tuple_raises(self) -> None:
1194 params = MarkdownSplitterInput(
1195 input_mode="file", markdown_file="https://example.com/doc.md"
1196 )
1197 with self.assertRaisesRegex(ValueError, "tuple"):
1198 await resolve_markdown_text(
1199 params=params,
1200 fetch_remote=lambda _: b"raw bytes only",
1201 load_kvs=lambda _: "",
1202 )
1203
1204 async def test_fetch_remote_wrong_arity_raises(self) -> None:
1205 params = MarkdownSplitterInput(
1206 input_mode="file", markdown_file="https://example.com/doc.md"
1207 )
1208 with self.assertRaisesRegex(ValueError, "tuple"):
1209 await resolve_markdown_text(
1210 params=params,
1211 fetch_remote=lambda _: (b"x", "text/plain", "extra"),
1212 load_kvs=lambda _: "",
1213 )
1214
1215 async def test_fetch_remote_non_bytes_payload_raises(self) -> None:
1216 params = MarkdownSplitterInput(
1217 input_mode="file", markdown_file="https://example.com/doc.md"
1218 )
1219 with self.assertRaisesRegex(ValueError, "bytes"):
1220 await resolve_markdown_text(
1221 params=params,
1222 fetch_remote=lambda _: ("a string", "text/plain"),
1223 load_kvs=lambda _: "",
1224 )
1225
1226 async def test_fetch_remote_non_string_content_type_raises(self) -> None:
1227 params = MarkdownSplitterInput(
1228 input_mode="file", markdown_file="https://example.com/doc.md"
1229 )
1230 with self.assertRaisesRegex(ValueError, "content type"):
1231 await resolve_markdown_text(
1232 params=params,
1233 fetch_remote=lambda _: (b"x", 123),
1234 load_kvs=lambda _: "",
1235 )
1236
1237 async def test_load_kvs_non_string_result_raises(self) -> None:
1238 params = MarkdownSplitterInput(
1239 input_mode="file", kvs_store="store", markdown_file="kvs://MY_KEY"
1240 )
1241 with self.assertRaisesRegex(ValueError, "UTF-8 string"):
1242 await resolve_markdown_text(
1243 params=params,
1244 fetch_remote=lambda _: (b"", ""),
1245 load_kvs=lambda _: 42,
1246 )
1247
1248 async def test_converted_content_exactly_at_max_input_chars_passes(self) -> None:
1249 params = MarkdownSplitterInput(
1250 input_mode="file",
1251 markdown_file="https://example.com/doc.md",
1252 max_input_chars=9,
1253 )
1254 text, _ = await resolve_markdown_text(
1255 params=params,
1256 fetch_remote=lambda _: (b"data", "text/markdown"),
1257 load_kvs=lambda _: "",
1258 convert_remote_file=lambda _raw, _ct: "# content",
1259 )
1260 self.assertEqual(text, "# content")
1261
1262 async def test_file_mode_strips_surrounding_whitespace_from_source(self) -> None:
1263 received_urls: list[str] = []
1264 params = MarkdownSplitterInput(
1265 input_mode="file", markdown_file=" https://example.com/a.md "
1266 )
1267 await resolve_markdown_text(
1268 params=params,
1269 fetch_remote=lambda url: received_urls.append(url) or (b"# x", "text/markdown"),
1270 load_kvs=lambda _: "",
1271 )
1272 self.assertEqual(received_urls, ["https://example.com/a.md"])
1273
1274
1275
1276
1277
1278
1279
1280class SplitDocumentsTests(unittest.TestCase):
1281 """Splitting behaviour, exercised through the real LangChain splitters.
1282
1283 These tests pin the runtime contract: header→metadata mapping, the
1284 second pass that only re-splits oversized sections, empty-section
1285 filtering (zero-char chunks would be billed), and max_chunks limits.
1286 """
1287
1288 def _params(self, **kw) -> MarkdownSplitterInput:
1289 base: dict = dict(markdown_text="# placeholder")
1290 base.update(kw)
1291 return MarkdownSplitterInput(**base)
1292
1293 def test_splits_on_headers_and_keeps_header_chain(self) -> None:
1294 text = "# Guide\nintro\n## Install\nrun pip\n### Windows\nuse cmd"
1295 docs = split_documents(
1296 self._params(headers_to_split_on=["#", "##", "###"]), text
1297 )
1298 metas = [d.metadata for d in docs]
1299 self.assertIn({"Header 1": "Guide"}, metas)
1300 self.assertIn({"Header 1": "Guide", "Header 2": "Install"}, metas)
1301 self.assertIn(
1302 {"Header 1": "Guide", "Header 2": "Install", "Header 3": "Windows"}, metas
1303 )
1304
1305 def test_heading_anchor_links_are_removed_from_metadata(self) -> None:
1306
1307 text = "# Guide[](https://x.dev/#guide)\nintro\n## Basic usage[](https://x.dev/#basic-usage)\nbody"
1308 docs = split_documents(self._params(), text)
1309 self.assertEqual(
1310 docs[-1].metadata, {"Header 1": "Guide", "Header 2": "Basic usage"}
1311 )
1312
1313 def test_linked_heading_text_is_kept_without_the_url(self) -> None:
1314 text = "## [Install](https://x.dev/install) on [Linux](https://x.dev/linux)\nbody"
1315 docs = split_documents(self._params(), text)
1316 self.assertEqual(docs[0].metadata, {"Header 2": "Install on Linux"})
1317
1318 def test_heading_that_is_only_an_anchor_is_dropped_from_metadata(self) -> None:
1319 text = "# Guide\nintro\n## [](https://x.dev/#a)\nbody"
1320 docs = split_documents(self._params(), text)
1321 self.assertEqual(docs[-1].metadata, {"Header 1": "Guide"})
1322
1323 def test_body_links_are_untouched(self) -> None:
1324 text = "# Guide\nSee [docs](https://x.dev) and [](https://x.dev/#a)."
1325 docs = split_documents(self._params(), text)
1326 self.assertEqual(docs[0].page_content, "See [docs](https://x.dev) and [](https://x.dev/#a).")
1327
1328 def test_sections_under_cap_pass_through_untouched(self) -> None:
1329 text = "# Short\nabc"
1330 docs = split_documents(
1331 self._params(headers_to_split_on=["#"], max_chunk_chars=100), text
1332 )
1333 self.assertEqual(len(docs), 1)
1334 self.assertEqual(docs[0].page_content, "abc")
1335
1336 def test_only_oversized_sections_are_resplit(self) -> None:
1337 long_body = "word " * 80
1338 text = f"# Short\nabc\n\n# Long\n{long_body}"
1339 docs = split_documents(
1340 self._params(headers_to_split_on=["#"], max_chunk_chars=200), text
1341 )
1342 short = [d for d in docs if d.metadata.get("Header 1") == "Short"]
1343 long_ = [d for d in docs if d.metadata.get("Header 1") == "Long"]
1344 self.assertEqual(len(short), 1)
1345 self.assertEqual(short[0].page_content, "abc")
1346 self.assertGreater(len(long_), 1)
1347 self.assertTrue(all(len(d.page_content) <= 200 for d in long_))
1348
1349 def test_empty_sections_are_dropped(self) -> None:
1350
1351 text = "# Empty\n\n# Full\nreal content"
1352 docs = split_documents(self._params(headers_to_split_on=["#"]), text)
1353 contents = [d.page_content for d in docs]
1354 self.assertNotIn("", contents)
1355 self.assertEqual(contents, ["real content"])
1356
1357 def test_no_headers_returns_whole_document(self) -> None:
1358 text = "# looks like a header\nbut no splitting requested\n\nmore"
1359 docs = split_documents(self._params(headers_to_split_on=[]), text)
1360 self.assertEqual(len(docs), 1)
1361
1362 self.assertEqual(docs[0].page_content.split(), text.split())
1363
1364 def test_strip_headers_false_keeps_headers_in_content(self) -> None:
1365 text = "# Title\nbody"
1366 docs = split_documents(
1367 self._params(headers_to_split_on=["#"], strip_headers=False), text
1368 )
1369 self.assertIn("# Title", docs[0].page_content)
1370
1371 def test_chunk_count_equal_to_max_chunks_passes(self) -> None:
1372 text = "# A\none\n\n# B\ntwo"
1373 docs = split_documents(
1374 self._params(headers_to_split_on=["#"], max_chunks=2), text
1375 )
1376 self.assertEqual(len(docs), 2)
1377
1378 def test_impossible_chunk_count_fails_before_resplitting(self) -> None:
1379
1380
1381 text = "word " * 20_000
1382 with patch(
1383 "src.processing.RecursiveCharacterTextSplitter.split_documents",
1384 side_effect=AssertionError("re-split should not run"),
1385 ):
1386 with self.assertRaisesRegex(ValueError, "max_chunks"):
1387 split_documents(
1388 self._params(headers_to_split_on=[], max_chunk_chars=100, max_chunks=10),
1389 text,
1390 )
1391
1392 def test_early_check_does_not_reject_feasible_splits(self) -> None:
1393 text = "word " * 100
1394 docs = split_documents(
1395 self._params(headers_to_split_on=[], max_chunk_chars=200, max_chunks=5), text
1396 )
1397 self.assertLessEqual(len(docs), 5)
1398
1399 def test_cjk_text_splits_at_sentence_ends(self) -> None:
1400
1401
1402 sentence = "这是一个用于测试的中文句子，它没有空格。"
1403 docs = split_documents(
1404 self._params(headers_to_split_on=[], max_chunk_chars=100), sentence * 20
1405 )
1406 self.assertGreater(len(docs), 1)
1407 for d in docs[:-1]:
1408 self.assertTrue(d.page_content.endswith("。"), d.page_content[-5:])
1409
1410 def test_token_cap_bounds_every_chunk(self) -> None:
1411 enc = tiktoken.get_encoding("cl100k_base")
1412 for text in ("word " * 3000, "这是一个用于测试的中文句子，它没有空格。" * 300):
1413 docs = split_documents(self._params(headers_to_split_on=[]), text)
1414 self.assertGreater(len(docs), 1)
1415 self.assertLessEqual(max(len(enc.encode(d.page_content)) for d in docs), 512)
1416
1417 def test_token_chunks_fill_close_to_the_cap(self) -> None:
1418
1419
1420
1421 enc = tiktoken.get_encoding("cl100k_base")
1422 words = ("retrieval augmented generation needs well sized chunks " * 600).strip()
1423 docs = split_documents(self._params(headers_to_split_on=[]), words)
1424 sizes = sorted(len(enc.encode(d.page_content)) for d in docs[:-1])
1425 self.assertLessEqual(sizes[-1], 512)
1426 self.assertGreaterEqual(sizes[len(sizes) // 2], 460)
1427
1428 def test_sections_under_token_cap_stay_whole(self) -> None:
1429 docs = split_documents(self._params(headers_to_split_on=["#"]), "# A\nshort body")
1430 self.assertEqual([d.page_content for d in docs], ["short body"])
1431
1432 def test_char_cap_overrides_token_cap(self) -> None:
1433 docs = split_documents(
1434 self._params(headers_to_split_on=[], max_chunk_chars=300), "word " * 600
1435 )
1436 self.assertLessEqual(max(len(d.page_content) for d in docs), 300)
1437
1438 def test_no_cap_when_both_disabled(self) -> None:
1439 text = "word " * 3000
1440 docs = split_documents(
1441 self._params(headers_to_split_on=[], max_chunk_tokens=None), text
1442 )
1443 self.assertEqual(len(docs), 1)
1444
1445 def test_token_mode_early_max_chunks_check(self) -> None:
1446 with patch(
1447 "src.processing.RecursiveCharacterTextSplitter.split_documents",
1448 side_effect=AssertionError("re-split should not run"),
1449 ):
1450 with self.assertRaisesRegex(ValueError, "max_chunks"):
1451 split_documents(
1452 self._params(headers_to_split_on=[], max_chunk_tokens=50, max_chunks=10),
1453 "word " * 5000,
1454 )
1455
1456 def test_chunk_count_exceeding_max_chunks_raises(self) -> None:
1457 text = "# A\none\n\n# B\ntwo"
1458 with self.assertRaisesRegex(ValueError, "max_chunks"):
1459 split_documents(self._params(headers_to_split_on=["#"], max_chunks=1), text)
1460
1461
1462
1463
1464
1465
1466
1467class SchemaCongruenceTests(unittest.TestCase):
1468 """Pin the contract between INPUT_SCHEMA.json and the model.
1469
1470 The platform applies schema ``default`` values to API runs, so source
1471 fields must only be prefilled (Console form and Apify's daily test),
1472 never defaulted, or a caller's own input could be ignored.
1473 """
1474
1475 @classmethod
1476 def setUpClass(cls) -> None:
1477 path = Path(__file__).resolve().parent.parent / ".actor" / "INPUT_SCHEMA.json"
1478 cls.schema = json.loads(path.read_text())
1479
1480 def test_source_fields_have_no_defaults(self) -> None:
1481 for field in ("input_mode", "markdown_text", "markdown_file"):
1482 self.assertNotIn("default", self.schema["properties"][field], field)
1483
1484 def test_markdown_file_prefill_is_a_valid_url_input(self) -> None:
1485
1486 prefill = self.schema["properties"]["markdown_file"]["prefill"]
1487 obj = MarkdownSplitterInput(markdown_file=prefill)
1488 self.assertEqual(obj.input_mode, "file")
1489
1490 def test_kvs_store_is_a_read_only_store_picker(self) -> None:
1491 field = self.schema["properties"]["kvs_store"]
1492 self.assertEqual(field["editor"], "resourcePicker")
1493 self.assertEqual(field["resourceType"], "keyValueStore")
1494 self.assertEqual(field["resourcePermissions"], ["READ"])
1495
1496 def test_numeric_limits_match_model(self) -> None:
1497 props = self.schema["properties"]
1498 cases = {
1499 "max_input_chars": {"default": 2_000_000, "minimum": 1, "maximum": 5_000_000},
1500 "max_chunks": {"default": 5_000, "minimum": 1, "maximum": 50_000},
1501 "max_chunk_chars": {"minimum": 100, "maximum": 100_000},
1502 "max_chunk_tokens": {"default": 512, "minimum": 50, "maximum": 8192},
1503 }
1504 for field, expected in cases.items():
1505 for key, value in expected.items():
1506 self.assertEqual(
1507 props[field].get(key), value, f"{field}.{key} drifted from the model"
1508 )
1509
1510 def test_null_chunk_cap_passes_platform_validation(self) -> None:
1511
1512
1513 for field in ("max_chunk_chars", "max_chunk_tokens"):
1514 self.assertIs(self.schema["properties"][field].get("nullable"), True, field)
1515
1516 def test_tokenizer_enum_matches_model(self) -> None:
1517 field = self.schema["properties"]["tokenizer"]
1518 self.assertEqual(field["enum"], ["cl100k_base", "o200k_base"])
1519 self.assertEqual(field["default"], "cl100k_base")
1520
1521 def test_headers_default_matches_model(self) -> None:
1522 self.assertEqual(
1523 self.schema["properties"]["headers_to_split_on"]["default"],
1524 MarkdownSplitterInput(markdown_text="# H").headers_to_split_on,
1525 )
1526
1527 def test_dataset_schema_matches_chunk_model(self) -> None:
1528 path = Path(__file__).resolve().parent.parent / ".actor" / "dataset_schema.json"
1529 fields = json.loads(path.read_text())["fields"]
1530 self.assertEqual(set(fields["properties"]), set(MarkdownChunk.model_fields))
1531 self.assertEqual(set(fields["required"]), set(MarkdownChunk.model_fields))
1532
1533 def test_every_model_field_is_represented(self) -> None:
1534 schema_fields = set(self.schema["properties"])
1535 model_fields = set(MarkdownSplitterInput.model_fields)
1536 self.assertEqual(schema_fields, model_fields)
1537
1538
1539
1540
1541
1542
1543
1544class ConvertToMarkdownTests(unittest.TestCase):
1545 def _fake_markitdown_module(self, text_content: str | None) -> ModuleType:
1546 """Build a fake markitdown module whose converter returns given text_content."""
1547
1548 class _Result:
1549 pass
1550
1551 _Result.text_content = text_content
1552
1553 class _StreamInfo:
1554 def __init__(self, **fields) -> None:
1555 self.__dict__.update(fields)
1556
1557 class _Converter:
1558 def __init__(self) -> None:
1559 self.last_info: _StreamInfo | None = None
1560
1561
1562 def convert_stream(self, _stream, *, stream_info=None):
1563 self.last_info = stream_info
1564 return _Result()
1565
1566 _instance = _Converter()
1567
1568 fake = ModuleType("markitdown")
1569 fake.MarkItDown = lambda enable_plugins=False: _instance
1570 fake.StreamInfo = _StreamInfo
1571 fake._instance = _instance
1572 return fake
1573
1574 def test_returns_text_content(self) -> None:
1575 fake = self._fake_markitdown_module("# Title\n\nBody")
1576 with patch.dict("sys.modules", {"markitdown": fake}):
1577 result = convert_to_markdown(b"<h1>Title</h1><p>Body</p>", "text/html")
1578 self.assertEqual(result, "# Title\n\nBody")
1579
1580 def test_passes_mimetype_to_converter(self) -> None:
1581 fake = self._fake_markitdown_module("# ok")
1582 with patch.dict("sys.modules", {"markitdown": fake}):
1583 convert_to_markdown(b"data", "application/pdf")
1584 self.assertEqual(fake._instance.last_info.mimetype, "application/pdf")
1585
1586 def test_empty_content_type_leaves_mimetype_to_sniffing(self) -> None:
1587 fake = self._fake_markitdown_module("# ok")
1588 with patch.dict("sys.modules", {"markitdown": fake}):
1589 convert_to_markdown(b"data", "")
1590 self.assertIsNone(fake._instance.last_info.mimetype)
1591
1592 def test_splits_charset_and_passes_url_and_extension(self) -> None:
1593 fake = self._fake_markitdown_module("# ok")
1594 with patch.dict("sys.modules", {"markitdown": fake}):
1595 convert_to_markdown(
1596 b"data",
1597 "Text/HTML; charset=ISO-8859-1",
1598 url="https://example.com/docs/page.HTML?x=1",
1599 )
1600 info = fake._instance.last_info
1601 self.assertEqual(info.mimetype, "text/html")
1602 self.assertEqual(info.charset, "ISO-8859-1")
1603 self.assertEqual(info.url, "https://example.com/docs/page.HTML?x=1")
1604 self.assertEqual(info.extension, ".html")
1605
1606 def test_octet_stream_is_treated_as_unknown(self) -> None:
1607
1608 fake = self._fake_markitdown_module("# ok")
1609 with patch.dict("sys.modules", {"markitdown": fake}):
1610 convert_to_markdown(b"data", "application/octet-stream")
1611 self.assertIsNone(fake._instance.last_info.mimetype)
1612
1613 def test_real_markitdown_honours_declared_charset(self) -> None:
1614 html = "<h1>Menu</h1><p>café</p>".encode("latin-1")
1615 result = convert_to_markdown(html, "text/html; charset=ISO-8859-1")
1616 self.assertIn("café", result)
1617
1618 def test_real_markitdown_converts_declared_csv_to_table(self) -> None:
1619 result = convert_to_markdown(b"name,qty\napple,3\n", "text/csv")
1620 self.assertIn("| name | qty |", result)
1621
1622 def test_empty_result_mentions_scanned_pdfs(self) -> None:
1623 fake = self._fake_markitdown_module(" ")
1624 with patch.dict("sys.modules", {"markitdown": fake}):
1625 with self.assertRaisesRegex(ValueError, "OCR"):
1626 convert_to_markdown(b"%PDF", "application/pdf")
1627
1628 def test_raises_when_result_is_none(self) -> None:
1629 fake = self._fake_markitdown_module(None)
1630 with patch.dict("sys.modules", {"markitdown": fake}):
1631 with self.assertRaises(ValueError, msg="None text_content should raise"):
1632 convert_to_markdown(b"data", "text/html")
1633
1634 def test_raises_when_result_is_whitespace_only(self) -> None:
1635 fake = self._fake_markitdown_module(" \n ")
1636 with patch.dict("sys.modules", {"markitdown": fake}):
1637 with self.assertRaises(ValueError):
1638 convert_to_markdown(b"data", "text/html")
1639
1640 def test_passes_raw_bytes_as_stream(self) -> None:
1641 import io as _io
1642
1643 received_streams: list[bytes] = []
1644
1645 class _Converter:
1646 def convert_stream(self, stream, *, stream_info=None):
1647 received_streams.append(stream.read())
1648
1649 class _R:
1650 text_content = "# ok"
1651
1652 return _R()
1653
1654 fake = ModuleType("markitdown")
1655 fake.MarkItDown = lambda enable_plugins=False: _Converter()
1656 fake.StreamInfo = lambda **fields: fields
1657 with patch.dict("sys.modules", {"markitdown": fake}):
1658 convert_to_markdown(b"\x25\x50\x44\x46", "application/pdf")
1659
1660 self.assertEqual(received_streams, [b"\x25\x50\x44\x46"])
1661
1662
1663if __name__ == "__main__":
1664 unittest.main()
