from __future__ import annotations from artenschutz_router.ingest.chunker import chunk_by_paragraph_marker, chunk_by_size def test_paragraph_chunker_splits_at_section_markers(): text = """Vor § 44. § 44 Vorschriften für besonders geschützte und bestimmte andere Tier- und Pflanzenarten (1) Es ist verboten, 1. wild lebenden Tieren der besonders geschützten Arten nachzustellen, sie zu fangen, … § 45b Schutz wild lebender Vögel (1) Beim Betrieb von Windenergieanlagen … Anlage 1 (zu § 54 Absatz 4) Bestimmte besonders geschützte Arten … """ chunks = chunk_by_paragraph_marker(text) anchors = [c.anchor for c in chunks] assert any(a and a.startswith("§ 44") for a in anchors) assert any(a and a.startswith("§ 45b") for a in anchors) assert any(a and a.startswith("Anlage 1") for a in anchors) # Pre-§44 prefix should NOT become its own chunk (we anchor on markers). assert "Vor § 44" not in chunks[0].text def test_paragraph_chunker_returns_single_when_no_markers(): text = "Plain prose without any paragraph markers, just regular text." chunks = chunk_by_paragraph_marker(text) assert len(chunks) == 1 assert chunks[0].anchor is None def test_size_chunker_respects_target_size(): paragraphs = ["abc def ghi jkl"] * 200 text = "\n\n".join(paragraphs) chunks = chunk_by_size(text, target_chars=200, overlap_chars=20) assert len(chunks) > 1 for c in chunks: # Overlap may push slightly over; tolerate 2x. assert len(c.text) < 600 def test_size_chunker_handles_empty(): assert chunk_by_size("") == [] assert chunk_by_size(" \n ") == []