m2 (AI Agent)
6da874b2e6
Phase 1: router skeleton + Qdrant read path - FastAPI app with /health, /search/generic, /ingest endpoints - Qdrant reader (hybrid dense+sparse w/ RRF, plus payload scroll) - BGE-M3 TEI client for query-side embedding - memory-api client proxying writes to /memory/store (m2-memory untouched) - Pydantic Fact + Exemplar payloads, discriminated by `kind` - Dockerfile + docker-compose joining the coolify Docker network Phase 2: authoritative-source ingest pipeline - PDF text extraction + paragraph-aware (§ N) and size chunkers - Loaders for bnatschg, mhbasp, biotopwertliste, state-<slug> - CLI: artenschutz-ingest --source <name> [--states ...] [--dry-run] - Helper script to fetch BNatSchG from gesetze-im-internet.de - End-to-end smoke script (scripts/smoke.sh) for Coolify validation 22/22 unit tests pass. Real-world dry-runs verified against mhbasp Anhang 4, biotopwertlisteNEU and Berlin Kartierstandards PDFs from GST-DATA. See COOLIFY-DEPLOY.md for staging deploy + smoke procedure.
50 lines
1.6 KiB
Python
50 lines
1.6 KiB
Python
from __future__ import annotations
|
|
|
|
from artenschutz_router.ingest.chunker import chunk_by_paragraph_marker, chunk_by_size
|
|
|
|
|
|
def test_paragraph_chunker_splits_at_section_markers():
|
|
text = """Vor § 44.
|
|
|
|
§ 44 Vorschriften für besonders geschützte und bestimmte andere Tier- und Pflanzenarten
|
|
|
|
(1) Es ist verboten,
|
|
1. wild lebenden Tieren der besonders geschützten Arten nachzustellen, sie zu fangen, …
|
|
|
|
§ 45b Schutz wild lebender Vögel
|
|
|
|
(1) Beim Betrieb von Windenergieanlagen …
|
|
|
|
Anlage 1 (zu § 54 Absatz 4)
|
|
|
|
Bestimmte besonders geschützte Arten …
|
|
"""
|
|
chunks = chunk_by_paragraph_marker(text)
|
|
anchors = [c.anchor for c in chunks]
|
|
assert any(a and a.startswith("§ 44") for a in anchors)
|
|
assert any(a and a.startswith("§ 45b") for a in anchors)
|
|
assert any(a and a.startswith("Anlage 1") for a in anchors)
|
|
# Pre-§44 prefix should NOT become its own chunk (we anchor on markers).
|
|
assert "Vor § 44" not in chunks[0].text
|
|
|
|
|
|
def test_paragraph_chunker_returns_single_when_no_markers():
|
|
text = "Plain prose without any paragraph markers, just regular text."
|
|
chunks = chunk_by_paragraph_marker(text)
|
|
assert len(chunks) == 1
|
|
assert chunks[0].anchor is None
|
|
|
|
|
|
def test_size_chunker_respects_target_size():
|
|
paragraphs = ["abc def ghi jkl"] * 200
|
|
text = "\n\n".join(paragraphs)
|
|
chunks = chunk_by_size(text, target_chars=200, overlap_chars=20)
|
|
assert len(chunks) > 1
|
|
for c in chunks:
|
|
# Overlap may push slightly over; tolerate 2x.
|
|
assert len(c.text) < 600
|
|
|
|
|
|
def test_size_chunker_handles_empty():
|
|
assert chunk_by_size("") == []
|
|
assert chunk_by_size(" \n ") == []
|