From 6da874b2e664fef9af2a490ec7c5f87234eb808f Mon Sep 17 00:00:00 2001 From: "m2 (AI Agent)" Date: Thu, 14 May 2026 16:36:00 +0200 Subject: [PATCH] =?UTF-8?q?Initial=20commit=20=E2=80=94=20phase=201=20+=20?= =?UTF-8?q?2=20of=20artenschutz-digest=20concept?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1: router skeleton + Qdrant read path - FastAPI app with /health, /search/generic, /ingest endpoints - Qdrant reader (hybrid dense+sparse w/ RRF, plus payload scroll) - BGE-M3 TEI client for query-side embedding - memory-api client proxying writes to /memory/store (m2-memory untouched) - Pydantic Fact + Exemplar payloads, discriminated by `kind` - Dockerfile + docker-compose joining the coolify Docker network Phase 2: authoritative-source ingest pipeline - PDF text extraction + paragraph-aware (§ N) and size chunkers - Loaders for bnatschg, mhbasp, biotopwertliste, state- - CLI: artenschutz-ingest --source [--states ...] [--dry-run] - Helper script to fetch BNatSchG from gesetze-im-internet.de - End-to-end smoke script (scripts/smoke.sh) for Coolify validation 22/22 unit tests pass. Real-world dry-runs verified against mhbasp Anhang 4, biotopwertlisteNEU and Berlin Kartierstandards PDFs from GST-DATA. See COOLIFY-DEPLOY.md for staging deploy + smoke procedure. --- .env.example | 22 +++ .gitignore | 19 +++ COOLIFY-DEPLOY.md | 128 ++++++++++++++++ Dockerfile | 25 +++ README.md | 67 +++++++++ docker-compose.yml | 24 +++ pyproject.toml | 42 ++++++ scripts/fetch_bnatschg.py | 112 ++++++++++++++ scripts/smoke.sh | 72 +++++++++ sources/README.md | 22 +++ sources/biotopwertliste/.gitkeep | 0 sources/bnatschg/.gitkeep | 0 sources/mhbasp/.gitkeep | 0 sources/states/baden_wuerttemberg/.gitkeep | 0 sources/states/bayern/.gitkeep | 0 sources/states/berlin/.gitkeep | 0 sources/states/brandenburg/.gitkeep | 0 sources/states/sachsen/.gitkeep | 0 src/artenschutz_router/__init__.py | 3 + src/artenschutz_router/clients/__init__.py | 1 + src/artenschutz_router/clients/bge.py | 74 +++++++++ src/artenschutz_router/clients/memory_api.py | 57 +++++++ src/artenschutz_router/clients/qdrant.py | 112 ++++++++++++++ src/artenschutz_router/config.py | 29 ++++ src/artenschutz_router/ingest/__init__.py | 1 + .../ingest/authoritative/__init__.py | 10 ++ .../ingest/authoritative/base.py | 30 ++++ .../ingest/authoritative/biotopwertliste.py | 53 +++++++ .../ingest/authoritative/bnatschg.py | 58 +++++++ .../ingest/authoritative/mhbasp.py | 49 ++++++ .../ingest/authoritative/state_docs.py | 54 +++++++ src/artenschutz_router/ingest/chunker.py | 93 ++++++++++++ src/artenschutz_router/ingest/cli.py | 136 +++++++++++++++++ src/artenschutz_router/ingest/pdf_text.py | 27 ++++ src/artenschutz_router/main.py | 93 ++++++++++++ src/artenschutz_router/models.py | 142 ++++++++++++++++++ src/artenschutz_router/routes/__init__.py | 1 + src/artenschutz_router/routes/ingest.py | 43 ++++++ src/artenschutz_router/routes/search.py | 62 ++++++++ tests/__init__.py | 0 tests/test_app.py | 30 ++++ tests/test_bnatschg_loader.py | 58 +++++++ tests/test_chunker.py | 50 ++++++ tests/test_cli.py | 49 ++++++ tests/test_filter_builder.py | 30 ++++ tests/test_models.py | 91 +++++++++++ 46 files changed, 1969 insertions(+) create mode 100644 .env.example create mode 100644 .gitignore create mode 100644 COOLIFY-DEPLOY.md create mode 100644 Dockerfile create mode 100644 README.md create mode 100644 docker-compose.yml create mode 100644 pyproject.toml create mode 100755 scripts/fetch_bnatschg.py create mode 100755 scripts/smoke.sh create mode 100644 sources/README.md create mode 100644 sources/biotopwertliste/.gitkeep create mode 100644 sources/bnatschg/.gitkeep create mode 100644 sources/mhbasp/.gitkeep create mode 100644 sources/states/baden_wuerttemberg/.gitkeep create mode 100644 sources/states/bayern/.gitkeep create mode 100644 sources/states/berlin/.gitkeep create mode 100644 sources/states/brandenburg/.gitkeep create mode 100644 sources/states/sachsen/.gitkeep create mode 100644 src/artenschutz_router/__init__.py create mode 100644 src/artenschutz_router/clients/__init__.py create mode 100644 src/artenschutz_router/clients/bge.py create mode 100644 src/artenschutz_router/clients/memory_api.py create mode 100644 src/artenschutz_router/clients/qdrant.py create mode 100644 src/artenschutz_router/config.py create mode 100644 src/artenschutz_router/ingest/__init__.py create mode 100644 src/artenschutz_router/ingest/authoritative/__init__.py create mode 100644 src/artenschutz_router/ingest/authoritative/base.py create mode 100644 src/artenschutz_router/ingest/authoritative/biotopwertliste.py create mode 100644 src/artenschutz_router/ingest/authoritative/bnatschg.py create mode 100644 src/artenschutz_router/ingest/authoritative/mhbasp.py create mode 100644 src/artenschutz_router/ingest/authoritative/state_docs.py create mode 100644 src/artenschutz_router/ingest/chunker.py create mode 100644 src/artenschutz_router/ingest/cli.py create mode 100644 src/artenschutz_router/ingest/pdf_text.py create mode 100644 src/artenschutz_router/main.py create mode 100644 src/artenschutz_router/models.py create mode 100644 src/artenschutz_router/routes/__init__.py create mode 100644 src/artenschutz_router/routes/ingest.py create mode 100644 src/artenschutz_router/routes/search.py create mode 100644 tests/__init__.py create mode 100644 tests/test_app.py create mode 100644 tests/test_bnatschg_loader.py create mode 100644 tests/test_chunker.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_filter_builder.py create mode 100644 tests/test_models.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..ffaa928 --- /dev/null +++ b/.env.example @@ -0,0 +1,22 @@ +# Qdrant + BGE-M3 — same services agent.memory.system uses. +# In the Coolify Docker network these resolve by container name. +# For local dev pointed at a remote stack, use container IPs from +# docker inspect or memory-api's resolved endpoints. +QDRANT_URL=http://memory-qdrant:6333 +QDRANT_COLLECTION=agent_memory +BGE_TEI_URL=http://memory-embeddings:8000 + +# memory-api is only used for the WRITE path (/ingest → /memory/store). +# Reads go directly to Qdrant. +MEMORY_API_URL=http://memory-api:8000 + +# agent_id partition in Qdrant. Default scopes by client; override per deploy. +# Examples: +# gruenstifter — production client +# gruenstifter_staging — same client, staging env (cleanly deletable) +ARTENSCHUTZ_AGENT_ID=gruenstifter + +# Service binding +ROUTER_HOST=0.0.0.0 +ROUTER_PORT=8080 +LOG_LEVEL=INFO diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..640efa8 --- /dev/null +++ b/.gitignore @@ -0,0 +1,19 @@ +__pycache__/ +*.pyc +*.pyo +.pytest_cache/ +.ruff_cache/ +.venv/ +venv/ +.env +*.egg-info/ +build/ +dist/ + +# Local source dumps (real authoritative sources are large; keep .gitkeep markers) +sources/**/*.pdf +sources/**/*.docx +sources/**/*.txt +sources/**/*.html +!sources/**/.gitkeep +!sources/**/README.md diff --git a/COOLIFY-DEPLOY.md b/COOLIFY-DEPLOY.md new file mode 100644 index 0000000..5217cd0 --- /dev/null +++ b/COOLIFY-DEPLOY.md @@ -0,0 +1,128 @@ +# Coolify staging deploy — artenschutz-router + +Goal: stand up the router on Coolify staging, in the same Docker network as +`agent.memory.system`, so it can reach `memory-qdrant`, `memory-embeddings`, +and `memory-api` by container name. Then run an end-to-end smoke ingest + +search against real data. + +## Pre-flight + +You should already have on staging Coolify: + +- A running `agent.memory.system` (memory-api + memory-qdrant + memory-embeddings) +- The Docker network name those services live on (probably `coolify` — check + `docker network ls` on the staging host) + +Note down: + +| Setting | Value (example) | +|---|---| +| Coolify project | `gruenstifter-staging` | +| Docker network | `coolify` | +| memory-api container name | `memory-api-` | +| memory-qdrant container name | `memory-qdrant-` | +| memory-embeddings container name | `memory-embeddings-` | + +> The *full* container names include Coolify's stack hash suffix. The +> compose file uses the short service names (`memory-api`, etc.) which work +> when the router is on the same Coolify-managed network — Coolify creates +> service-name aliases. If they don't resolve, fall back to container IPs +> via `docker inspect`. + +## Path A — Push to forgejo, let Coolify auto-deploy (preferred) + +```bash +cd /home/m2/artenschutz-router +git init -b main +git add . +git commit -m "Initial commit — phase 1 + 2" +git remote add origin :gruenstifter/artenschutz-router.git +git push -u origin main +``` + +In Coolify staging: + +1. New Resource → **Application** → **Public Git Repository** (or Private, + add the SSH key) +2. Source: the forgejo URL above +3. Build Pack: **Dockerfile** +4. Set environment variables: + ``` + QDRANT_URL=http://memory-qdrant:6333 + QDRANT_COLLECTION=agent_memory + BGE_TEI_URL=http://memory-embeddings:8000 + MEMORY_API_URL=http://memory-api:8000 + ARTENSCHUTZ_AGENT_ID=gruenstifter_staging + ROUTER_PORT=8080 + LOG_LEVEL=INFO + ``` + `gruenstifter_staging` keeps smoke data cleanly separable from any future + production write. +5. Network: join the same network as `agent.memory.system` (usually `coolify`). +6. Port: 8080 (no public exposure needed for smoke — internal-only is fine). +7. Deploy. + +## Path B — Docker Compose on the Coolify host + +If you'd rather not push to forgejo yet: + +```bash +# On the Coolify host (or via SSH): +scp -r /home/m2/artenschutz-router coolify-host:/opt/coolify-apps/ +ssh coolify-host +cd /opt/coolify-apps/artenschutz-router +cp .env.example .env +# Edit .env to set ARTENSCHUTZ_AGENT_ID=gruenstifter_staging +docker compose build +docker compose up -d +docker compose logs -f artenschutz-router +``` + +This bypasses Coolify's UI but uses the same `coolify` external network the +memory stack joins. + +## Verify it's up + +From any shell on the Coolify host: + +```bash +docker exec artenschutz-router curl -sS http://localhost:8080/health +# expect: {"status":"ok","qdrant":true,"bge":true,"memory_api":true} +``` + +`status: "degraded"` with one of the three `false` means that service isn't +reachable from inside the router's network namespace — fix container names / +network attachment before continuing. + +## End-to-end smoke (uses the test data already symlinked in `sources/`) + +```bash +# 1. Fetch BNatSchG into the container's sources/bnatschg/ if not already. +docker exec artenschutz-router python scripts/fetch_bnatschg.py + +# 2. Run the smoke (ingests small batches + queries them back). +docker exec artenschutz-router bash scripts/smoke.sh +``` + +The smoke script ingests ~5 records per source, then runs three searches: + +- Hybrid query `"Fledermäuse Kartierung"` filtered to `fact_type=method` +- Hybrid query `"besonders geschützte Arten"` filtered to `fact_type=legal` +- Payload-only scroll on `scope.states=BE` to verify state filtering + +Expected: each search returns ≥1 hit. If hybrid returns 0 hits but scroll +works, BGE-M3 embedding is failing — check the logs. + +## Cleanup (after smoke) + +Drop the staging agent_id from Qdrant: + +```bash +docker exec memory-qdrant- curl -X POST \ + http://localhost:6333/collections/agent_memory/points/delete \ + -H 'Content-Type: application/json' \ + -d '{"filter":{"must":[{"key":"agent_id","match":{"value":"gruenstifter_staging"}}]}}' +``` + +This wipes every point we wrote under the staging agent_id. Nothing else is +affected because partition isolation is by `agent_id`. diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..971c5e3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,25 @@ +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PIP_NO_CACHE_DIR=1 + +# curl: for smoke.sh + /health probes. +RUN apt-get update && apt-get install -y --no-install-recommends curl && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY pyproject.toml ./ +COPY src ./src +COPY scripts ./scripts +COPY sources ./sources + +RUN pip install --upgrade pip && pip install . + +# sources/ is bind-mountable so an operator can drop files (esp. bnatschg.txt) +# without a rebuild. +VOLUME ["/app/sources"] + +EXPOSE 8080 + +CMD ["uvicorn", "artenschutz_router.main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..dd1cb07 --- /dev/null +++ b/README.md @@ -0,0 +1,67 @@ +# artenschutz-router + +Sidecar service for Artenschutzgutachten domain queries against +`agent.memory.system`. **m2-memory stays untouched.** + +- **Writes** proxy memory-api `/memory/store` — the existing endpoint accepts + arbitrary `metadata: dict`, so our Fact / Exemplar payloads slot in. +- **Reads** go directly to Qdrant for rich payload-filtered hybrid queries + (dense + sparse via the existing BGE-M3 TEI service). + +See `.planning/notes/artenschutz-digest-concept.md` in `GST-DATA` for the +draft concept that birthed this. + +## Endpoints + +| Method | Path | Purpose | +|---|---|---| +| `GET` | `/health` | Live-probe Qdrant, BGE-M3, memory-api | +| `POST` | `/search/generic` | Hybrid search (or scroll if no `query`); arbitrary payload filters | +| `POST` | `/ingest` | Push a Fact or Exemplar record | + +Domain-specific endpoints (`/facts/search`, `/exemplar/search`, +`/mitigation/search`) are deferred to a later phase — they will be thin +wrappers over `/search/generic`. + +## Quick start (local dev) + +```bash +cp .env.example .env +# Edit .env to point QDRANT_URL / BGE_TEI_URL / MEMORY_API_URL at your stack +docker compose up --build +curl http://localhost:8080/health +``` + +If you're targeting the production Coolify memory stack, leave the default +container-name URLs and ensure the `coolify` Docker network is joined. + +## Running ingest + +```bash +# After you've placed authoritative source files into sources/... +artenschutz-ingest --source bnatschg --router-url http://localhost:8080 +artenschutz-ingest --source mhbasp --router-url http://localhost:8080 +artenschutz-ingest --source biotopwertliste --router-url http://localhost:8080 +artenschutz-ingest --source state-berlin --router-url http://localhost:8080 +``` + +See `sources/README.md` for which files each source expects. + +## Tests + +```bash +pip install -e .[dev] +pytest +``` + +## Architecture + +``` +m2-gpt agents + ↓ +artenschutz-router (this repo) + ├─ /ingest ─→ memory-api /memory/store (unchanged) + └─ /search/generic ─→ Qdrant /points/query + BGE-M3 TEI /embed[_sparse] + ↓ + agent_memory collection (Qdrant) +``` diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..fc50ea4 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,24 @@ +# Local dev compose. Joins the existing Coolify "coolify" network so it can +# resolve memory-qdrant / memory-embeddings / memory-api by container name. +# If those services aren't reachable, override the *_URL env vars in .env. + +services: + artenschutz-router: + build: . + container_name: artenschutz-router + env_file: + - .env + ports: + - "8080:8080" + networks: + - coolify + restart: unless-stopped + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health')"] + interval: 30s + timeout: 5s + retries: 3 + +networks: + coolify: + external: true diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b8e0409 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,42 @@ +[project] +name = "artenschutz-router" +version = "0.1.0" +description = "Sidecar router for Artenschutz domain queries against agent.memory.system" +requires-python = ">=3.11" +readme = "README.md" + +dependencies = [ + "fastapi>=0.110", + "uvicorn[standard]>=0.27", + "pydantic>=2.6", + "pydantic-settings>=2.2", + "httpx>=0.27", + "qdrant-client>=1.9", + "pypdf>=4.0", + "python-dateutil>=2.9", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "pytest-asyncio>=0.23", + "ruff>=0.4", +] + +[project.scripts] +artenschutz-ingest = "artenschutz_router.ingest.cli:main" + +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +asyncio_mode = "auto" +testpaths = ["tests"] + +[tool.ruff] +line-length = 100 +target-version = "py311" diff --git a/scripts/fetch_bnatschg.py b/scripts/fetch_bnatschg.py new file mode 100755 index 0000000..4aeca7b --- /dev/null +++ b/scripts/fetch_bnatschg.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Fetch the consolidated text of BNatSchG §44 and §45b from +gesetze-im-internet.de and emit a plain-text file suitable for the +bnatschg ingest loader. + +Usage: + python scripts/fetch_bnatschg.py [--out sources/bnatschg/bnatschg.txt] + +The output uses one heading line per section ("§ N ..." or "Anlage N ...") +exactly as the bnatschg loader's paragraph splitter expects. + +Best-effort HTML scrape. If gesetze-im-internet.de changes layout, paste +text manually instead (see sources/README.md). +""" +from __future__ import annotations + +import argparse +import re +import sys +from html.parser import HTMLParser +from pathlib import Path +from urllib.request import Request, urlopen + +# §s most relevant for Artenschutzgutachten. Add more as needed. +SECTIONS = [ + ("§ 44 Vorschriften für besonders geschützte und bestimmte andere Tier- und Pflanzenarten", + "https://www.gesetze-im-internet.de/bnatschg_2009/__44.html"), + ("§ 45 Ausnahmen; Ermächtigung zum Erlass von Rechtsverordnungen", + "https://www.gesetze-im-internet.de/bnatschg_2009/__45.html"), + ("§ 45b Schutz wild lebender Vogelarten bei der Errichtung und dem Betrieb von Windenergieanlagen an Land", + "https://www.gesetze-im-internet.de/bnatschg_2009/__45b.html"), + ("Anlage 1 (zu § 45b Absatz 1 bis 5) Liste der bei der Errichtung und dem Betrieb von Windenergieanlagen an Land kollisionsgefährdeten Brutvogelarten", + "https://www.gesetze-im-internet.de/bnatschg_2009/anlage_1.html"), +] + + +class TextExtractor(HTMLParser): + def __init__(self) -> None: + super().__init__() + self._buf: list[str] = [] + self._skip = 0 # depth of