m2 (AI Agent)
6da874b2e6
Phase 1: router skeleton + Qdrant read path - FastAPI app with /health, /search/generic, /ingest endpoints - Qdrant reader (hybrid dense+sparse w/ RRF, plus payload scroll) - BGE-M3 TEI client for query-side embedding - memory-api client proxying writes to /memory/store (m2-memory untouched) - Pydantic Fact + Exemplar payloads, discriminated by `kind` - Dockerfile + docker-compose joining the coolify Docker network Phase 2: authoritative-source ingest pipeline - PDF text extraction + paragraph-aware (§ N) and size chunkers - Loaders for bnatschg, mhbasp, biotopwertliste, state-<slug> - CLI: artenschutz-ingest --source <name> [--states ...] [--dry-run] - Helper script to fetch BNatSchG from gesetze-im-internet.de - End-to-end smoke script (scripts/smoke.sh) for Coolify validation 22/22 unit tests pass. Real-world dry-runs verified against mhbasp Anhang 4, biotopwertlisteNEU and Berlin Kartierstandards PDFs from GST-DATA. See COOLIFY-DEPLOY.md for staging deploy + smoke procedure.
72 lines
2.9 KiB
Bash
Executable file
72 lines
2.9 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# End-to-end smoke for artenschutz-router against the live memory stack.
|
|
# Runs from inside the container; ROUTER_URL defaults to localhost:8080.
|
|
set -euo pipefail
|
|
|
|
ROUTER_URL="${ROUTER_URL:-http://localhost:8080}"
|
|
|
|
step() { printf '\n\033[1;36m== %s ==\033[0m\n' "$*"; }
|
|
pass() { printf '\033[1;32m ✓ %s\033[0m\n' "$*"; }
|
|
fail() { printf '\033[1;31m ✗ %s\033[0m\n' "$*"; exit 1; }
|
|
|
|
step "1/5 health"
|
|
health=$(curl -sS "$ROUTER_URL/health")
|
|
echo " $health"
|
|
echo "$health" | grep -q '"status":"ok"' || fail "health is not ok — fix services first"
|
|
pass "router can reach Qdrant + BGE-M3 + memory-api"
|
|
|
|
ingest_source() {
|
|
local source="$1" ; shift
|
|
local limit="${1:-5}"
|
|
step "ingest --source $source --limit $limit"
|
|
python -m artenschutz_router.ingest.cli \
|
|
--source "$source" \
|
|
--router-url "$ROUTER_URL" \
|
|
--limit "$limit" "$@" \
|
|
2>&1 | tail -5
|
|
}
|
|
|
|
step "2/5 fetch BNatSchG if missing"
|
|
if [ ! -s sources/bnatschg/bnatschg.txt ]; then
|
|
python scripts/fetch_bnatschg.py
|
|
fi
|
|
|
|
step "3/5 ingest small batches"
|
|
ingest_source bnatschg 5
|
|
ingest_source mhbasp 5
|
|
ingest_source biotopwertliste 3 --states BY
|
|
ingest_source state-berlin 5
|
|
|
|
step "4/5 search — hybrid, fact_type=method"
|
|
result=$(curl -sS -X POST "$ROUTER_URL/search/generic" \
|
|
-H 'Content-Type: application/json' \
|
|
-d '{"query":"Fledermäuse Kartierung","metadata_filter":{"fact_type":"method"},"limit":3}')
|
|
hits=$(echo "$result" | python -c "import sys,json; print(len(json.load(sys.stdin)['hits']))")
|
|
echo " hits: $hits"
|
|
[ "$hits" -ge 1 ] || fail "hybrid search returned 0 hits — check BGE-M3"
|
|
pass "hybrid retrieval works"
|
|
|
|
step "4b/5 search — hybrid, fact_type=legal"
|
|
result=$(curl -sS -X POST "$ROUTER_URL/search/generic" \
|
|
-H 'Content-Type: application/json' \
|
|
-d '{"query":"besonders geschützte Arten","metadata_filter":{"fact_type":"legal"},"limit":3}')
|
|
hits=$(echo "$result" | python -c "import sys,json; print(len(json.load(sys.stdin)['hits']))")
|
|
echo " hits: $hits"
|
|
[ "$hits" -ge 1 ] || fail "legal hybrid search returned 0 hits"
|
|
pass "BNatSchG indexed and queryable"
|
|
|
|
step "5/5 scroll — scope.states contains BE"
|
|
result=$(curl -sS -X POST "$ROUTER_URL/search/generic" \
|
|
-H 'Content-Type: application/json' \
|
|
-d '{"metadata_filter":{"scope.states":["BE"]},"limit":3}')
|
|
hits=$(echo "$result" | python -c "import sys,json; print(len(json.load(sys.stdin)['hits']))")
|
|
echo " hits: $hits"
|
|
[ "$hits" -ge 1 ] || fail "scope.states=BE scroll returned 0 hits — Berlin ingest missed?"
|
|
pass "payload filtering on nested keys works"
|
|
|
|
printf '\n\033[1;32mALL SMOKE STEPS PASSED\033[0m\n'
|
|
echo
|
|
echo "To clean up the staging data later:"
|
|
echo " curl -X POST <qdrant>:6333/collections/agent_memory/points/delete \\"
|
|
echo " -H 'Content-Type: application/json' \\"
|
|
echo " -d '{\"filter\":{\"must\":[{\"key\":\"agent_id\",\"match\":{\"value\":\"gruenstifter_staging\"}}]}}'"
|