diff --git a/agents/nasr/MEMORY.md b/agents/nasr/MEMORY.md index dae8ec4..e1a48d9 100644 --- a/agents/nasr/MEMORY.md +++ b/agents/nasr/MEMORY.md @@ -20,3 +20,29 @@ Modules to build: 3. Multi-LLM orchestrator with artifact checks 4. GPU/Docker autoconfig (CUDA/ROCm/Metal) 5. Migration tooling for existing crypto projects + +## GMI Clinic +- GMI = Machine.Machine's clinic partner in Berlin — major strategic account +- Projects on disk: GMI-Cancer-Pathways/ (Breast, Colorectal, NSCLC pathways) +- Context from group chat ingested into memory (machine.machine.zip — 2026-04-22) +- Planned: dedicated GMI sub-agent, but Smithers holds GMI context for now +- Clinical pathway work is high priority — ask Mariusz before kickoff (Friday 2026-04-25) + +## Hermes Agent (upcoming) +- A special agent being planned — Mariusz will give details +- Will be integrated with Smithers's context +- Likely uses the Hermes agent framework +- Park this: wait for Mariusz's feedback before acting + +## Active Projects (on disk at ~/.openclaw/workspace/projects/) +- GMI-Cancer-Pathways — clinical pathway docs (Breast, Colorectal, NSCLC) +- quantum-trading-intelligence — knowledge base +- trading-2dexy — full Xcode project (iOS/macOS crypto trading app) +- trading-platform — (in development) +- fleet-bus — Machine.Machine fleet message bus + +## Skills Available (18 installed) +clinical-pathway, patient-pathway, quantum-trading, trading-app-dev, 2dexy-sync, +ml-training, xcode-remote, agency-agents, m2-memory, rlm-memory, harness-engine, +intent-elicit, intent-router, spec-discovery, unified-search, mm-pdf, build-verify, +cursor-agent diff --git a/scripts/ingest/memory_api_fix.py b/scripts/ingest/memory_api_fix.py new file mode 100644 index 0000000..950d508 --- /dev/null +++ b/scripts/ingest/memory_api_fix.py @@ -0,0 +1,650 @@ +"""Main Agent Memory class - the primary interface.""" + +import logging +from datetime import datetime, timedelta +from typing import Any +from uuid import UUID + +import redis.asyncio as redis +from qdrant_client import AsyncQdrantClient, models +from qdrant_client.models import ( + NamedSparseVector, + NamedVector, + PointStruct, + SparseVector, +) + +from config import get_settings +from embeddings import BGEm3Client +from hybrid_search import HybridSearcher +from models import ( + Memory, + MemoryCreate, + MemorySource, + MemoryType, + SearchQuery, + SearchResult, +) +from routing.router import QueryRouter +from routing.stats_store import RoutingStatsStore + +logger = logging.getLogger(__name__) + + +class AgentMemory: + """ + Main interface for agent memory operations. + + Provides: + - Store memories with automatic embedding + - Hybrid search (dense + sparse) + - Time-based retrieval + - Importance decay + - Memory consolidation + """ + + def __init__( + self, + agent_id: str, + qdrant: AsyncQdrantClient | None = None, + embedding_client: BGEm3Client | None = None, + redis_client: redis.Redis | None = None, + ): + self.agent_id = agent_id + self.settings = get_settings() + + # Initialize clients + self._qdrant = qdrant + self._embeddings = embedding_client + self._redis = redis_client + self._searcher: HybridSearcher | None = None + self._router: QueryRouter | None = None + # Only close clients this instance created — not externally-shared ones + self._owns_qdrant = qdrant is None + self._owns_embeddings = embedding_client is None + self._owns_redis = redis_client is None + + self._initialized = False + + async def _ensure_initialized(self): + """Lazy initialization of clients.""" + if self._initialized: + return + + # Qdrant client + if self._qdrant is None: + self._qdrant = AsyncQdrantClient( + url=self.settings.qdrant_url, + api_key=self.settings.qdrant_api_key, + ) + + # Redis client (optional) + if self._redis is None and self.settings.cache_embeddings: + try: + self._redis = redis.from_url(self.settings.redis_url) + await self._redis.ping() + except Exception as e: + logger.warning(f"Redis connection failed, caching disabled: {e}") + self._redis = None + + # Embedding client + if self._embeddings is None: + self._embeddings = BGEm3Client(redis_client=self._redis) + + # Hybrid searcher + self._searcher = HybridSearcher( + qdrant=self._qdrant, + embedding_client=self._embeddings, + ) + + # Query router (M4 smart routing) + stats_store = None + if self._redis: + stats_store = RoutingStatsStore(self._redis) + self._router = QueryRouter( + qdrant=self._qdrant, + embedding_client=self._embeddings, + stats_store=stats_store, + ) + + self._initialized = True + + async def close(self): + """Close connections owned by this instance (not externally-shared clients).""" + if self._embeddings and self._owns_embeddings: + await self._embeddings.close() + if self._redis and self._owns_redis: + await self._redis.close() + if self._qdrant and self._owns_qdrant: + await self._qdrant.close() + + async def __aenter__(self): + await self._ensure_initialized() + return self + + async def __aexit__(self, *args): + await self.close() + + # ========================================================================= + # STORE OPERATIONS + # ========================================================================= + + async def store( + self, + content: str, + memory_type: MemoryType = MemoryType.EPISODIC, + importance: float | None = None, + source: MemorySource = MemorySource.CONVERSATION, + entities: list[str] | None = None, + session_id: str | None = None, + user_id: str | None = None, + language: str = "en", + metadata: dict[str, Any] | None = None, + ) -> Memory: + """ + Store a new memory with automatic embedding. + + Args: + content: The memory content + memory_type: Type of memory (episodic, semantic, working) + importance: Importance score (0-1), defaults to settings default + source: Source of the memory + entities: Extracted entities + session_id: Session ID for episodic memories + user_id: Associated user ID + language: ISO language code + metadata: Additional metadata + + Returns: + The created Memory object + """ + await self._ensure_initialized() + + # Create memory object + memory = Memory( + content=content, + memory_type=memory_type, + agent_id=self.agent_id, + session_id=session_id, + user_id=user_id, + importance=importance or self.settings.default_importance, + initial_importance=importance or self.settings.default_importance, + source=source, + entities=entities or [], + language=language, + metadata=metadata or {}, + ) + + # Episodic memories start unconsolidated + if memory_type == MemoryType.EPISODIC: + memory.consolidated = False + + # Generate embeddings + embedding_response = await self._embeddings.embed_full(content) + + memory.has_dense = embedding_response.dense is not None + memory.has_sparse = embedding_response.sparse is not None + memory.has_colbert = embedding_response.colbert is not None + + # Build vectors dict for Qdrant + vectors = {} + + if embedding_response.dense: + vectors["dense"] = embedding_response.dense + + # Prepare point + point = PointStruct( + id=str(memory.id), + vector=vectors, + payload=memory.model_dump(mode="json"), + ) + + # If sparse embeddings, add as named sparse vector + if embedding_response.sparse: + sparse_indices = list(embedding_response.sparse.keys()) + sparse_values = list(embedding_response.sparse.values()) + + # Qdrant requires sparse vectors in a specific format + point.vector["sparse"] = SparseVector( + indices=sparse_indices, + values=sparse_values, + ) + + # Upsert to Qdrant + await self._qdrant.upsert( + collection_name=self.settings.collection_name, + points=[point], + ) + + logger.info(f"Stored memory {memory.id} for agent {self.agent_id}") + + return memory + + async def store_batch( + self, + memories: list[MemoryCreate], + ) -> list[Memory]: + """Store multiple memories efficiently.""" + await self._ensure_initialized() + + if not memories: + return [] + + # Create memory objects + memory_objects = [] + contents = [] + + for mem_create in memories: + memory = Memory( + content=mem_create.content, + memory_type=mem_create.memory_type, + agent_id=self.agent_id, + session_id=mem_create.session_id, + user_id=mem_create.user_id, + importance=mem_create.importance, + initial_importance=mem_create.importance, + source=mem_create.source, + entities=mem_create.entities, + language=mem_create.language, + metadata=mem_create.metadata, + ) + # Episodic memories start unconsolidated + if mem_create.memory_type == MemoryType.EPISODIC: + memory.consolidated = False + memory_objects.append(memory) + contents.append(mem_create.content) + + # Batch embed + embedding_responses = await self._embeddings.embed_batch( + contents, + include_sparse=True, + include_colbert=False, + ) + + # Build points + points = [] + for memory, emb_response in zip(memory_objects, embedding_responses): + memory.has_dense = emb_response.dense is not None + memory.has_sparse = emb_response.sparse is not None + + vectors = {} + if emb_response.dense: + vectors["dense"] = emb_response.dense + + point = PointStruct( + id=str(memory.id), + vector=vectors, + payload=memory.model_dump(mode="json"), + ) + + if emb_response.sparse: + point.vector["sparse"] = SparseVector( + indices=list(emb_response.sparse.keys()), + values=list(emb_response.sparse.values()), + ) + + points.append(point) + + # Batch upsert + await self._qdrant.upsert( + collection_name=self.settings.collection_name, + points=points, + ) + + logger.info(f"Stored {len(points)} memories for agent {self.agent_id}") + + return memory_objects + + # ========================================================================= + # SEARCH OPERATIONS + # ========================================================================= + + async def search( + self, + query: str, + memory_types: list[MemoryType] | None = None, + limit: int = 10, + min_importance: float = 0.0, + session_id: str | None = None, + user_id: str | None = None, + after: datetime | None = None, + before: datetime | None = None, + routing_strategy: str | None = None, + ) -> list[SearchResult]: + """ + Search memories using hybrid search. + + Args: + query: Search query text + memory_types: Filter by memory types + limit: Maximum results + min_importance: Minimum importance threshold + session_id: Filter by session + user_id: Filter by user + after: Only memories after this time + before: Only memories before this time + routing_strategy: Smart routing strategy ('auto', 'lookup', + 'standard', 'deep', 'synthesis'). None = legacy hybrid search. + + Returns: + List of SearchResult with relevance scores + """ + await self._ensure_initialized() + + # --- Smart routing path (M4) --- + if routing_strategy is not None: + routing_result = await self._router.route( + query=query, + agent_id=self.agent_id, + strategy=routing_strategy, + memory_types=memory_types, + limit=limit, + min_importance=min_importance, + session_id=session_id, + user_id=user_id, + after=after, + before=before, + ) + results = routing_result.strategy_result.results + for result in results: + await self._update_access(result.memory.id) + return results + + # --- Legacy hybrid search path (default, backward compat) --- + search_query = SearchQuery( + query=query, + memory_types=memory_types, + limit=limit, + min_importance=min_importance, + session_id=session_id, + user_id=user_id, + after=after, + before=before, + ) + + results = await self._searcher.search(search_query, self.agent_id) + + # Update access count and last_accessed for retrieved memories + for result in results: + await self._update_access(result.memory.id) + + return results + + async def _update_access(self, memory_id: UUID): + """Update access tracking for a memory.""" + try: + await self._qdrant.set_payload( + collection_name=self.settings.collection_name, + payload={ + "last_accessed": datetime.utcnow().isoformat(), + }, + points=[str(memory_id)], + ) + except Exception as e: + logger.warning(f"Failed to update access for {memory_id}: {e}") + + # ========================================================================= + # RETRIEVAL OPERATIONS + # ========================================================================= + + async def get(self, memory_id: UUID) -> Memory | None: + """Get a specific memory by ID.""" + await self._ensure_initialized() + + results = await self._qdrant.retrieve( + collection_name=self.settings.collection_name, + ids=[str(memory_id)], + with_payload=True, + ) + + if results: + return Memory(**results[0].payload) + return None + + async def get_recent( + self, + memory_type: MemoryType | None = None, + hours: int = 24, + limit: int = 50, + session_id: str | None = None, + ) -> list[Memory]: + """Get recent memories by time.""" + await self._ensure_initialized() + + # Build filter + must_conditions = [ + models.FieldCondition( + key="agent_id", + match=models.MatchValue(value=self.agent_id), + ), + models.FieldCondition( + key="timestamp", + range=models.Range( + gte=(datetime.utcnow() - timedelta(hours=hours)).timestamp(), + ), + ), + ] + + if memory_type: + must_conditions.append( + models.FieldCondition( + key="memory_type", + match=models.MatchValue(value=memory_type.value), + ) + ) + + if session_id: + must_conditions.append( + models.FieldCondition( + key="session_id", + match=models.MatchValue(value=session_id), + ) + ) + + # Scroll through results + results, _ = await self._qdrant.scroll( + collection_name=self.settings.collection_name, + scroll_filter=models.Filter(must=must_conditions), + limit=limit, + with_payload=True, + order_by=models.OrderBy( + key="timestamp", + direction=models.Direction.DESC, + ), + ) + + return [Memory(**r.payload) for r in results] + + async def get_by_entities( + self, + entities: list[str], + memory_type: MemoryType | None = None, + limit: int = 20, + ) -> list[Memory]: + """Get memories containing specific entities.""" + await self._ensure_initialized() + + must_conditions = [ + models.FieldCondition( + key="agent_id", + match=models.MatchValue(value=self.agent_id), + ), + ] + + # Match any of the entities + for entity in entities: + must_conditions.append( + models.FieldCondition( + key="entities", + match=models.MatchValue(value=entity), + ) + ) + + if memory_type: + must_conditions.append( + models.FieldCondition( + key="memory_type", + match=models.MatchValue(value=memory_type.value), + ) + ) + + results, _ = await self._qdrant.scroll( + collection_name=self.settings.collection_name, + scroll_filter=models.Filter(must=must_conditions), + limit=limit, + with_payload=True, + ) + + return [Memory(**r.payload) for r in results] + + # ========================================================================= + # UPDATE OPERATIONS + # ========================================================================= + + async def update_importance( + self, + memory_id: UUID, + importance: float, + ): + """Update importance score for a memory.""" + await self._ensure_initialized() + + await self._qdrant.set_payload( + collection_name=self.settings.collection_name, + payload={"importance": max(0.0, min(1.0, importance))}, + points=[str(memory_id)], + ) + + async def reinforce(self, memory_id: UUID, boost: float = 0.1): + """Reinforce a memory (increase importance).""" + memory = await self.get(memory_id) + if memory: + new_importance = min(1.0, memory.importance + boost) + await self.update_importance(memory_id, new_importance) + + async def decay_importance(self, days_old: int = 7): + """Apply importance decay to old memories.""" + await self._ensure_initialized() + + cutoff = datetime.utcnow() - timedelta(days=days_old) + + # Find old memories + results, _ = await self._qdrant.scroll( + collection_name=self.settings.collection_name, + scroll_filter=models.Filter( + must=[ + models.FieldCondition( + key="agent_id", + match=models.MatchValue(value=self.agent_id), + ), + models.FieldCondition( + key="timestamp", + range=models.Range(lte=cutoff.timestamp()), + ), + ] + ), + limit=1000, + with_payload=True, + ) + + decay_rate = self.settings.importance_decay_rate + + for point in results: + memory = Memory(**point.payload) + days_since = (datetime.utcnow() - memory.timestamp).days + decayed = memory.initial_importance * (1 - decay_rate * days_since) + decayed = max(0.05, decayed) # Minimum importance + + if abs(decayed - memory.importance) > 0.01: + await self.update_importance(memory.id, decayed) + + # ========================================================================= + # DELETE OPERATIONS + # ========================================================================= + + async def delete(self, memory_id: UUID) -> bool: + """Delete a specific memory.""" + await self._ensure_initialized() + + await self._qdrant.delete( + collection_name=self.settings.collection_name, + points_selector=models.PointIdsList( + points=[str(memory_id)], + ), + ) + return True + + async def delete_session(self, session_id: str) -> int: + """Delete all memories from a session.""" + await self._ensure_initialized() + + result = await self._qdrant.delete( + collection_name=self.settings.collection_name, + points_selector=models.FilterSelector( + filter=models.Filter( + must=[ + models.FieldCondition( + key="agent_id", + match=models.MatchValue(value=self.agent_id), + ), + models.FieldCondition( + key="session_id", + match=models.MatchValue(value=session_id), + ), + ] + ) + ), + ) + return result.status + + async def clear_all(self): + """Delete all memories for this agent. Use with caution!""" + await self._ensure_initialized() + + await self._qdrant.delete( + collection_name=self.settings.collection_name, + points_selector=models.FilterSelector( + filter=models.Filter( + must=[ + models.FieldCondition( + key="agent_id", + match=models.MatchValue(value=self.agent_id), + ), + ] + ) + ), + ) + logger.warning(f"Cleared all memories for agent {self.agent_id}") + + # ========================================================================= + # STATS + # ========================================================================= + + async def count( + self, + memory_type: MemoryType | None = None, + ) -> int: + """Count memories for this agent.""" + await self._ensure_initialized() + + must_conditions = [ + models.FieldCondition( + key="agent_id", + match=models.MatchValue(value=self.agent_id), + ), + ] + + if memory_type: + must_conditions.append( + models.FieldCondition( + key="memory_type", + match=models.MatchValue(value=memory_type.value), + ) + ) + + result = await self._qdrant.count( + collection_name=self.settings.collection_name, + count_filter=models.Filter(must=must_conditions), + ) + + return result.count diff --git a/scripts/ingest/push_to_memory.py b/scripts/ingest/push_to_memory.py new file mode 100644 index 0000000..7477df1 --- /dev/null +++ b/scripts/ingest/push_to_memory.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +""" +Push a telegram_export.py JSONL file into the memory API. + +Usage: + python3 push_to_memory.py \ + --agent-id nasr \ + --api http://172.18.0.20:8000 \ + [--dry-run] [--filter nasr,gmi,clinic] +""" +import argparse +import json +import sys +import time +import urllib.request +import urllib.error + +GMI_KEYWORDS = { + "gmi", "clinic", "cancer", "pathway", "oncology", "breast", "nsclc", + "colorectal", "clinical", "patient", "diagnosis", "treatment", "smithers", + "nasr", "salman", "medical", "physician", "physician-scientist", + "trading", "2dexy", "dexy", "quantum", "crypto" +} + +def importance(row: dict) -> float: + sender = (row.get("metadata", {}).get("sender") or "").lower() + content = row.get("content", "").lower() + base = 0.5 + if sender == "nasr salman": + base = 0.75 + elif sender in ("m2", "mar!0"): + base = 0.55 + if any(kw in content for kw in GMI_KEYWORDS): + base = min(base + 0.2, 0.9) + return round(base, 2) + +def entities_from(row: dict) -> list[str]: + ents = list(row.get("entities", [])) + sender = row.get("metadata", {}).get("sender") + if sender: + ents.append(sender.lower().replace(" ", "-")) + return list(set(ents))[:10] + +def store(api_url: str, payload: dict) -> bool: + data = json.dumps(payload).encode() + req = urllib.request.Request( + f"{api_url}/memory/store", + data=data, + headers={"Content-Type": "application/json"}, + method="POST" + ) + try: + with urllib.request.urlopen(req, timeout=15) as resp: + return resp.status == 200 + except urllib.error.HTTPError as e: + print(f" HTTP {e.code}: {e.read()[:100]}", file=sys.stderr) + return False + except Exception as e: + print(f" Error: {e}", file=sys.stderr) + return False + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("input", help="JSONL file from telegram_export.py") + ap.add_argument("--agent-id", default="nasr") + ap.add_argument("--api", default="http://172.18.0.20:8000") + ap.add_argument("--dry-run", action="store_true") + ap.add_argument("--filter", default="", help="comma-separated keywords to filter for") + ap.add_argument("--min-length", type=int, default=20, help="min content length to ingest") + ap.add_argument("--batch-delay", type=float, default=0.05, help="seconds between requests") + args = ap.parse_args() + + filter_kws = set(k.strip().lower() for k in args.filter.split(",") if k.strip()) + + rows = [] + with open(args.input) as f: + for line in f: + line = line.strip() + if line: + rows.append(json.loads(line)) + + # Filter + def passes(row: dict) -> bool: + content = row.get("content", "") + if not content or len(content) < args.min_length: + return False + if row.get("metadata", {}).get("is_service"): + return False + if filter_kws: + return any(kw in content.lower() for kw in filter_kws) + return True + + filtered = [r for r in rows if passes(r)] + print(f"Rows: {len(rows)} total → {len(filtered)} after filter (min_len={args.min_length}, filter={filter_kws or 'none'})") + + if args.dry_run: + print("DRY RUN — first 3 payloads:") + for r in filtered[:3]: + payload = { + "content": r["content"][:300], + "agent_id": args.agent_id, + "memory_type": r.get("memory_type", "episodic"), + "importance": importance(r), + "entities": entities_from(r), + "metadata": r.get("metadata", {}), + } + print(json.dumps(payload, indent=2)[:400]) + return + + ok = 0 + fail = 0 + for i, row in enumerate(filtered): + payload = { + "content": row["content"], + "agent_id": args.agent_id, + "memory_type": row.get("memory_type", "episodic"), + "importance": importance(row), + "entities": entities_from(row), + "metadata": row.get("metadata", {}), + } + if store(args.api, payload): + ok += 1 + else: + fail += 1 + if (i + 1) % 100 == 0: + print(f" [{i+1}/{len(filtered)}] ok={ok} fail={fail}") + if args.batch_delay: + time.sleep(args.batch_delay) + + print(f"Done: {ok} stored, {fail} failed out of {len(filtered)}") + +if __name__ == "__main__": + main() diff --git a/scripts/ingest/telegram_export.py b/scripts/ingest/telegram_export.py new file mode 100644 index 0000000..c94ab3a --- /dev/null +++ b/scripts/ingest/telegram_export.py @@ -0,0 +1,531 @@ +""" +Parse a Telegram Desktop HTML export (e.g. parlobyg.zip) into a JSONL file +whose rows match the agent.memory.system Memory schema. + +Stdlib only — no bs4/lxml dependency. The export structure is regular enough +that a depth-tracking HTMLParser handles it. + +Usage: + python -m ingest.telegram_export \ + [--agent-id m2] [--chat-id -1003815414577] [--chat-slug parlobyg] + +The zip can be a Telegram Desktop "Export chat history" archive OR an already- +extracted directory containing messages*.html. +""" + +from __future__ import annotations + +import argparse +import hashlib +import html +import json +import os +import re +import sys +import tempfile +import uuid +import zipfile +from dataclasses import dataclass, field +from datetime import datetime, timezone, timedelta +from html.parser import HTMLParser +from pathlib import Path +from typing import Iterator + + +# --------------------------------------------------------------------------- +# Message model (flat, pre-serialization) +# --------------------------------------------------------------------------- + + +@dataclass +class TGMessage: + message_id: str + html_file: str + sender: str | None + timestamp: datetime | None + text: str + is_service: bool + is_joined: bool # continuation of prior sender (no userpic/from_name) + reply_to_id: str | None + mentions: list[str] + attachments: list[str] + reactions: list[tuple[str, int]] + forwarded_from: str | None + + +# --------------------------------------------------------------------------- +# HTML parser +# --------------------------------------------------------------------------- + + +_TS_RE = re.compile( + r"(\d{2})\.(\d{2})\.(\d{4})\s+(\d{2}):(\d{2}):(\d{2})\s+UTC([+\-]\d{2}:\d{2})?" +) +_REPLY_ID_RE = re.compile(r"GoToMessage\((\d+)\)") + + +def _parse_title_ts(title: str) -> datetime | None: + """Parse a '.date.details' title like '15.03.2026 17:10:23 UTC+01:00'.""" + m = _TS_RE.search(title or "") + if not m: + return None + dd, mm, yyyy, hh, mi, ss, tz = m.groups() + if tz: + sign = 1 if tz[0] == "+" else -1 + hrs, mins = tz[1:].split(":") + offset = timezone(sign * timedelta(hours=int(hrs), minutes=int(mins))) + else: + offset = timezone.utc + return datetime( + int(yyyy), int(mm), int(dd), + int(hh), int(mi), int(ss), + tzinfo=offset, + ) + + +class _TelegramExportParser(HTMLParser): + """ + Walks a messages*.html file and yields one TGMessage per
. + + Works via depth tracking: when we enter a top-level message div, start a + new collection buffer; when it closes, emit. + """ + + def __init__(self, html_file: str): + super().__init__(convert_charrefs=True) + self.html_file = html_file + self.messages: list[TGMessage] = [] + + # outer stack of (tag, class, attrs) + self._stack: list[tuple[str, str, dict]] = [] + # index into self._stack where the current message div starts, or None + self._msg_depth: int | None = None + # current message being built + self._cur: dict | None = None + # which sub-section we're currently capturing text into + # one of: None | "from_name" | "text" | "reply_to" | "reaction_emoji" + # | "forwarded_from" | "service_body" + self._capture: str | None = None + self._capture_buf: list[str] = [] + # reaction accumulator: last seen emoji waiting for count (user-pics length is count) + self._pending_reactions: list[tuple[str, int]] = [] + self._reaction_userpic_depth: int | None = None + self._current_reaction_emoji: str | None = None + self._current_reaction_count: int = 0 + # track seen sender to fill in "joined" continuations + self._last_sender: str | None = None + + # -- helpers ----------------------------------------------------------- + + def _classes(self, attrs: dict) -> set[str]: + return set((attrs.get("class") or "").split()) + + def _in_message(self) -> bool: + return self._msg_depth is not None + + def _flush_text(self) -> str: + s = "".join(self._capture_buf) + self._capture_buf.clear() + return s + + # -- HTMLParser hooks -------------------------------------------------- + + def handle_starttag(self, tag, attrs): + attr_d = dict(attrs) + self._stack.append((tag, attr_d.get("class", ""), attr_d)) + classes = self._classes(attr_d) + + if tag == "div" and "message" in classes and attr_d.get("id", "").startswith("message"): + self._msg_depth = len(self._stack) - 1 + mid = attr_d["id"][len("message"):] + self._cur = { + "message_id": mid, + "html_file": self.html_file, + "sender": None, + "timestamp": None, + "text_parts": [], + "is_service": "service" in classes, + "is_joined": "joined" in classes, + "reply_to_id": None, + "mentions": [], + "attachments": [], + "reactions": [], + "forwarded_from": None, + } + return + + if not self._in_message() or self._cur is None: + return + + # inside a message + if tag == "div": + if "from_name" in classes: + self._capture = "from_name" + self._capture_buf.clear() + elif "pull_right" in classes and "date" in classes and "details" in classes: + ts = _parse_title_ts(attr_d.get("title", "")) + if ts: + self._cur["timestamp"] = ts + elif "reply_to" in classes and "details" in classes: + self._capture = "reply_to" + self._capture_buf.clear() + elif "text" in classes and self._capture != "from_name": + self._capture = "text" + self._capture_buf.clear() + elif "forwarded" in classes and "body" in classes: + # forwarded block; the inner .from_name capture will fire + self._cur["forwarded_from"] = "" # sentinel: we saw a forward + elif "body" in classes and "details" in classes and self._cur["is_service"]: + # service message body + self._capture = "service_body" + self._capture_buf.clear() + elif tag == "span": + if "emoji" in classes and self._capture != "text": + # reaction emoji + self._capture = "reaction_emoji" + self._capture_buf.clear() + elif "userpics" in classes: + self._reaction_userpic_depth = len(self._stack) - 1 + self._current_reaction_count = 0 + elif tag == "a": + if self._capture == "text": + # mention link + onclick = attr_d.get("onclick", "") or "" + if "ShowMentionName" in onclick: + # keep accumulating; the link text lands in capture buffer + pass + if self._capture == "reply_to": + m = _REPLY_ID_RE.search(attr_d.get("onclick", "") or "") + if m: + self._cur["reply_to_id"] = m.group(1) + href = attr_d.get("href", "") or "" + if href.startswith(("photos/", "files/", "voice_messages/", "video_files/", "round_video_messages/", "stickers/")): + self._cur["attachments"].append(href) + elif tag == "div" and self._reaction_userpic_depth is not None: + # count userpic divs inside a reactions .userpics (one per reactor) + pass + + if tag == "div" and self._reaction_userpic_depth is not None: + classes = self._classes(attr_d) + if "userpic" in classes: + self._current_reaction_count += 1 + + if tag == "br" and self._capture == "text": + self._capture_buf.append("\n") + + def handle_endtag(self, tag): + if not self._stack: + return + # pop current tag frame + self._stack.pop() + depth = len(self._stack) + + # close of the message div? + if self._msg_depth is not None and depth == self._msg_depth and self._cur is not None: + # finalize + sender = self._cur["sender"] + if self._cur["is_joined"] and not sender: + sender = self._last_sender + if sender: + self._last_sender = sender + text = "".join(self._cur["text_parts"]).strip() + ts = self._cur["timestamp"] + self.messages.append(TGMessage( + message_id=self._cur["message_id"], + html_file=self._cur["html_file"], + sender=sender, + timestamp=ts, + text=text, + is_service=self._cur["is_service"], + is_joined=self._cur["is_joined"], + reply_to_id=self._cur["reply_to_id"], + mentions=self._cur["mentions"], + attachments=self._cur["attachments"], + reactions=self._cur["reactions"], + forwarded_from=self._cur["forwarded_from"] or None, + )) + self._cur = None + self._msg_depth = None + self._capture = None + self._capture_buf.clear() + return + + if self._reaction_userpic_depth is not None and depth == self._reaction_userpic_depth: + # closed a userpics span — commit reaction + if self._cur and self._current_reaction_emoji: + self._cur["reactions"].append( + (self._current_reaction_emoji, max(1, self._current_reaction_count)) + ) + self._current_reaction_emoji = None + self._current_reaction_count = 0 + self._reaction_userpic_depth = None + return + + # close of a capturing section? + if self._capture == "from_name" and tag == "div": + text = self._flush_text().strip() + if self._cur and self._cur["forwarded_from"] == "": + # forwarded header fills forwarded_from, not sender + self._cur["forwarded_from"] = text + elif self._cur: + self._cur["sender"] = text + self._capture = None + elif self._capture == "text" and tag == "div": + # finalize text + raw = self._flush_text() + if self._cur: + self._cur["text_parts"].append(raw) + self._capture = None + elif self._capture == "reply_to" and tag == "div": + self._capture = None + self._capture_buf.clear() + elif self._capture == "reaction_emoji" and tag == "span": + self._current_reaction_emoji = self._flush_text().strip() + self._capture = None + elif self._capture == "service_body" and tag == "div": + svc = self._flush_text().strip() + if self._cur: + self._cur["text_parts"].append(svc) + self._capture = None + + def handle_data(self, data): + if self._capture: + self._capture_buf.append(data) + + +def parse_html_file(path: Path) -> list[TGMessage]: + with open(path, encoding="utf-8") as f: + raw = f.read() + parser = _TelegramExportParser(html_file=path.name) + parser.feed(raw) + parser.close() + return parser.messages + + +# --------------------------------------------------------------------------- +# Export directory handling +# --------------------------------------------------------------------------- + + +def _sorted_message_htmls(root: Path) -> list[Path]: + """messages.html, messages2.html, messages3.html … in the order TG exports them.""" + files = list(root.glob("messages*.html")) + + def key(p: Path) -> int: + stem = p.stem # "messages" or "messages3" + num = stem[len("messages"):] + return int(num) if num.isdigit() else 1 + + return sorted(files, key=key) + + +def _resolve_export_dir(source: Path) -> tuple[Path, tempfile.TemporaryDirectory | None]: + """Return a directory containing messages*.html, extracting the zip if needed.""" + if source.is_dir(): + return source, None + + if not source.exists() or source.suffix.lower() != ".zip": + raise SystemExit(f"not a zip or directory: {source}") + + tmp = tempfile.TemporaryDirectory(prefix="tg_export_") + with zipfile.ZipFile(source) as z: + z.extractall(tmp.name) + + # look for messages.html at any depth + root = Path(tmp.name) + candidates = list(root.rglob("messages.html")) + if not candidates: + raise SystemExit(f"no messages.html inside {source}") + export_root = candidates[0].parent + return export_root, tmp + + +# --------------------------------------------------------------------------- +# Memory-row builder +# --------------------------------------------------------------------------- + + +def _deterministic_uuid(*parts: str) -> str: + h = hashlib.sha256("||".join(parts).encode()).hexdigest() + # Craft a UUIDv4-shaped string from the hash so qdrant accepts it. + return str(uuid.UUID(h[:32])) + + +def _importance(msg: TGMessage) -> float: + if msg.is_service: + return 0.15 + if not msg.text: + return 0.25 if msg.attachments else 0.1 + n = len(msg.text) + if n < 4: + return 0.25 + if n < 20: + return 0.35 + if n < 200: + return 0.5 + return 0.6 + + +def _content(msg: TGMessage, chat_title: str) -> str: + """Human+embedding friendly line: 'Sender: text' plus hints.""" + sender = msg.sender or "(unknown)" + body = msg.text or "" + if msg.is_service: + return f"[service] {body}" + if msg.attachments and not body: + body = f"(attachment: {', '.join(msg.attachments[:3])})" + prefix = f"{sender}" + if msg.forwarded_from: + prefix += f" (forwarded from {msg.forwarded_from})" + return f"{prefix}: {body}".strip() + + +def _entities(msg: TGMessage) -> list[str]: + ents: list[str] = [] + if msg.sender: + ents.append(msg.sender) + ents.extend(msg.mentions) + # pull @handles and #tags from the body + if msg.text: + ents.extend(re.findall(r"@[A-Za-z0-9_]+", msg.text)) + ents.extend(re.findall(r"#\w+", msg.text)) + # dedupe, keep order + seen = set() + out = [] + for e in ents: + k = e.lower() + if k in seen: + continue + seen.add(k) + out.append(e) + return out + + +def messages_to_rows( + messages: list[TGMessage], + *, + agent_id: str, + chat_title: str, + chat_id: str, + chat_slug: str, + skip_service: bool = False, +) -> Iterator[dict]: + for msg in messages: + if skip_service and msg.is_service: + continue + ts = msg.timestamp or datetime(2026, 3, 15, tzinfo=timezone.utc) + row_id = _deterministic_uuid(chat_id, msg.html_file, msg.message_id) + yield { + "id": row_id, + "content": _content(msg, chat_title), + "memory_type": "episodic", + "agent_id": agent_id, + "importance": _importance(msg), + "initial_importance": _importance(msg), + "timestamp": ts.astimezone(timezone.utc).isoformat(), + "entities": _entities(msg), + "session_id": f"tg:{chat_slug}", + "metadata": { + "chat_title": chat_title, + "chat_id": chat_id, + "chat_slug": chat_slug, + "tg_message_id": msg.message_id, + "html_file": msg.html_file, + "sender": msg.sender, + "reply_to_tg_id": msg.reply_to_id, + "mentions": msg.mentions, + "attachments": msg.attachments, + "reactions": [[e, c] for e, c in msg.reactions], + "forwarded_from": msg.forwarded_from, + "is_service": msg.is_service, + "is_joined": msg.is_joined, + }, + "consolidated": False, + "consolidated_into": [], + "consolidation_batch_id": None, + "retrieval_count": 0, + "utilization_count": 0, + "outcome_count": 0, + "last_retrieved": None, + "last_utilized": None, + "last_boosted": None, + "importance_history": [_importance(msg)], + "boost_cooldown_until": None, + "has_colbert": False, + "colbert_token_count": 0, + "_source": "telegram_export", + "_source_file": msg.html_file, + } + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("source", help="path to .zip export or extracted directory") + p.add_argument("output", help="output .jsonl path") + p.add_argument("--agent-id", default="m2") + p.add_argument("--chat-id", default="-1003815414577", + help="Telegram numeric chat id (recorded in metadata)") + p.add_argument("--chat-slug", default="parlobyg") + p.add_argument("--chat-title", default=None, + help="override auto-detected chat title") + p.add_argument("--skip-service", action="store_true", + help="drop service messages (joins/renames/etc.)") + args = p.parse_args(argv) + + source = Path(args.source).expanduser().resolve() + export_dir, tmp_holder = _resolve_export_dir(source) + + html_files = _sorted_message_htmls(export_dir) + if not html_files: + print(f"no messages*.html in {export_dir}", file=sys.stderr) + return 1 + + chat_title = args.chat_title + if not chat_title: + # extract from first messages.html
+ try: + head = html_files[0].read_text(encoding="utf-8") + m = re.search(r'class="text bold"\s*>\s*([^<]+?)\s*<', head) + if m: + chat_title = html.unescape(m.group(1)).strip() + except Exception: + pass + chat_title = chat_title or args.chat_slug + + total = 0 + per_file: dict[str, int] = {} + out_path = Path(args.output).expanduser().resolve() + out_path.parent.mkdir(parents=True, exist_ok=True) + with open(out_path, "w", encoding="utf-8") as out: + for htmlf in html_files: + msgs = parse_html_file(htmlf) + rows = list(messages_to_rows( + msgs, + agent_id=args.agent_id, + chat_title=chat_title, + chat_id=args.chat_id, + chat_slug=args.chat_slug, + skip_service=args.skip_service, + )) + for r in rows: + out.write(json.dumps(r, ensure_ascii=False) + "\n") + total += len(rows) + per_file[htmlf.name] = len(rows) + print(f" {htmlf.name}: {len(rows)} rows", file=sys.stderr) + + print(f"wrote {total} rows to {out_path}", file=sys.stderr) + print(f" chat_title={chat_title!r} chat_id={args.chat_id} agent_id={args.agent_id}", + file=sys.stderr) + + if tmp_holder is not None: + tmp_holder.cleanup() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/telegram/.gitignore b/telegram/.gitignore new file mode 100644 index 0000000..2d8d574 --- /dev/null +++ b/telegram/.gitignore @@ -0,0 +1,7 @@ +# Large zips tracked via manifest.json, not committed to git +# machine.machine.zip (572KB) IS committed — small enough +m2.zip +parlobyg.zip +MuhlAI.zip +m2-devops.zip +love-travel Georg.zip diff --git a/telegram/machine.machine.zip b/telegram/machine.machine.zip new file mode 100644 index 0000000..42c1e19 Binary files /dev/null and b/telegram/machine.machine.zip differ diff --git a/telegram/manifest.json b/telegram/manifest.json new file mode 100644 index 0000000..9de19eb --- /dev/null +++ b/telegram/manifest.json @@ -0,0 +1,32 @@ +[ + { + "file": "MuhlAI.zip", + "size_mb": 219.1, + "md5": "7d49fb709f597e821c0f684c137b8fb9" + }, + { + "file": "love-travel Georg.zip", + "size_mb": 0.1, + "md5": "05a729d4d7c5159131ab5f988c66b0f2" + }, + { + "file": "m2-devops.zip", + "size_mb": 0.1, + "md5": "55f861aba705ae76e9bd9094360a0444" + }, + { + "file": "m2.zip", + "size_mb": 63.1, + "md5": "607a0d177badc2c425d653ca93a7165f" + }, + { + "file": "machine.machine.zip", + "size_mb": 0.6, + "md5": "ffdbe81f1205db409cedba9773def470" + }, + { + "file": "parlobyg.zip", + "size_mb": 40.4, + "md5": "ec59ae36b31a75bc28b1eac26d265637" + } +] \ No newline at end of file