From 4dae5477ce479ab9363dd75b6caf0dec23fbe745 Mon Sep 17 00:00:00 2001 From: Patrick Asmus Date: Fri, 3 Jul 2026 08:04:27 +0200 Subject: [PATCH] Add application, Dockerfile, CI workflow and documentation - FastAPI-based Matrix chat export tool with E2E support - Dockerfile with Python 3.12-slim base image - Docker Compose with dedicated network configuration - Gitea Actions workflow for automated image builds - README with setup instructions and feature overview - MIT license Co-Authored-By: Claude Opus 4.6 --- .gitea/workflows/docker-publish.yml | 38 ++ Dockerfile | 18 + LICENSE | 37 +- README.md | 58 +- app/__init__.py | 0 app/exporter.py | 837 ++++++++++++++++++++++++++++ app/main.py | 242 ++++++++ app/templates/chat_export.html | 266 +++++++++ app/templates/index.html | 654 ++++++++++++++++++++++ docker-compose.yml | 26 + requirements.txt | 7 + 11 files changed, 2161 insertions(+), 22 deletions(-) create mode 100644 .gitea/workflows/docker-publish.yml create mode 100644 Dockerfile create mode 100644 app/__init__.py create mode 100644 app/exporter.py create mode 100644 app/main.py create mode 100644 app/templates/chat_export.html create mode 100644 app/templates/index.html create mode 100644 docker-compose.yml create mode 100644 requirements.txt diff --git a/.gitea/workflows/docker-publish.yml b/.gitea/workflows/docker-publish.yml new file mode 100644 index 0000000..20b6396 --- /dev/null +++ b/.gitea/workflows/docker-publish.yml @@ -0,0 +1,38 @@ +name: Build & Push Docker Image + +on: + push: + tags: + - 'v*' + +jobs: + build-and-push: + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Version aus Tag extrahieren + id: version + run: | + VERSION=$(echo "${{ gitea.ref_name }}" | sed 's/^v//') + echo "VERSION=$VERSION" >> $GITHUB_OUTPUT + + - name: Bei Registry einloggen + run: | + echo "${{ secrets.REGISTRY_TOKEN }}" | \ + docker login ${{ vars.REGISTRY_URL }} \ + --username "${{ secrets.REGISTRY_USER }}" \ + --password-stdin + + - name: Docker Image bauen + run: | + docker build \ + -t ${{ vars.REGISTRY_URL }}/${{ gitea.repository }}:${{ steps.version.outputs.VERSION }} \ + -t ${{ vars.REGISTRY_URL }}/${{ gitea.repository }}:latest \ + . + + - name: Docker Image pushen + run: | + docker push ${{ vars.REGISTRY_URL }}/${{ gitea.repository }}:${{ steps.version.outputs.VERSION }} + docker push ${{ vars.REGISTRY_URL }}/${{ gitea.repository }}:latest diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c89bbb4 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.12-slim + +RUN apt-get update && apt-get install -y --no-install-recommends \ + libolm-dev gcc python3-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +COPY app/ ./app/ + +RUN mkdir -p /app/exports /app/store + +EXPOSE 8080 + +CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8080", "--log-level", "info"] diff --git a/LICENSE b/LICENSE index 100a0a5..9e6581c 100644 --- a/LICENSE +++ b/LICENSE @@ -1,26 +1,25 @@ -GNU AFFERO GENERAL PUBLIC LICENSE -Version 3, 19 November 2007 +MIT License ---- +Copyright (c) Patrick Asmus (scriptos) -Copyright © - -Name: Patrick Asmus (scriptos) -Email: support@techniverse.net Website: https://www.patrick-asmus.de Blog: https://www.cleveradmin.de +Email: support@techniverse.net ---- +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as published by -the Free Software Foundation, either version 3 of the License, or -(at your option) any later version. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md index dc14603..5f57e8c 100644 --- a/README.md +++ b/README.md @@ -4,10 +4,10 @@

-

Name des Projekts

+

Matrix Chat Export

- Kurzbeschreibung des Projekts/Anwendung, um die es geht + Web-basiertes Tool zum Exportieren von Matrix-Chaträumen inkl. Medien und E2E-Verschlüsselung

@@ -22,7 +22,59 @@

-CONTENT BEREICH +## Funktionen + +- **Matrix-Chaträume exportieren** — vollständiger Nachrichtenverlauf als HTML-Datei +- **Medien-Download** — Bilder, Videos, Audio, Dateien und Sticker werden mitgespeichert +- **E2E-Verschlüsselung** — Import von Megolm-Schlüsseln zum Entschlüsseln verschlüsselter Nachrichten +- **Inkrementeller Export** — bei erneutem Export werden nur neue Nachrichten geladen +- **ZIP-Download** — fertigen Export als ZIP-Archiv herunterladen +- **Zwei Login-Methoden** — Anmeldung per Access Token oder Benutzername/Passwort +- **Session-Wiederherstellung** — bestehende Sitzungen werden automatisch wiederhergestellt + +## Voraussetzungen + +- Docker & Docker Compose + +## Installation + +```bash +git clone https://git.techniverse.net/scriptos/matrix-chat-export.git +cd matrix-chat-export +docker compose up -d +``` + +Die Anwendung ist anschließend unter `http://:8092` erreichbar. + +## Konfiguration + +| Variable | Beschreibung | Standard | +|----------|-------------|----------| +| `TZ` | Zeitzone | `Europe/Berlin` | +| `LOG_LEVEL` | Log-Level (`DEBUG`, `INFO`, `WARNING`, `ERROR`) | `INFO` | + +### Volumes + +| Pfad (Container) | Beschreibung | +|-------------------|-------------| +| `/app/exports` | Gespeicherte Chat-Exports | +| `/app/store` | Session-Daten und importierte Schlüssel | + +## Verwendung + +1. Anwendung im Browser öffnen +2. Mit dem Matrix-Homeserver verbinden (Token oder Passwort) +3. Optional: E2E-Schlüssel importieren (für verschlüsselte Räume) +4. Raum auswählen und Export starten +5. Export als HTML ansehen oder als ZIP herunterladen + +> **Hinweis:** Es wird kein Admin-Token benötigt. Ein normaler Benutzer-Access-Token ist ausreichend. +> Die Anwendung greift ausschließlich auf Räume zu, in denen der Benutzer Mitglied ist. + +## Technologie + +- **Backend:** Python 3.12, FastAPI, matrix-nio +- **Container:** Docker mit Python-Slim Base Image

diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/exporter.py b/app/exporter.py new file mode 100644 index 0000000..be4691d --- /dev/null +++ b/app/exporter.py @@ -0,0 +1,837 @@ +import asyncio +import json +import logging +import hashlib +import zipfile +from pathlib import Path +from datetime import datetime, timezone +from typing import AsyncGenerator +import shutil +from urllib.parse import quote + +import aiohttp +import aiofiles + +from nio import ( + AsyncClient, + AsyncClientConfig, + LoginResponse, + LoginError, + RoomMessagesResponse, + RoomMessagesError, + MegolmEvent, + DownloadResponse, + DownloadError, +) + +logger = logging.getLogger(__name__) + +SENDER_COLORS = [ + "#ff6b6b", "#ffa502", "#7bed9f", "#70a1ff", "#5352ed", "#ff6348", + "#2ed573", "#1e90ff", "#ff4757", "#eccc68", "#a29bfe", "#fd79a8", + "#00cec9", "#e17055", "#6c5ce7", "#fab1a0", +] + + +def _color_for_user(user_id: str) -> str: + h = int(hashlib.md5(user_id.encode()).hexdigest()[:8], 16) + return SENDER_COLORS[h % len(SENDER_COLORS)] + + +def _ts_to_datetime(ts_ms: int) -> datetime: + return datetime.fromtimestamp(ts_ms / 1000, tz=timezone.utc) + + +def _format_size(size: int) -> str: + if size > 1_048_576: + return f"{size / 1_048_576:.1f} MB" + if size > 1024: + return f"{size / 1024:.0f} KB" + return f"{size} B" + + +EXT_MAP = { + "image/jpeg": ".jpg", "image/png": ".png", "image/gif": ".gif", + "image/webp": ".webp", "video/mp4": ".mp4", "video/webm": ".webm", + "video/quicktime": ".mov", "audio/ogg": ".ogg", "audio/mpeg": ".mp3", + "audio/mp4": ".m4a", "audio/webm": ".weba", "audio/aac": ".aac", + "application/pdf": ".pdf", +} + + +MEDIA_PURGED = "__PURGED__" + + +class MatrixExporter: + + def __init__(self, store_path: str, export_path: str): + self.store_path = Path(store_path) + self.export_path = Path(export_path) + self.client: AsyncClient | None = None + self.user_id: str | None = None + self.homeserver: str = "" + self.access_token: str = "" + self.credentials_file = self.store_path / "credentials.json" + self.store_path.mkdir(parents=True, exist_ok=True) + self.export_path.mkdir(parents=True, exist_ok=True) + self._keys_imported = False + + # ------------------------------------------------------------------ + # Connection + # ------------------------------------------------------------------ + + async def connect_password(self, homeserver: str, username: str, password: str) -> dict: + self.homeserver = homeserver.rstrip("/") + nio_store = self.store_path / "nio_store" + nio_store.mkdir(parents=True, exist_ok=True) + + cfg = AsyncClientConfig(store_sync_tokens=True, encryption_enabled=True) + self.client = AsyncClient(self.homeserver, username, store_path=str(nio_store), config=cfg) + + resp = await self.client.login(password) + if isinstance(resp, LoginError): + raise Exception(f"Login fehlgeschlagen: {resp.message}") + + self.user_id = resp.user_id + self.access_token = resp.access_token + + self.credentials_file.write_text(json.dumps({ + "homeserver": self.homeserver, + "user_id": resp.user_id, + "device_id": resp.device_id, + "access_token": resp.access_token, + }, indent=2)) + + return {"user_id": resp.user_id, "device_id": resp.device_id, "new_session": True} + + async def connect_token(self, homeserver: str, token: str) -> dict: + self.homeserver = homeserver.rstrip("/") + self.access_token = token + + async with aiohttp.ClientSession() as s: + async with s.get( + f"{self.homeserver}/_matrix/client/v3/account/whoami", + headers={"Authorization": f"Bearer {token}"}, + ) as r: + if r.status != 200: + raise Exception(f"Token ungültig (HTTP {r.status})") + data = await r.json() + self.user_id = data["user_id"] + device_id = data.get("device_id", "EXPORT") + + nio_store = self.store_path / "nio_store" + nio_store.mkdir(parents=True, exist_ok=True) + + cfg = AsyncClientConfig(store_sync_tokens=True, encryption_enabled=True) + self.client = AsyncClient( + self.homeserver, self.user_id, + store_path=str(nio_store), config=cfg, + ) + self.client.restore_login(self.user_id, device_id, token) + + self.credentials_file.write_text(json.dumps({ + "homeserver": self.homeserver, + "user_id": self.user_id, + "device_id": device_id, + "access_token": token, + }, indent=2)) + + return {"user_id": self.user_id, "device_id": device_id, "new_session": False} + + async def restore_session(self) -> dict | None: + if not self.credentials_file.exists(): + return None + creds = json.loads(self.credentials_file.read_text()) + self.homeserver = creds["homeserver"] + self.user_id = creds["user_id"] + self.access_token = creds["access_token"] + + nio_store = self.store_path / "nio_store" + nio_store.mkdir(parents=True, exist_ok=True) + cfg = AsyncClientConfig(store_sync_tokens=True, encryption_enabled=True) + self.client = AsyncClient( + self.homeserver, self.user_id, + store_path=str(nio_store), config=cfg, + ) + self.client.restore_login(creds["user_id"], creds["device_id"], creds["access_token"]) + return { + "user_id": self.user_id, + "device_id": creds["device_id"], + "homeserver": self.homeserver, + "restored": True, + } + + async def disconnect(self): + if self.client: + await self.client.close() + self.client = None + + # ------------------------------------------------------------------ + # E2E Key import + # ------------------------------------------------------------------ + + async def import_keys(self, key_file: str, passphrase: str) -> dict: + if not self.client: + raise Exception("Nicht verbunden") + self.client.load_store() + keys = await self.client.import_keys(key_file, passphrase) + self._keys_imported = True + return {"success": True, "message": f"Schlüssel importiert ({keys} Sessions)"} + + @property + def keys_imported(self) -> bool: + return self._keys_imported + + # ------------------------------------------------------------------ + # Room listing + # ------------------------------------------------------------------ + + async def _fetch_room_info(self, session: aiohttp.ClientSession, room_id: str, headers: dict) -> dict: + encoded = quote(room_id, safe="") + name = room_id + member_count = 0 + encrypted = False + topic = "" + members: list[str] = [] + + async with session.get( + f"{self.homeserver}/_matrix/client/v3/rooms/{encoded}/state", + headers=headers, + ) as r: + if r.status == 200: + states = await r.json() + for st in states: + t = st.get("type", "") + c = st.get("content", {}) + if t == "m.room.name": + name = c.get("name", room_id) + elif t == "m.room.encryption": + encrypted = True + elif t == "m.room.topic": + topic = c.get("topic", "") + elif t == "m.room.member" and c.get("membership") == "join": + member_count += 1 + if st.get("state_key") != self.user_id: + members.append(c.get("displayname", st.get("state_key", "?"))) + + if name == room_id and members: + name = ", ".join(members[:3]) + if len(members) > 3: + name += f" +{len(members) - 3}" + + return { + "room_id": room_id, + "name": name, + "member_count": member_count, + "encrypted": encrypted, + "topic": topic, + } + + async def list_rooms(self) -> list[dict]: + if not self.access_token: + raise Exception("Nicht verbunden") + + headers = {"Authorization": f"Bearer {self.access_token}"} + rooms = [] + + async with aiohttp.ClientSession() as s: + async with s.get(f"{self.homeserver}/_matrix/client/v3/joined_rooms", headers=headers) as r: + data = await r.json() + room_ids = data.get("joined_rooms", []) + + for room_id in room_ids: + try: + info = await self._fetch_room_info(s, room_id, headers) + rooms.append(info) + except Exception: + pass + + rooms.sort(key=lambda r: r["name"].lower()) + return rooms + + async def list_rooms_stream(self) -> AsyncGenerator[dict, None]: + if not self.access_token: + raise Exception("Nicht verbunden") + + headers = {"Authorization": f"Bearer {self.access_token}"} + + async with aiohttp.ClientSession() as s: + async with s.get(f"{self.homeserver}/_matrix/client/v3/joined_rooms", headers=headers) as r: + data = await r.json() + room_ids = data.get("joined_rooms", []) + + yield {"type": "total", "count": len(room_ids)} + + for room_id in room_ids: + try: + info = await self._fetch_room_info(s, room_id, headers) + yield {"type": "room", "data": info} + except Exception as e: + logger.warning("Raum-Info fehlgeschlagen für %s: %s", room_id, e) + + # ------------------------------------------------------------------ + # Export + # ------------------------------------------------------------------ + + async def export_room(self, room_id: str) -> AsyncGenerator[dict, None]: + if not self.client: + yield _log("error", "Nicht verbunden") + return + + safe_name = room_id.replace("!", "").replace(":", "_").replace(".", "_") + room_dir = self.export_path / safe_name + room_dir.mkdir(parents=True, exist_ok=True) + media_dir = room_dir / "media" + media_dir.mkdir(exist_ok=True) + + yield _log("info", "Starte Export …") + + # --- sync to get pagination token --- + yield _log("info", "Synchronisiere mit Server …", progress=2) + try: + sync_filter = json.dumps({ + "room": {"timeline": {"limit": 1}, "state": {"lazy_load_members": True}}, + "presence": {"types": []}, + "account_data": {"types": []}, + }) + await self.client.sync(timeout=30000, sync_filter=sync_filter) + start_token = self.client.next_batch + except Exception as e: + yield _log("error", f"Sync fehlgeschlagen: {e}") + return + + if not start_token: + yield _log("error", "Kein Sync-Token erhalten") + return + + # --- load state for incremental --- + state_file = room_dir / ".state.json" + existing_ids: set[str] = set() + existing_events: list[dict] = [] + events_file = room_dir / "events.jsonl" + + if events_file.exists(): + try: + for line in events_file.read_text(encoding="utf-8").splitlines(): + if line.strip(): + ev = json.loads(line) + existing_ids.add(ev.get("event_id", "")) + existing_events.append(ev) + yield _log("info", f"{len(existing_events)} vorhandene Events geladen (inkrementell)", progress=3) + except Exception: + existing_events = [] + existing_ids = set() + + # --- fetch messages --- + yield _log("info", "Lade Nachrichten …", progress=5) + + raw_events: list[dict] = [] + from_token = start_token + page = 0 + decrypted_count = 0 + encrypted_fail = 0 + + while True: + page += 1 + try: + resp = await self.client.room_messages(room_id, start=from_token, limit=200) + except Exception as e: + yield _log("warning", f"Fehler bei Seite {page}: {e}") + break + + if isinstance(resp, RoomMessagesError): + yield _log("error", f"API-Fehler: {resp.message}") + break + + if not resp.chunk: + break + + for event in resp.chunk: + source = getattr(event, "source", {}) + eid = source.get("event_id", "") + if eid in existing_ids: + continue + + if isinstance(event, MegolmEvent): + try: + decrypted = self.client.decrypt_event(event) + if decrypted: + source = decrypted.source + decrypted_count += 1 + else: + encrypted_fail += 1 + except Exception: + encrypted_fail += 1 + + raw_events.append(source) + existing_ids.add(eid) + + from_token = resp.end + if page % 3 == 0: + yield _log("progress", f"Seite {page}: {len(raw_events)} neue Nachrichten", progress=min(5 + page, 40)) + + if not resp.end: + break + + raw_events.reverse() + + yield _log("info", + f"{len(raw_events)} neue Nachrichten ({decrypted_count} entschlüsselt, " + f"{encrypted_fail} nicht entschlüsselbar)", progress=42) + + # --- merge with existing --- + all_events = existing_events + raw_events + all_events.sort(key=lambda e: e.get("origin_server_ts", 0)) + + # --- save events --- + async with aiofiles.open(events_file, "w", encoding="utf-8") as f: + for ev in all_events: + await f.write(json.dumps(ev, ensure_ascii=False) + "\n") + + yield _log("info", f"Gesamt: {len(all_events)} Events", progress=45) + + # --- download media --- + yield _log("info", "Lade Mediendateien …", progress=46) + + media_map: dict[str, str] = {} + media_items = [e for e in all_events if _has_media(e)] + media_total = len(media_items) + media_errors = 0 + media_purged = 0 + + async with aiohttp.ClientSession() as http_session: + for i, ev in enumerate(media_items): + eid = ev.get("event_id", "") + fname = await self._download_media(ev, media_dir, http_session) + if fname == MEDIA_PURGED: + media_purged += 1 + elif fname: + media_map[eid] = fname + else: + media_errors += 1 + + if (i + 1) % 10 == 0 or i == media_total - 1: + yield _log("progress", + f"Medien: {i + 1}/{media_total} ({len(media_map)} OK, " + f"{media_purged} gelöscht, {media_errors} Fehler)", + progress=int(46 + 40 * (i + 1) / max(media_total, 1))) + + parts = [f"{len(media_map)}/{media_total} Medien heruntergeladen"] + if media_purged: + parts.append(f"{media_purged} nicht mehr verfügbar (Server-Löschpolicy)") + if media_errors: + parts.append(f"{media_errors} Fehler") + yield _log("info", ", ".join(parts), progress=88) + + # --- get room name + members --- + room_name = room_id + member_names: dict[str, str] = {} + try: + headers = {"Authorization": f"Bearer {self.access_token}"} + encoded_rid = quote(room_id, safe="") + async with aiohttp.ClientSession() as s: + async with s.get( + f"{self.homeserver}/_matrix/client/v3/rooms/{encoded_rid}/state", + headers=headers, + ) as r: + if r.status == 200: + for st in await r.json(): + t = st.get("type", "") + c = st.get("content", {}) + if t == "m.room.name": + room_name = c.get("name", room_id) + elif t == "m.room.member" and c.get("membership") == "join": + uid = st.get("state_key", "") + member_names[uid] = c.get("displayname", uid.split(":")[0].lstrip("@")) + except Exception: + pass + + if room_name == room_id and member_names: + others = [n for uid, n in member_names.items() if uid != self.user_id] + if others: + room_name = ", ".join(others[:3]) + + # --- generate HTML --- + yield _log("info", "Generiere HTML …", progress=90) + html_path = room_dir / "chat.html" + await self._generate_html(all_events, media_map, member_names, room_name, self.user_id or "", html_path) + + # --- create ZIP --- + yield _log("info", "Erstelle ZIP-Archiv …", progress=95) + zip_result = self.create_zip(safe_name) + zip_api_path = f"/api/exports/{safe_name}/zip" if zip_result else None + + # --- save state --- + state = { + "room_id": room_id, + "room_name": room_name, + "total_events": len(all_events), + "total_media": len(media_map), + "last_export": datetime.now().isoformat(), + "last_token": from_token, + } + state_file.write_text(json.dumps(state, indent=2)) + + rel_path = f"/exports/{safe_name}/chat.html" + yield _log("done", + f"Export abgeschlossen! {len(all_events)} Nachrichten, {len(media_map)} Medien", + progress=100, html_path=rel_path, zip_path=zip_api_path) + + # ------------------------------------------------------------------ + # Media download + # ------------------------------------------------------------------ + + async def _download_media(self, ev: dict, media_dir: Path, + session: aiohttp.ClientSession | None = None) -> str | None: + content = ev.get("content", {}) + file_info = content.get("file", {}) + encrypted = bool(file_info and file_info.get("url")) + + mxc_url = file_info.get("url", "") if encrypted else content.get("url", "") + if not mxc_url or not mxc_url.startswith("mxc://"): + mxc_url = content.get("avatar_url") or "" + if not mxc_url or not mxc_url.startswith("mxc://"): + return None + encrypted = False + + parts = mxc_url[6:].split("/", 1) + if len(parts) != 2: + return None + server_name, media_id = parts + + mimetype = content.get("info", {}).get("mimetype", "") + ext = EXT_MAP.get(mimetype, "") + filename = f"{media_id}{ext}" + filepath = media_dir / filename + + if filepath.exists(): + return filename + + data = None + eid = ev.get("event_id", "?") + + try: + resp = await self.client.download(mxc_url) + if isinstance(resp, DownloadResponse): + data = resp.body + elif isinstance(resp, DownloadError): + logger.debug("nio download fehlgeschlagen für %s (event %s): %s", mxc_url, eid, resp.message) + except Exception as e: + logger.debug("nio download exception für %s (event %s): %s", mxc_url, eid, e) + + if data is None: + headers = {"Authorization": f"Bearer {self.access_token}"} + endpoints = [ + f"/_matrix/client/v1/media/download/{server_name}/{media_id}", + f"/_matrix/media/v3/download/{server_name}/{media_id}", + ] + own_session = session is None + s = session or aiohttp.ClientSession() + ep_404_count = 0 + try: + for ep in endpoints: + try: + async with s.get(f"{self.homeserver}{ep}", headers=headers) as r: + if r.status == 200: + data = await r.read() + break + elif r.status == 404: + ep_404_count += 1 + logger.debug("HTTP 404 für %s%s (event %s)", self.homeserver, ep, eid) + else: + logger.debug("HTTP %s für %s%s (event %s)", r.status, self.homeserver, ep, eid) + except Exception as e: + logger.debug("HTTP exception für %s%s (event %s): %s", self.homeserver, ep, eid, e) + continue + finally: + if own_session: + await s.close() + + if data is None and ep_404_count == len(endpoints): + logger.info("Media gelöscht (404 auf allen Endpoints): %s (event %s)", mxc_url, eid) + return MEDIA_PURGED + + if data is None: + logger.warning("Media-Download fehlgeschlagen: %s (event %s)", mxc_url, eid) + return None + + if encrypted and file_info: + try: + from nio.crypto.attachments import decrypt_attachment + data = decrypt_attachment( + data, + file_info["key"]["k"], + file_info["hashes"]["sha256"], + file_info["iv"], + ) + except Exception as e: + logger.warning("Entschlüsselung fehlgeschlagen für %s: %s", mxc_url, e) + return None + + async with aiofiles.open(filepath, "wb") as f: + await f.write(data) + return filename + + # ------------------------------------------------------------------ + # Preprocessing for client-side rendering + # ------------------------------------------------------------------ + + def _preprocess_messages(self, events: list[dict], media_map: dict[str, str], + member_names: dict[str, str], own_user_id: str) -> list[dict]: + messages: list[dict] = [] + last_date = "" + + for ev in events: + ts = ev.get("origin_server_ts", 0) + dt = _ts_to_datetime(ts) + date_str = dt.strftime("%A, %d. %B %Y") + time_str = dt.strftime("%H:%M") + etype = ev.get("type", "") + content = ev.get("content", {}) + + if date_str != last_date: + messages.append({"t": "date", "text": date_str}) + last_date = date_str + + if etype == "m.room.message": + sender = ev.get("sender", "") + display = member_names.get(sender, sender.split(":")[0].lstrip("@")) + msgtype = content.get("msgtype", "m.text") + eid = ev.get("event_id", "") + mfile = media_map.get(eid, "") + + msg: dict = { + "t": "msg", + "s": sender, + "n": display, + "o": sender == own_user_id, + "ts": time_str, + "mt": msgtype, + "b": content.get("body", ""), + "c": _color_for_user(sender), + } + + fb = content.get("formatted_body", "") + if fb: + msg["fb"] = fb + if mfile: + msg["m"] = mfile + if ev.get("unsigned", {}).get("redacted_because"): + msg["del"] = True + if msgtype == "m.audio": + if "org.matrix.msc3245.voice" in content or "org.matrix.msc1767.audio" in content: + msg["voice"] = True + if msgtype == "m.file": + msg["fs"] = _format_size(content.get("info", {}).get("size", 0)) + if msgtype == "m.location": + msg["geo"] = content.get("geo_uri", "") + + messages.append(msg) + + elif etype == "m.room.encrypted": + sender = ev.get("sender", "") + messages.append({ + "t": "enc", + "o": sender == own_user_id, + "ts": time_str, + }) + + elif etype == "m.room.member": + membership = content.get("membership", "") + display = content.get("displayname", ev.get("state_key", "?")) + prev = ev.get("unsigned", {}).get("prev_content", {}) + prev_membership = prev.get("membership", "") + + if membership == "join" and prev_membership == "join": + old_name = prev.get("displayname") or "" + new_name = content.get("displayname") or "" + old_avatar = prev.get("avatar_url") or "" + new_avatar = content.get("avatar_url") or "" + + if old_name != new_name: + if old_name and new_name: + messages.append({"t": "sys", "text": f"{old_name} heißt jetzt {new_name}"}) + elif new_name: + messages.append({"t": "sys", "text": f"{ev.get('state_key', '?')} heißt jetzt {new_name}"}) + + if old_avatar != new_avatar: + eid = ev.get("event_id", "") + mfile = media_map.get(eid, "") + text = f"{display} hat das Profilbild geändert" if new_avatar else f"{display} hat das Profilbild entfernt" + messages.append({"t": "avatar", "text": text, "m": mfile, "ts": time_str}) + + elif membership == "join": + messages.append({"t": "sys", "text": f"{display} ist beigetreten"}) + elif membership == "leave": + sender = ev.get("sender", "") + if sender != ev.get("state_key", ""): + messages.append({"t": "sys", "text": f"{display} wurde entfernt"}) + else: + messages.append({"t": "sys", "text": f"{display} hat den Raum verlassen"}) + elif membership == "invite": + messages.append({"t": "sys", "text": f"{display} wurde eingeladen"}) + elif membership == "ban": + messages.append({"t": "sys", "text": f"{display} wurde gebannt"}) + + elif etype == "m.room.topic": + topic = content.get("topic", "") + if topic: + messages.append({"t": "sys", "text": f"Thema geändert: {topic}"}) + else: + messages.append({"t": "sys", "text": "Thema entfernt"}) + + elif etype == "m.room.name": + rname = content.get("name", "") + if rname: + messages.append({"t": "sys", "text": f"Raumname geändert zu: {rname}"}) + + elif etype == "m.room.avatar": + eid = ev.get("event_id", "") + mfile = media_map.get(eid, "") + messages.append({"t": "avatar", "text": "Raumbild wurde geändert", "m": mfile, "ts": time_str}) + + elif etype == "m.sticker": + sender = ev.get("sender", "") + eid = ev.get("event_id", "") + mfile = media_map.get(eid, "") + if mfile: + messages.append({ + "t": "sticker", + "o": sender == own_user_id, + "ts": time_str, + "b": content.get("body", "Sticker"), + "m": mfile, + }) + + return messages + + # ------------------------------------------------------------------ + # HTML generation (client-side rendered) + # ------------------------------------------------------------------ + + async def _generate_html( + self, + events: list[dict], + media_map: dict[str, str], + member_names: dict[str, str], + room_name: str, + own_user_id: str, + output_path: Path, + ): + messages = self._preprocess_messages(events, media_map, member_names, own_user_id) + + total = len(events) + first_ts = events[0].get("origin_server_ts", 0) if events else 0 + last_ts = events[-1].get("origin_server_ts", 0) if events else 0 + first_date = _ts_to_datetime(first_ts).strftime("%d.%m.%Y") if first_ts else "" + last_date = _ts_to_datetime(last_ts).strftime("%d.%m.%Y") if last_ts else "" + now = datetime.now().strftime("%d.%m.%Y %H:%M") + + data = { + "roomName": room_name, + "total": total, + "firstDate": first_date, + "lastDate": last_date, + "exportDate": now, + "messages": messages, + } + + json_str = json.dumps(data, ensure_ascii=False).replace(" Path | None: + room_dir = self.export_path / room_dir_name + if not room_dir.exists(): + return None + + html_file = room_dir / "chat.html" + if not html_file.exists(): + return None + + zip_path = room_dir / "export.zip" + + with zipfile.ZipFile(zip_path, "w") as zf: + zf.write(html_file, "chat.html", compress_type=zipfile.ZIP_DEFLATED) + + media_dir = room_dir / "media" + if media_dir.exists(): + for fpath in sorted(media_dir.iterdir()): + if fpath.is_file(): + zf.write(fpath, f"media/{fpath.name}", compress_type=zipfile.ZIP_STORED) + + return zip_path + + # ------------------------------------------------------------------ + # Existing exports + # ------------------------------------------------------------------ + + def delete_export(self, room_dir_name: str) -> bool: + room_dir = self.export_path / room_dir_name + if not room_dir.exists(): + return False + shutil.rmtree(room_dir) + return True + + def list_exports(self) -> list[dict]: + exports = [] + if not self.export_path.exists(): + return exports + for d in sorted(self.export_path.iterdir()): + if d.is_dir() and (d / "chat.html").exists(): + state = {} + sf = d / ".state.json" + if sf.exists(): + try: + state = json.loads(sf.read_text()) + except Exception: + pass + exports.append({ + "dir_name": d.name, + "room_id": state.get("room_id", d.name), + "room_name": state.get("room_name", d.name), + "last_export": state.get("last_export", "?"), + "total_events": state.get("total_events", 0), + "total_media": state.get("total_media", 0), + "html_path": f"/exports/{d.name}/chat.html", + "zip_path": f"/api/exports/{d.name}/zip", + }) + return exports + + +# ====================================================================== +# Helper functions +# ====================================================================== + +def _log(typ: str, msg: str, progress: int = 0, **extra) -> dict: + d = {"type": typ, "message": msg, "progress": progress} + d.update(extra) + return d + + +def _esc(text: str) -> str: + return (text + .replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """)) + + +def _has_media(ev: dict) -> bool: + c = ev.get("content", {}) + if c.get("url") or c.get("file", {}).get("url"): + return True + if ev.get("type") == "m.room.member" and (c.get("avatar_url") or "").startswith("mxc://"): + prev = ev.get("unsigned", {}).get("prev_content", {}) + if c.get("avatar_url") != prev.get("avatar_url", ""): + return True + return False diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..da12632 --- /dev/null +++ b/app/main.py @@ -0,0 +1,242 @@ +import asyncio +import json +import logging +import os +from pathlib import Path + +from fastapi import FastAPI, Request, UploadFile, File, Form, HTTPException +from fastapi.responses import HTMLResponse, StreamingResponse, JSONResponse, FileResponse +from fastapi.staticfiles import StaticFiles + +from app.exporter import MatrixExporter + +LOG_LEVEL = os.environ.get("LOG_LEVEL", "INFO").upper() +logging.basicConfig(level=getattr(logging, LOG_LEVEL, logging.INFO), + format="%(asctime)s %(levelname)s %(name)s: %(message)s") +logger = logging.getLogger(__name__) + +EXPORTS_DIR = Path("/app/exports") +STORE_DIR = Path("/app/store") +EXPORTS_DIR.mkdir(parents=True, exist_ok=True) +STORE_DIR.mkdir(parents=True, exist_ok=True) + +app = FastAPI(title="Matrix Chat Export") + +exporter = MatrixExporter(store_path=str(STORE_DIR), export_path=str(EXPORTS_DIR)) + +export_tasks: dict[str, asyncio.Task] = {} +export_logs: dict[str, list[dict]] = {} + + +# ------------------------------------------------------------------ +# Web UI +# ------------------------------------------------------------------ + +@app.get("/", response_class=HTMLResponse) +async def index(): + html_path = Path(__file__).parent / "templates" / "index.html" + return HTMLResponse(html_path.read_text(encoding="utf-8")) + + +# ------------------------------------------------------------------ +# API: Connection +# ------------------------------------------------------------------ + +@app.post("/api/connect") +async def connect(request: Request): + data = await request.json() + method = data.get("method", "token") + homeserver = data.get("homeserver", "").strip().rstrip("/") + + if not homeserver: + raise HTTPException(400, "Homeserver URL fehlt") + + try: + if method == "password": + username = data.get("username", "").strip() + password = data.get("password", "") + if not username or not password: + raise HTTPException(400, "Benutzername und Passwort erforderlich") + result = await exporter.connect_password(homeserver, username, password) + else: + token = data.get("token", "").strip() + if not token: + raise HTTPException(400, "Access Token erforderlich") + result = await exporter.connect_token(homeserver, token) + + return result + except Exception as e: + raise HTTPException(400, str(e)) + + +@app.get("/api/session") +async def check_session(): + result = await exporter.restore_session() + if result: + return result + return JSONResponse({"connected": False}, status_code=200) + + +@app.post("/api/disconnect") +async def disconnect(): + await exporter.disconnect() + creds = STORE_DIR / "credentials.json" + if creds.exists(): + creds.unlink() + return {"ok": True} + + +# ------------------------------------------------------------------ +# API: E2E Keys +# ------------------------------------------------------------------ + +@app.post("/api/import-keys") +async def import_keys(file: UploadFile = File(...), passphrase: str = Form(...)): + if not exporter.client: + raise HTTPException(400, "Nicht verbunden") + + key_path = STORE_DIR / "imported_keys.txt" + content = await file.read() + key_path.write_bytes(content) + + try: + result = await exporter.import_keys(str(key_path), passphrase) + return result + except Exception as e: + raise HTTPException(400, f"Import fehlgeschlagen: {e}") + + +# ------------------------------------------------------------------ +# API: Rooms +# ------------------------------------------------------------------ + +@app.get("/api/rooms") +async def list_rooms(): + if not exporter.access_token: + raise HTTPException(400, "Nicht verbunden") + try: + rooms = await exporter.list_rooms() + return rooms + except Exception as e: + raise HTTPException(500, str(e)) + + +@app.get("/api/rooms/stream") +async def rooms_stream(): + if not exporter.access_token: + raise HTTPException(400, "Nicht verbunden") + + async def stream(): + try: + async for item in exporter.list_rooms_stream(): + yield f"data: {json.dumps(item, ensure_ascii=False)}\n\n" + yield f"data: {json.dumps({'type': 'done'})}\n\n" + except Exception as e: + yield f"data: {json.dumps({'type': 'error', 'message': str(e)})}\n\n" + + return StreamingResponse(stream(), media_type="text/event-stream", headers={ + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + }) + + +# ------------------------------------------------------------------ +# API: Export +# ------------------------------------------------------------------ + +@app.post("/api/export") +async def start_export(request: Request): + data = await request.json() + room_id = data.get("room_id", "") + if not room_id: + raise HTTPException(400, "room_id fehlt") + + if room_id in export_tasks and not export_tasks[room_id].done(): + raise HTTPException(409, "Export läuft bereits") + + export_logs[room_id] = [] + + async def run(): + try: + async for entry in exporter.export_room(room_id): + export_logs.setdefault(room_id, []).append(entry) + except Exception as e: + export_logs.setdefault(room_id, []).append({ + "type": "error", "message": str(e), "progress": 0, + }) + + task = asyncio.create_task(run()) + export_tasks[room_id] = task + return {"status": "started", "room_id": room_id} + + +@app.get("/api/export/progress/{room_id:path}") +async def export_progress(room_id: str): + async def stream(): + last_idx = 0 + while True: + logs = export_logs.get(room_id, []) + while last_idx < len(logs): + yield f"data: {json.dumps(logs[last_idx], ensure_ascii=False)}\n\n" + last_idx += 1 + + task = export_tasks.get(room_id) + if task and task.done(): + if last_idx >= len(export_logs.get(room_id, [])): + yield f"data: {json.dumps({'type': 'stream_end'})}\n\n" + break + + await asyncio.sleep(0.3) + + return StreamingResponse(stream(), media_type="text/event-stream", headers={ + "Cache-Control": "no-cache", + "X-Accel-Buffering": "no", + }) + + +@app.get("/api/exports") +async def list_exports(): + return exporter.list_exports() + + +@app.delete("/api/exports/{dir_name}") +async def delete_export(dir_name: str): + if not exporter.delete_export(dir_name): + raise HTTPException(404, "Export nicht gefunden") + return {"ok": True} + + +# ------------------------------------------------------------------ +# API: ZIP download +# ------------------------------------------------------------------ + +@app.get("/api/exports/{dir_name}/zip") +async def download_zip(dir_name: str): + zip_path = exporter.create_zip(dir_name) + if not zip_path or not zip_path.exists(): + raise HTTPException(404, "Export nicht gefunden") + + state_file = EXPORTS_DIR / dir_name / ".state.json" + filename = dir_name + if state_file.exists(): + try: + state = json.loads(state_file.read_text()) + room_name = state.get("room_name", dir_name) + safe = "".join(c for c in room_name if c.isalnum() or c in " -_äöüÄÖÜß").strip() + if safe: + filename = safe + except Exception: + pass + + return FileResponse( + str(zip_path), + media_type="application/zip", + filename=f"Chat-Export {filename}.zip", + ) + + +# ------------------------------------------------------------------ +# Serve exported files +# ------------------------------------------------------------------ + +app.mount("/exports", StaticFiles(directory=str(EXPORTS_DIR)), name="exports") diff --git a/app/templates/chat_export.html b/app/templates/chat_export.html new file mode 100644 index 0000000..5be9700 --- /dev/null +++ b/app/templates/chat_export.html @@ -0,0 +1,266 @@ + + + + + +Chat Export + + + +
+
+
💬
+
+

Lade...

+
+
+
+
+
Nachrichten werden geladen...
+
+
+ + + + + + + + diff --git a/app/templates/index.html b/app/templates/index.html new file mode 100644 index 0000000..2280cbf --- /dev/null +++ b/app/templates/index.html @@ -0,0 +1,654 @@ + + + + + +Matrix Chat Export + + + +
+

▩ Matrix Chat Export

+

Exportiere verschlüsselte Matrix-Chats als schöne HTML-Dateien mit Medien

+ + +
+

1 Verbinden

+ +
+
Passwort
+
Access Token
+
+ +
+
+ + +
+
+ + +
+
+ + +
+
+ + + +
+ + +
+
+
+ + +
+

2 E2E-Schlüssel importieren

+ +
+ Wichtig: Exportiere deine E2E-Schlüssel aus Element: + Einstellungen → Sicherheit → "E2E-Raumschlüssel exportieren". + Die exportierte Datei und das Passwort werden hier benötigt. +
+ +
+ + +
+
+ + +
+ +
+ + +
+
+
+ + +
+

3 Raum auswählen

+ +
+ +
+ + +
+
+ +
+ +
+
+ + + + + +
+

Vorherige Exports

+
Lade …
+
+
+ + + + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..8f46f5d --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,26 @@ +services: + matrix-chat-export: + build: . + container_name: matrix-chat-export + restart: unless-stopped + ports: + - "8092:8080" + volumes: + - ./data/exports:/app/exports + - ./data/store:/app/store + environment: + - TZ=Europe/Berlin + - LOG_LEVEL=INFO + networks: + matrixexport_net: + ipv4_address: 172.27.83.10 + +networks: + matrixexport_net: + name: matrixexport.dockernetwork.local + driver: bridge + ipam: + config: + - subnet: 172.27.83.0/24 + gateway: 172.27.83.1 + ip_range: 172.27.83.128/25 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..7105965 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +fastapi==0.115.6 +uvicorn[standard]==0.34.0 +matrix-nio[e2e]==0.25.2 +python-multipart==0.0.20 +aiofiles==24.1.0 +jinja2==3.1.5 +aiohttp==3.11.11